dovecot - Jun 2013 - doveadm index crashes when indexing shared mailboxes

Hi,

We store our mail archive in a tree of subfolders. I am trying to speed 
up text searching on our mail archive but when running "doveadm -D -v 
index -u neil shared/Exalon/Aandeelhouders" the following output is 
produced:

doveadm(neil): Debug: Loading modules from directory: 
/usr/lib/dovecot/modules
doveadm(neil): Debug: Module loaded: 
/usr/lib/dovecot/modules/lib20_fts_plugin.so
doveadm(neil): Debug: Module loaded: 
/usr/lib/dovecot/modules/lib21_fts_squat_plugin.so
doveadm(neil): Debug: Loading modules from directory: 
/usr/lib/dovecot/modules/doveadm
doveadm(neil): Debug: Skipping module doveadm_acl_plugin, because 
dlopen() failed: /usr/lib/dovecot/modules/doveadm/lib10_dovea 
dm_acl_plugin.so: undefined symbol: acl_user_module (this is usually 
intentional, so just ignore this message)
doveadm(neil): Debug: Skipping module doveadm_expire_plugin, because 
dlopen() failed: /usr/lib/dovecot/modules/doveadm/lib10_do 
veadm_expire_plugin.so: undefined symbol: expire_set_deinit (this is 
usually intentional, so just ignore this message)
doveadm(neil): Debug: Skipping module doveadm_quota_plugin, because 
dlopen() failed: /usr/lib/dovecot/modules/doveadm/lib10_dov 
eadm_quota_plugin.so: undefined symbol: quota_user_module (this is 
usually intentional, so just ignore this message)
doveadm(neil): Debug: Skipping module doveadm_zlib_plugin, because 
dlopen() failed: /usr/lib/dovecot/modules/doveadm/lib10_dove 
adm_zlib_plugin.so: undefined symbol: i_stream_create_deflate (this is 
usually intentional, so just ignore this message)
doveadm(neil): Debug: Module loaded: 
/usr/lib/dovecot/modules/doveadm/lib20_doveadm_fts_plugin.so
doveadm(neil): Debug: auth input: neil system_groups_user=neil uid=1000 
gid=1000 home=/home/neil
doveadm(neil): Debug: Effective uid=1000, gid=1000, home=/home/neil
doveadm(neil): Debug: Namespace inbox: type=private, prefix=, sep=, 
inbox=yes, hidden=no, list=yes, subscriptions=yes location= 
mbox:~/mail:INBOX=/var/mail/neil
doveadm(neil): Debug: fs: root=/home/neil/mail, index=, control=, 
inbox=/var/mail/neil, altdoveadm(neil): Debug: Namespace : type=shared,
prefix=shared/, sep=/,
inbox=no, hidden=no, list=yes, subscriptions=no location= mbox:~imapshared
doveadm(neil): Debug: mbox: INBOX defaulted to /home/imapshared/inbox
doveadm(neil): Debug: fs: root=/home/imapshared, index=, control=, 
inbox=/home/imapshared/inbox, altdoveadm(neil): Info:
shared/Exalon/Aandeelhouders: Caching mails seq=1..27
27/27
doveadm(neil): Panic: file mbox-storage.c: line 711 
(mbox_transaction_unlock): assertion failed: 
(mbox->box.transaction_count 
 >                                                       0 || 
mbox->mbox_lock_type == F_UNLCK)
doveadm(neil): Error: Raw backtrace: 
/usr/lib/dovecot/libdovecot.so.0(+0x3c0ba) [0x7fb91d7460ba] -> 
/usr/lib/dovecot/libdovecot .so.0(default_fatal_handler+0x2a) 
[0x7fb91d74617a] -> /usr/lib/dovecot/libdovecot.so.0(i_fatal+0) 
[0x7fb91d71d85a] -> /usr/lib/ dovecot/libdovecot-storage.so.0(+0x54d80) 
[0x7fb91d9e4d80] -> /usr/lib/dovecot/libdovecot-storage.so.0(+0x54dee) 
[0x7fb91d9e4de e] -> 
/usr/lib/dovecot/modules/lib20_fts_plugin.so(+0xb0e2) [0x7fb91c2c30e2] 
-> /usr/lib/dovecot/libdovecot-storage.so.0(mailbo 
x_transaction_commit_get_changes+0x3d) [0x7fb91da0010d] -> 
/usr/lib/dovecot/libdovecot-storage.so.0(mailbox_transaction_commit+ 
0x15) [0x7fb91da00145] -> 
/usr/lib/dovecot/modules/lib21_fts_squat_plugin.so(+0x3927) 
[0x7fb91c0ab927] -> /usr/lib/dovecot/modu 
les/lib21_fts_squat_plugin.so(+0x39b1) [0x7fb91c0ab9b1] -> 
/usr/lib/dovecot/modules/lib20_fts_plugin.so(+0x649c) [0x7fb91c2be49 c] 
-> 
/usr/lib/dovecot/modules/lib20_fts_plugin.so(fts_backend_update_deinit+0x2b) 
[0x7fb91c2be59b] -> /usr/lib/dovecot/modules 
/lib20_fts_plugin.so(+0xb03c) [0x7fb91c2c303c] -> 
/usr/lib/dovecot/modules/lib20_fts_plugin.so(+0xb0d3) [0x7fb91c2c30d3] 
-> /us 
r/lib/dovecot/libdovecot-storage.so.0(mailbox_transaction_commit_get_changes+0x3d)
[0x7fb91da0010d] -> /usr/lib/dovecot/libdove 
cot-storage.so.0(mailbox_transaction_commit+0x15) [0x7fb91da00145] -> 
/usr/bin/doveadm(+0x15430) [0x7fb91e0fb430] -> /usr/bin/d 
oveadm(+0x11f66) [0x7fb91e0f7f66] -> /usr/bin/doveadm(+0x12a41) 
[0x7fb91e0f8a41] -> /usr/bin/doveadm(doveadm_mail_try_run+0x161 ) 
[0x7fb91e0f8f01] -> /usr/bin/doveadm(main+0x3d1) [0x7fb91e0f7b21] -> 
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf5) [0x7fb91d363ea5] 
-> /usr/bin/doveadm(+0x11d55) [0x7fb91e0f7d55]
Aborted (core dumped)


When indexing is triggered using the imap server it logs:

Jun 10 12:32:16 indexer: Error: Indexer worker disconnected, discarding 
1 requests for neil
Jun 10 12:32:16 indexer-worker(neil): Fatal: master: 
service(indexer-worker): child 22173 killed with signal 6 (core dumps 
disabled)
Jun 10 12:32:16 imap(neil): Error: indexer failed to index mailbox 
shared/Suppliers/NXP
Jun 10 12:32:16 indexer-worker: Debug: Loading modules from directory: 
/usr/lib/dovecot/modules
Jun 10 12:32:16 indexer-worker: Debug: Module loaded: 
/usr/lib/dovecot/modules/lib20_fts_plugin.so
Jun 10 12:32:16 indexer-worker: Debug: Module loaded: 
/usr/lib/dovecot/modules/lib21_fts_squat_plugin.so
Jun 10 12:32:16 indexer-worker: Debug: auth input: neil 
system_groups_user=neil uid=1000 gid=1000 home=/home/neil
Jun 10 12:32:16 indexer-worker(neil): Debug: Effective uid=1000, 
gid=1000, home=/home/neil
Jun 10 12:32:16 indexer-worker(neil): Debug: Namespace inbox: 
type=private, prefix=, sep=, inbox=yes, hidden=no, list=yes, 
subscriptions=yes location=mbox:~/mail:INBOX=/var/mail/nei
l
Jun 10 12:32:16 indexer-worker(neil): Debug: fs: root=/home/neil/mail, 
index=, control=, inbox=/var/mail/neil, altJun 10 12:32:16 indexer-worker(neil):
Debug: Namespace : type=shared,
prefix=shared/, sep=/, inbox=no, hidden=no, list=yes, subscriptions=no 
location=mbox:~imapshared
Jun 10 12:32:16 indexer-worker(neil): Debug: mbox: INBOX defaulted to 
/home/imapshared/inbox
Jun 10 12:32:16 indexer-worker(neil): Debug: fs: root=/home/imapshared, 
index=, control=, inbox=/home/imapshared/inbox, altJun 10 12:32:16
indexer-worker(neil): Panic: file mbox-storage.c: line
711 (mbox_transaction_unlock): assertion failed: 
(mbox->box.transaction_count > 0 || mbox->mbox_lock_type == F
_UNLCK)
Jun 10 12:32:16 indexer-worker(neil): Error: Raw backtrace: 
/usr/lib/dovecot/libdovecot.so.0(+0x3c0ba) [0x7f9da1ef60ba] -> 
/usr/lib/dovecot/libdovecot.so.0(+0x3c0fe) [0x7f9da1ef60fe
] -> /usr/lib/dovecot/libdovecot.so.0(i_fatal+0) [0x7f9da1ecd85a] -> 
/usr/lib/dovecot/libdovecot-storage.so.0(+0x54d80) [0x7f9da2194d80] -> 
/usr/lib/dovecot/libdovecot-storage.so.0(
+0x54dee) [0x7f9da2194dee] -> 
/usr/lib/dovecot/modules/lib20_fts_plugin.so(+0xb0e2) [0x7f9da0a730e2] 
-> 
/usr/lib/dovecot/libdovecot-storage.so.0(mailbox_transaction_commit_get_chang
es+0x3d) [0x7f9da21b010d] -> 
/usr/lib/dovecot/libdovecot-storage.so.0(mailbox_transaction_commit+0x15)
[0x7f9da21b0145]
-> /usr/lib/dovecot/modules/lib21_fts_squat_plugin.so(+0x3927
) [0x7f9da085b927] -> 
/usr/lib/dovecot/modules/lib21_fts_squat_plugin.so(+0x39b1) 
[0x7f9da085b9b1] -> 
/usr/lib/dovecot/modules/lib20_fts_plugin.so(+0x649c) [0x7f9da0a6e49c] 
-> /usr/
lib/dovecot/modules/lib20_fts_plugin.so(fts_backend_update_deinit+0x2b) 
[0x7f9da0a6e59b] -> 
/usr/lib/dovecot/modules/lib20_fts_plugin.so(+0xb03c) [0x7f9da0a7303c] 
-> /usr/lib/doveco
t/modules/lib20_fts_plugin.so(+0xb0d3) [0x7f9da0a730d3] -> 
/usr/lib/dovecot/libdovecot-storage.so.0(mailbox_transaction_commit_get_changes+0x3d)
[0x7f9da21b010d] -> /usr/lib/dovecot
/libdovecot-storage.so.0(mailbox_transaction_commit+0x15) 
[0x7f9da21b0145] -> dovecot/indexer-worker(+0x287c) [0x7f9da265f87c] -> 
/usr/lib/dovecot/libdovecot.so.0(io_loop_call_io+0x
36) [0x7f9da1f02f76] -> 
/usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run+0xa7) 
[0x7f9da1f03c37] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_run+0x28) 
[0x7f9da1f02988] -> /usr/li
b/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7f9da1eef3c3] -> 
dovecot/indexer-worker(main+0xfe) [0x7f9da265f10e] -> 
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf5)
[0x7f9da1b13ea5] -> dovecot/indexer-worker(+0x21b5) [0x7f9da265f1b5]

neil at athina:~$ dovecot -n
# 2.1.7: /etc/dovecot/dovecot.conf
# OS: Linux 3.8.0-23-generic x86_64 Ubuntu 13.04
auth_verbose = yes
disable_plaintext_auth = no
log_path = /var/log/dovecot.log
mail_debug = yes
mail_plugins = fts fts_squat
mail_privileged_group = mail
namespace {
   list = yes
   location = mbox:~imapshared
   prefix = shared/
   separator = /
   subscriptions = no
   type = shared
}
namespace inbox {
   inbox = yes
   location = mbox:~/mail:INBOX=/var/mail/%u
   mailbox Drafts {
     special_use = \Drafts
   }
   mailbox Junk {
     special_use = \Junk
   }
   mailbox Sent {
     special_use = \Sent
   }
   mailbox "Sent Messages" {
     special_use = \Sent
   }
   mailbox Trash {
     special_use = \Trash
   }
   prefix }
passdb {
   driver = pam
}
plugin {
   fts = squat
   fts_squat = partial=4 full=10
   sieve = ~/.dovecot.sieve
   sieve_dir = ~/sieve
}
protocols = " imap"
ssl_cert = </etc/ssl/certs/dovecot.pem
ssl_key = </etc/ssl/private/dovecot.pem
userdb {
   driver = passwd
}
verbose_ssl = yes
protocol imap {
   imap_client_workarounds = tb-extra-mailbox-sep tb-lsub-flags
}

-- 
Willem-Jan de Hoog

On 10.6.2013, at 14.56, W. de Hoog <wdehoog at exalondelft.nl> wrote:
> We store our mail archive in a tree of subfolders. I am trying to speed up
text searching on our mail archive but when running "doveadm -D -v index -u
neil shared/Exalon/Aandeelhouders" the following output is produced:
..
> doveadm(neil): Panic: file mbox-storage.c: line 711
(mbox_transaction_unlock): assertion failed: (mbox->box.transaction_count
>                                                       0 ||
mbox->mbox_lock_type == F_UNLCK)
mbox is a pretty difficult format, and I wouldn't recommend using it for
anything else than small simple legacy setups.
> # 2.1.7: /etc/dovecot/dovecot.conf
If this happens also with v2.2 I could look into it.