dovecot - Feb 2015 - Dovecot 2.2.15, Panic: array.h: line 197 (array_idx_i): assertion failed: (idx * array->element_size < array->buffer->used)

Hi all

Version: 2.2.15
OS: SLES11 SP3 x86_64

Problem with replication for some users:

Log Server 1:
Feb 24 09:29:42 v063ik dovecot: dsync-local(user at domain.com): Error:
read(v063il.pnet.ch) failed: <no error>

Log Server 2:
Feb 24 09:29:42 v063il dovecot: dsync-server(user at domain.com): Panic: file
../../../src/lib/array.h: line 197 (array_idx_i): assertion failed: (idx *
array->element_size < array->buffer->used)
Feb 24 09:29:42 v063il dovecot: dsync-server(user at domain.com): Error: Raw
backtrace: /usr/lib64/dovecot/libdovecot.so.0(+0x75f63) [0x7fac77eeaf63] ->
/usr/lib64/dovecot/libdovecot.so.0(+0x75ff6) [0x7fac77eeaff6] ->
/usr/lib64/dovecot/libdovecot.so.0(+0x27595) [0x7fac77e9c595] ->
/usr/lib64/dovecot/modules/lib01_acl_plugin.so(+0x71ba) [0x7fac772c41ba] ->
/usr/lib64/dovecot/modules/lib01_acl_plugin.so(acl_attribute_iter_next+0x64)
[0x7fac772c5bb4] -> /usr/lib64/dovecot/libdovecot-storage.so.0(+0xb11ca)
[0x7fac782071ca] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(index_storage_mailbox_delete_pre+0xd8)
[0x7fac782074a8] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(index_storage_mailbox_delete+0x9)
[0x7fac782075a9] ->
/usr/lib64/dovecot/modules/lib15_notify_plugin.so(+0x2080) [0x7fac76eaa080]
-> /usr/lib64/dovecot/modules/lib01_acl_plugin.so(+0xea66) [0x7fac772cba66]
-> /usr/lib64/dovecot/libdovecot-storage.so.0(mailbox_delete+0x2f)
[0x7fac781db70f] -> dovecot/doveadm-server [10.1.100.49 user at domain.com
recv_mailbox_tree_deletes](dsync_brain_mailbox_tree_sync_change+0x319)
[0x7fac786b6249] -> dovecot/doveadm-server [10.1.100.49 user at domain.com
recv_mailbox_tree_deletes](dsync_brain_recv_mailbox_tree_deletes+0x212)
[0x7fac786b54d2] -> dovecot/doveadm-server [10.1.100.49 user at domain.com
recv_mailbox_tree_deletes](dsync_brain_run+0x5eb) [0x7fac786b28db] ->
dovecot/doveadm-server [10.1.100.49 user at domain.com
recv_mailbox_tree_deletes](+0x1ef7d) [0x7fac786b2f7d] ->
dovecot/doveadm-server [10.1.100.49 user at domain.com
recv_mailbox_tree_deletes](+0x3203f) [0x7fac786c603f] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0x49) [0x7fac77efcc29] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0xb4)
[0x7fac77efde24] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x9) [0x7fac77efccc9]
-> /usr/lib64/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7fac77efcd58]
-> dovecot/doveadm-server [10.1.100.49 user at domain.com
recv_mailbox_tree_deletes](+0x104c0) [0x7fac786a44c0] ->
dovecot/doveadm-server [10.1.100.49 user at domain.com
recv_mailbox_tree_deletes](+0x135c3) [0x7fac786a75c3] ->
dovecot/doveadm-server [10.1.100.49 user at domain.com
recv_mailbox_tree_deletes](+0x1d3d2) [0x7fac786b13d2] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0x49) [0x7fac77efcc29] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0xb4)
[0x7fac77efde24] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x9) [0x7fac77efccc9]
-> /usr/lib64/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7fac77efcd58]
-> /usr/lib64/dovecot/libdovecot.so.0(master_service_run+0x13)
[0x7fac77ea14d3]
Feb 24 09:29:42 v063il dovecot: dsync-server(user at domain.com): Fatal: master:
service(doveadm): child 12663 killed with signal 6 (core dumped)

Backtrace of core dump of server 2:
v063il:/usr/lib64/dovecot # gdb /usr/lib/dovecot/doveadm-server
/data/vmail/epost.ch/1003539/core
GNU gdb (GDB) SUSE (7.5.1-0.7.29)
Copyright (C) 2012 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later
<http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show
copying"
and "show warranty" for details.
This GDB was configured as "x86_64-suse-linux".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/lib/dovecot/doveadm-server...done.
[New LWP 12509]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Missing separate debuginfo for /lib64/libbz2.so.1
Try: zypper install -C
"debuginfo(build-id)=9d16aacb0b213c6ff10b2f162549f2cefee24697"
Missing separate debuginfo for /lib64/libgcc_s.so.1
Try: zypper install -C
"debuginfo(build-id)=3b149eccd897f1f37dce50ad22614043eba757a2"
Core was generated by `dovecot/doveadm-server'.
Program terminated with signal 6, Aborted.
#0  0x00007fe4cc640885 in *__GI_raise (sig=<optimized out>) at
../nptl/sysdeps/unix/sysv/linux/raise.c:64
64      ../nptl/sysdeps/unix/sysv/linux/raise.c: No such file or directory.
(gdb) bt full
#0  0x00007fe4cc640885 in *__GI_raise (sig=<optimized out>) at
../nptl/sysdeps/unix/sysv/linux/raise.c:64
        pid = <optimized out>
        selftid = <optimized out>
#1  0x00007fe4cc641e61 in *__GI_abort () at abort.c:92
        act = {__sigaction_handler = {sa_handler = 0x1, sa_sigaction = 0x1},
sa_mask = {__val = {140620662524321, 32, 4294967295, 33, 140620662296361,
              140620658199328, 140620670309600, 206158430224, 4294967295,
140736446740304, 157882997, 2994704, 140620661813248, 4294967295, 33, 0}},
          sa_flags = -856089480, sa_restorer = 0x7fe400000001}
        sigs = {__val = {32, 0 <repeats 15 times>}}
#2  0x00007fe4cc9fcf82 in default_fatal_finish (type=<optimized out>,
status=0) at failures.c:202
        backtrace = 0x7fe4cee02c90
"/usr/lib64/dovecot/libdovecot.so.0(+0x75f63) [0x7fe4cc9fcf63] ->
/usr/lib64/dovecot/libdovecot.so.0(+0x75ff6) [0x7fe4cc9fcff6] ->
/usr/lib64/dovecot/libdovecot.so.0(+0x27595) [0x7fe4cc9ae595] ->
/usr/"...
#3  0x00007fe4cc9fcff6 in i_internal_fatal_handler (ctx=0x7fffc1ea3890,
format=<optimized out>, args=<optimized out>) at failures.c:666
        status = 0
#4  0x00007fe4cc9ae595 in i_panic (format=<optimized out>) at
failures.c:276
        ctx = {type = LOG_TYPE_PANIC, exit_status = 0, timestamp = 0x0,
timestamp_usecs = 0}
        args = {{gp_offset = 40, fp_offset = 48, overflow_arg_area =
0x7fffc1ea3970, reg_save_area = 0x7fffc1ea38b0}}
#5  0x00007fe4cbdd61ba in array_idx_i (idx=<optimized out>,
array=<optimized out>) at ../../../src/lib/array.h:197
        __FUNCTION__ = "array_idx_i"
#6  acl_default_object_list_next (iter=<optimized out>,
rights_r=<optimized out>) at acl-api.c:216
No locals.
#7  0x00007fe4cbdd7bb4 in acl_attribute_iter_next_acl (aiter=<optimized
out>) at acl-attributes.c:193
No locals.
#8  acl_attribute_iter_next (iter=0x7fe4cede87f0) at acl-attributes.c:219
        key = 0x0
#9  0x00007fe4ccd191ca in mailbox_delete_all_attributes (t=0x7fe4ceddbf80,
type=MAIL_ATTRIBUTE_TYPE_SHARED) at index-storage.c:632
        iter = 0x7fe4cede87f0
        key = <optimized out>
        ret = 0
#10 0x00007fe4ccd194a8 in mailbox_expunge_all_data (box=<optimized out>)
at index-storage.c:670
No locals.
#11 index_storage_mailbox_delete_pre (box=0x7fe4cede5cb0) at index-storage.c:707
        status = {messages = 3435634784, recent = 32740, unseen = 3470679216,
uidvalidity = 32740, uidnext = 3470679216, first_unseen_seq = 32740,
          first_recent_uid = 1, last_cached_seq = 0, highest_modseq =
140620699884664, highest_pvt_modseq = 140620668173509, keywords = 0x0,
          permanent_flags = MAIL_FLAGGED, permanent_keywords = 0,
allow_new_keywords = 0, nonpermanent_modseqs = 0, no_modseq_tracking = 0,
have_guids = 0,
          have_save_guids = 0, have_only_guid128 = 0}
#12 0x00007fe4ccd195a9 in index_storage_mailbox_delete (box=0x30dd) at
index-storage.c:764
        ret = <optimized out>
#13 0x00007fe4cb9bc080 in notify_mailbox_delete (box=0x7fe4cede5cb0) at
notify-storage.c:231
        lbox = 0x7fe4cede65f0
#14 0x00007fe4cbddda66 in acl_mailbox_delete (box=0x7fe4cede5cb0) at
acl-mailbox.c:190
        ret = 0
#15 0x00007fe4ccced70f in mailbox_delete (box=0x7fe4cede5cb0) at
mail-storage.c:1357
        ret = <optimized out>
#16 0x00007fe4cd1c8249 in dsync_brain_mailbox_tree_sync_change
(brain=0x7fe4cedbc7d0, change=0x7fe4cedd5c60) at
dsync-brain-mailbox-tree-sync.c:182
        box = 0x7fe4cede5cb0
        destbox = <optimized out>
        errstr = <optimized out>
        func_name = 0x7fe4cd1dedf5 "mailbox_delete"
        storage_name = <optimized out>
        error = <optimized out>
        ret = 1
        __FUNCTION__ = "dsync_brain_mailbox_tree_sync_change"
#17 0x00007fe4cd1c74d2 in dsync_brain_mailbox_trees_sync (brain=<optimized
out>) at dsync-brain-mailbox-tree.c:295
No locals.
#18 dsync_brain_recv_mailbox_tree_deletes (brain=0x7fe4cedbc7d0) at
dsync-brain-mailbox-tree.c:471
        node = 0x0
        status = 0x7fe4cd1de985 "not found"
        deletes = 0x7fe4cedc32b0
        i = 30
        count = 30
        sep = <optimized out>
        __FUNCTION__ = "dsync_brain_recv_mailbox_tree_deletes"
#19 0x00007fe4cd1c48db in dsync_brain_run_real (changed_r=<optimized out>,
brain=<optimized out>) at dsync-brain.c:568
No locals.
#20 dsync_brain_run (brain=0x7fe4cedbc7d0, changed_r=0x7fffc1ea3d5f) at
dsync-brain.c:608
        _data_stack_cur_id = 5
        ret = true
#21 0x00007fe4cd1c4f7d in dsync_brain_run_io (context=<optimized out>) at
dsync-brain.c:89
        brain = 0x7fe4cedbc7d0
        changed = false
        try_pending = true
#22 0x00007fe4cd1d803f in dsync_ibc_stream_input (ibc=0x7fe4cedba7b0) at
dsync-ibc-stream.c:199
No locals.
#23 0x00007fe4cca0ec29 in io_loop_call_io (io=0x7fe4cedba660) at ioloop.c:498
        ioloop = 0x7fe4ced58110
        t_id = 4
        __FUNCTION__ = "io_loop_call_io"
#24 0x00007fe4cca0fe24 in io_loop_handler_run_internal (ioloop=<optimized
out>) at ioloop-epoll.c:220
        ctx = 0x7fe4ceda5920
        event = 0x7fe4ceda5a60
        list = 0x7fe4cedba640
        io = 0x30dd
        tv = {tv_sec = 59, tv_usec = 997329}
        msecs = <optimized out>
        ret = <optimized out>
        call = false
        __FUNCTION__ = "io_loop_handler_run_internal"
#25 0x00007fe4cca0ecc9 in io_loop_handler_run (ioloop=0x30dd) at ioloop.c:545
No locals.
#26 0x00007fe4cca0ed58 in io_loop_run (ioloop=0x7fe4ced58110) at ioloop.c:522
        __FUNCTION__ = "io_loop_run"
#27 0x00007fe4cd1b64c0 in cmd_dsync_server_run (_ctx=0x7fe4ceda4310,
user=0x7fe4cedb0d60) at doveadm-dsync.c:1043
        ibc = 0x7fe4cedba7b0
        brain = 0x7fe4cedbc7d0
        temp_prefix = <optimized out>
        state_str = 0x0
        sync_type = <optimized out>
        name = 0x7fe4ced7c730 "SSL 10.1.100.49"
        process_title_prefix = 0x7fe4ced46da8 "10.1.100.49 "
#28 0x00007fe4cd1b95c3 in doveadm_mail_next_user (ctx=0x7fe4ceda4310,
input=<optimized out>, error_r=0x7fffc1ea3f00) at doveadm-mail.c:314
        error = <optimized out>
        ip = <optimized out>
        ret = 0
        __FUNCTION__ = "doveadm_mail_next_user"
#29 0x00007fe4cd1c33d2 in doveadm_mail_cmd_server_run (input=<optimized
out>, ctx=<optimized out>, conn=<optimized out>) at
client-connection.c:123
No locals.
#30 client_handle_command (args=<optimized out>, conn=<optimized
out>) at client-connection.c:232
        ctx = 0x7fe4ceda4310
        flags = <optimized out>
        cmd_name = <optimized out>
        input = {module = 0x0, service = 0x7fe4cd1db44b "doveadm",
username = 0x7fe4ced44679 "user at domain.com", session_id = 0x0,
local_ip = {family = 2, u = {
              ip6 = {__in6_u = {__u6_addr8 = "\n\001d2",
'\000' <repeats 11 times>, __u6_addr16 = {266, 12900, 0, 0, 0, 0,
0, 0}, __u6_addr32 = {845414666, 0,
                    0, 0}}}, ip4 = {s_addr = 845414666}}}, remote_ip = {family =
2, u = {ip6 = {__in6_u = {__u6_addr8 = "\n\001d1", '\000'
<repeats 11 times>,
                  __u6_addr16 = {266, 12644, 0, 0, 0, 0, 0, 0}, __u6_addr32 =
{828637450, 0, 0, 0}}}, ip4 = {s_addr = 828637450}}}, local_port = 24042,
          remote_port = 43404, userdb_fields = 0x0, flags_override_add =
(unknown: 0), flags_override_remove = (unknown: 0), no_userdb_lookup = 0}
        argc = <optimized out>
#31 client_connection_input (conn=0x7fe4ced77dd0) at client-connection.c:345
        args = <optimized out>
        _data_stack_cur_id = 3
        line = <optimized out>
        ok = <optimized out>
#32 0x00007fe4cca0ec29 in io_loop_call_io (io=0x7fe4ced76d80) at ioloop.c:498
        ioloop = 0x7fe4ced4b730
        t_id = 2
        __FUNCTION__ = "io_loop_call_io"
#33 0x00007fe4cca0fe24 in io_loop_handler_run_internal (ioloop=<optimized
out>) at ioloop-epoll.c:220
        ctx = 0x7fe4ced57f80
        event = 0x7fe4ced50f10
        list = 0x7fe4ced76de0
        io = 0x30dd
        tv = {tv_sec = 2147483, tv_usec = 0}
        msecs = <optimized out>
        ret = <optimized out>
        call = false
        __FUNCTION__ = "io_loop_handler_run_internal"
#34 0x00007fe4cca0ecc9 in io_loop_handler_run (ioloop=0x30dd) at ioloop.c:545
No locals.
#35 0x00007fe4cca0ed58 in io_loop_run (ioloop=0x7fe4ced4b730) at ioloop.c:522
        __FUNCTION__ = "io_loop_run"
#36 0x00007fe4cc9b34d3 in master_service_run (service=0x7fe4ced4b5c0,
callback=0x30dd) at master-service.c:566
No locals.
#37 0x00007fe4cd1c3bbb in main (argc=1, argv=0x7fe4ced4b390) at main.c:92
        set_roots = {0x7fe4cd3e8cc0 <doveadm_setting_parser_info>, 0x0}
        error = 0x7fe4cd1b60b0 <_start>
"1\355I\211\321^H\211\342H\203\344\360PTL\213\005*.#"
(gdb)

Thanks for help.

Thom