Hi, We are seeing a frequent assertion failure on LMTP delivery with 2.3.0. This only appears to happen on CentOS/RHEL 7. Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Error: Raw backtrace: /usr/lib64/dovecot/libdovecot.so.0(+0xc8ec4) [0x7f226d036ec4] -> /usr/lib64/dovecot/libdovecot.so.0(+0xc8f7e) [0x7f226d036f7e] -> /usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7f226cfaa190] -> /usr/lib64/dovecot/libdovecot.so.0(+0xe469b) [0x7f226d05269b] -> /usr/lib64/dovecot/libdovecot-storage.so.0(mail_storage_service_user_unref+0xb4) [0x7f226d334b64] -> dovecot/lmtp(+0x63fa) [0x56438f3673fa] -> dovecot/lmtp(lmtp_local_deinit+0x5c) [0x56438f3678dc] -> dovecot/lmtp(client_state_reset+0x1a) [0x56438f366a5a] -> /usr/lib64/dovecot/libdovecot.so.0(smtp_server_connection_reset_state+0x3c) [0x7f226cfc196c] -> /usr/lib64/dovecot/libdovecot.so.0(smtp_server_command_completed+0x4d) [0x7f226cfbf31d] -> /usr/lib64/dovecot/libdovecot.so.0(+0x54848) [0x7f226cfc2848] -> /usr/lib64/dovecot/libdovecot.so.0(+0xf0970) [0x7f226d05e970] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0x65) [0x7f226d04e4b5] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x10f) [0x7f226d04fd8f] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x52) [0x7f226d04e5b2] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7f226d04e7d8] -> /usr/lib64/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7f226cfccbb3] -> dovecot/lmtp(main+0x226) [0x56438f3661a6] -> /lib64/libc.so.6(__libc_start_main+0xf5) [0x7f226cbccc05] -> dovecot/lmtp(+0x52d5) [0x56438f3662d5] Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Fatal: master: service(lmtp): child 29540 killed with signal 6 (core dumps disabled) From the exim log: 'LOG: MAIN cwd=/usr/local/cpanel 3 args: /usr/sbin/sendmail -v bob at l56ob1nf.cptestLOG: MAIN <= root at smoker-devautomerge-c7-2.dev.cpanel.net U=root P=local S=3146285 T="Ignore this test from Cpanel-Email-Message-Retrieve.t"LOG: MAIN cwd=/var/spool/exim 4 args: /usr/sbin/exim -v -Mc 1eemjH-0006uB-C2delivering 1eemjH-0006uB-C2 LMTP<< 220 smoker-devautomerge-c7-2.dev.cpanel.net Dovecot ready. LMTP>> LHLO smoker-devautomerge-c7-2.dev.cpanel.net LMTP<< 250-smoker-devautomerge-c7-2.dev.cpanel.net LMTP<< 250-8BITMIME LMTP<< 250-CHUNKING LMTP<< 250-ENHANCEDSTATUSCODES LMTP<< 250-PIPELINING LMTP<< 250-STARTTLS LMTP<< 250-VRFY LMTP<< 250 XCLIENT ADDR PORT PROTO HELO LOGIN TTL TIMEOUT LMTP>> MAIL FROM:<root at smoker-devautomerge-c7-2.dev.cpanel.net> LMTP<< 250 2.1.0 OK LMTP>> RCPT TO:<bob at l56ob1nf.cptest> LMTP<< 250 2.1.5 OK LMTP>> DATA LMTP<< 354 OK LMTP>> writing message and terminating "." LMTP>> QUIT LOG: MAIN == bob at l56ob1nf.cptest R=virtual_user T=dovecot_virtual_delivery defer (-1): LMTP connection closed after end of data?, Jan 24 10:00:55 smoker-devautomerge-c7-1 dovecot: lmtp(30217): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) Jan 24 10:00:55 smoker-devautomerge-c7-1 dovecot: lmtp(30233): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) Jan 24 10:05:15 smoker-devautomerge-c7-1 dovecot: lmtp(32618): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(375): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(403): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(424): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(430): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) Jan 24 12:32:16 smoker-devautomerge-c7-1 dovecot: lmtp(4435): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15232): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15238): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15244): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) doveconf -n: # 2.3.0 (c8b89eb): /etc/dovecot/dovecot.conf # OS: Linux 3.10.0-693.2.2.el7.x86_64 x86_64 CentOS Linux release 7.4.1708 (Core) auth_cache_size = 1 M auth_mechanisms = plain login auth_username_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$-=?^_{}~./@+%" disable_plaintext_auth = no first_valid_uid = 201 lda_mailbox_autocreate = yes lmtp_rcpt_check_quota = yes lmtp_save_to_detail_mailbox = yes lmtp_user_concurrency_limit = 4 mail_access_groups = dovecot mail_plugins = quota quota_clone zlib mail_prefetch_count = 20 maildir_very_dirty_syncs = yes namespace inbox { inbox = yes location = mailbox Archive { auto = create special_use = \Archive } mailbox Archives { auto = no special_use = \Archive } mailbox Drafts { auto = subscribe special_use = \Drafts } mailbox Junk { auto = create special_use = \Junk } mailbox Sent { auto = subscribe special_use = \Sent } mailbox "Sent Messages" { auto = no special_use = \Sent } mailbox Trash { auto = subscribe special_use = \Trash } mailbox spam { auto = create special_use = \Junk } prefix = INBOX. separator = . type = private } passdb { args = /usr/local/cpanel/etc/dovecot/cpauthd-dict.conf driver = dict result_failure = return-fail } passdb { args = /usr/local/cpanel/bin/dovecot-wrap driver = checkpassword skip = authenticated } plugin { acl = vfile:cache_secs=86400 quota_exceeded_message = Mailbox is full / Blocks limit exceeded / Inode limit exceeded } protocols = lmtp imap pop3 service auth { unix_listener auth-client { mode = 0666 } } service config { vsz_limit = 2 G } service dict { unix_listener dict { group = dovecot mode = 0660 } } service imap-login { client_limit = 500 inet_listener imap { address = *,:: } inet_listener imaps { address = *,:: } process_limit = 50 process_min_avail = 2 service_count = 0 vsz_limit = 128 M } service imap { process_limit = 512 vsz_limit = 512 M } service lmtp { client_limit = 1 process_limit = 500 unix_listener lmtp { group = mail mode = 0660 user = mailnull } vsz_limit = 512 M } service managesieve-login { client_limit = 500 process_limit = 50 process_min_avail = 2 service_count = 0 vsz_limit = 128 M } service managesieve { process_limit = 512 vsz_limit = 512 M } service pop3-login { client_limit = 500 inet_listener pop3 { address = *,:: } inet_listener pop3s { address = *,:: } process_limit = 50 process_min_avail = 2 service_count = 0 vsz_limit = 128 M } service pop3 { process_limit = 512 vsz_limit = 512 M } service quota-status { executable = quota-status -p postfix unix_listener quota-status { mode = 0666 } } ssl_cert = </etc/dovecot/ssl/dovecot.crt ssl_cipher_list = ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256 ssl_dh = # hidden, use -P to show it ssl_key = # hidden, use -P to show it userdb { driver = prefetch } userdb { args = /usr/local/cpanel/etc/dovecot/cpauthd-dict.conf driver = dict } userdb { args = /usr/local/cpanel/bin/dovecot-wrap driver = checkpassword } protocol imap { imap_capability = +NAMESPACE imap_idle_notify_interval = 24 mins imap_logout_format = in=%i, out=%o, bytes=%i/%o mail_max_userip_connections = 20 mail_plugins = acl quota imap_quota zlib imap_zlib quota_clone virtual namespace sent { hidden = yes list = no location = virtual:/usr/local/cpanel/etc/dovecot/virtual/sent:INDEX=~/mail/virtual/%u/sent prefix = sent separator = . } namespace spam { hidden = yes list = no location = virtual:/usr/local/cpanel/etc/dovecot/virtual/spam:INDEX=~/mail/virtual/%u/spam prefix = spam separator = . } } protocol pop3 { mail_max_userip_connections = 3 mail_plugins = quota quota quota_clone virtual zlib namespace sent { hidden = yes list = no location = virtual:/usr/local/cpanel/etc/dovecot/virtual/sent:INDEX=~/mail/virtual/%u/sent prefix = sent separator = . } namespace spam { hidden = yes list = no location = virtual:/usr/local/cpanel/etc/dovecot/virtual/spam:INDEX=~/mail/virtual/%u/spam prefix = spam separator = . } pop3_logout_format = top=%t/%p, retr=%r/%b, del=%d/%m, size=%s, bytes=%i/%o pop3_uidl_format = UID%u-%v } protocol lmtp { mail_plugins = quota quota_clone zlib postmaster_address = root quota_full_tempfail = no } protocol lda { mail_plugins = quota quota_clone zlib postmaster_address = root quota_full_tempfail = no } local_name smoker-devautomerge-c7-1.dev.cpanel.net { ssl_cert = </etc/dovecot/ssl/dovecot.crt ssl_key = # hidden, use -P to show it }
Hi! This is a bug in the new event code. Can you get a full gdb backtrace? Aki> On January 27, 2018 at 6:58 AM "J. Nick Koston" <nick at cpanel.net> wrote: > > > Hi, > > We are seeing a frequent assertion failure on LMTP delivery with 2.3.0. This only appears to happen on CentOS/RHEL 7. > > Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) > Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Error: Raw backtrace: /usr/lib64/dovecot/libdovecot.so.0(+0xc8ec4) [0x7f226d036ec4] -> /usr/lib64/dovecot/libdovecot.so.0(+0xc8f7e) [0x7f226d036f7e] -> /usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7f226cfaa190] -> /usr/lib64/dovecot/libdovecot.so.0(+0xe469b) [0x7f226d05269b] -> /usr/lib64/dovecot/libdovecot-storage.so.0(mail_storage_service_user_unref+0xb4) [0x7f226d334b64] -> dovecot/lmtp(+0x63fa) [0x56438f3673fa] -> dovecot/lmtp(lmtp_local_deinit+0x5c) [0x56438f3678dc] -> dovecot/lmtp(client_state_reset+0x1a) [0x56438f366a5a] -> /usr/lib64/dovecot/libdovecot.so.0(smtp_server_connection_reset_state+0x3c) [0x7f226cfc196c] -> /usr/lib64/dovecot/libdovecot.so.0(smtp_server_command_completed+0x4d) [0x7f226cfbf31d] -> /usr/lib64/dovecot/libdovecot.so.0(+0x54848) [0x7f226cfc2848] -> /usr/lib64/dovecot/libdovecot.so.0(+0xf0970) [0x7f226d05e970] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0x65) [0x7f226d04e4b5] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x10f) [0x7f226d04fd8f] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x52) [0x7f226d04e5b2] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7f226d04e7d8] -> /usr/lib64/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7f226cfccbb3] -> dovecot/lmtp(main+0x226) [0x56438f3661a6] -> /lib64/libc.so.6(__libc_start_main+0xf5) [0x7f226cbccc05] -> dovecot/lmtp(+0x52d5) [0x56438f3662d5] > Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Fatal: master: service(lmtp): child 29540 killed with signal 6 (core dumps disabled) > > > From the exim log: > 'LOG: MAIN cwd=/usr/local/cpanel 3 args: /usr/sbin/sendmail -v bob at l56ob1nf.cptestLOG: MAIN <= root at smoker-devautomerge-c7-2.dev.cpanel.net U=root P=local S=3146285 T="Ignore this test from Cpanel-Email-Message-Retrieve.t"LOG: MAIN cwd=/var/spool/exim 4 args: /usr/sbin/exim -v -Mc 1eemjH-0006uB-C2delivering 1eemjH-0006uB-C2 LMTP<< 220 smoker-devautomerge-c7-2.dev.cpanel.net Dovecot ready. LMTP>> LHLO smoker-devautomerge-c7-2.dev.cpanel.net > LMTP<< 250-smoker-devautomerge-c7-2.dev.cpanel.net LMTP<< 250-8BITMIME LMTP<< 250-CHUNKING LMTP<< 250-ENHANCEDSTATUSCODES LMTP<< 250-PIPELINING LMTP<< 250-STARTTLS LMTP<< 250-VRFY LMTP<< 250 XCLIENT ADDR PORT PROTO HELO LOGIN TTL TIMEOUT LMTP>> MAIL FROM:<root at smoker-devautomerge-c7-2.dev.cpanel.net> > LMTP<< 250 2.1.0 OK LMTP>> RCPT TO:<bob at l56ob1nf.cptest> > LMTP<< 250 2.1.5 OK LMTP>> DATA > LMTP<< 354 OK LMTP>> writing message and terminating "." LMTP>> QUIT > LOG: MAIN == bob at l56ob1nf.cptest R=virtual_user T=dovecot_virtual_delivery defer (-1): LMTP connection closed after end of data?, > > > > Jan 24 10:00:55 smoker-devautomerge-c7-1 dovecot: lmtp(30217): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) > Jan 24 10:00:55 smoker-devautomerge-c7-1 dovecot: lmtp(30233): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) > Jan 24 10:05:15 smoker-devautomerge-c7-1 dovecot: lmtp(32618): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) > Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(375): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) > Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(403): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) > Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(424): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) > Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(430): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) > Jan 24 12:32:16 smoker-devautomerge-c7-1 dovecot: lmtp(4435): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) > Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15232): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) > Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15238): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) > Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15244): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) > > doveconf -n: > > # 2.3.0 (c8b89eb): /etc/dovecot/dovecot.conf > # OS: Linux 3.10.0-693.2.2.el7.x86_64 x86_64 CentOS Linux release 7.4.1708 (Core) > auth_cache_size = 1 M > auth_mechanisms = plain login > auth_username_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$-=?^_{}~./@+%" > disable_plaintext_auth = no > first_valid_uid = 201 > lda_mailbox_autocreate = yes > lmtp_rcpt_check_quota = yes > lmtp_save_to_detail_mailbox = yes > lmtp_user_concurrency_limit = 4 > mail_access_groups = dovecot > mail_plugins = quota quota_clone zlib > mail_prefetch_count = 20 > maildir_very_dirty_syncs = yes > namespace inbox { > inbox = yes > location = > mailbox Archive { > auto = create > special_use = \Archive > } > mailbox Archives { > auto = no > special_use = \Archive > } > mailbox Drafts { > auto = subscribe > special_use = \Drafts > } > mailbox Junk { > auto = create > special_use = \Junk > } > mailbox Sent { > auto = subscribe > special_use = \Sent > } > mailbox "Sent Messages" { > auto = no > special_use = \Sent > } > mailbox Trash { > auto = subscribe > special_use = \Trash > } > mailbox spam { > auto = create > special_use = \Junk > } > prefix = INBOX. > separator = . > type = private > } > passdb { > args = /usr/local/cpanel/etc/dovecot/cpauthd-dict.conf > driver = dict > result_failure = return-fail > } > passdb { > args = /usr/local/cpanel/bin/dovecot-wrap > driver = checkpassword > skip = authenticated > } > plugin { > acl = vfile:cache_secs=86400 > quota_exceeded_message = Mailbox is full / Blocks limit exceeded / Inode limit exceeded > } > protocols = lmtp imap pop3 > service auth { > unix_listener auth-client { > mode = 0666 > } > } > service config { > vsz_limit = 2 G > } > service dict { > unix_listener dict { > group = dovecot > mode = 0660 > } > } > service imap-login { > client_limit = 500 > inet_listener imap { > address = *,:: > } > inet_listener imaps { > address = *,:: > } > process_limit = 50 > process_min_avail = 2 > service_count = 0 > vsz_limit = 128 M > } > service imap { > process_limit = 512 > vsz_limit = 512 M > } > service lmtp { > client_limit = 1 > process_limit = 500 > unix_listener lmtp { > group = mail > mode = 0660 > user = mailnull > } > vsz_limit = 512 M > } > service managesieve-login { > client_limit = 500 > process_limit = 50 > process_min_avail = 2 > service_count = 0 > vsz_limit = 128 M > } > service managesieve { > process_limit = 512 > vsz_limit = 512 M > } > service pop3-login { > client_limit = 500 > inet_listener pop3 { > address = *,:: > } > inet_listener pop3s { > address = *,:: > } > process_limit = 50 > process_min_avail = 2 > service_count = 0 > vsz_limit = 128 M > } > service pop3 { > process_limit = 512 > vsz_limit = 512 M > } > service quota-status { > executable = quota-status -p postfix > unix_listener quota-status { > mode = 0666 > } > } > ssl_cert = </etc/dovecot/ssl/dovecot.crt > ssl_cipher_list = ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256 > ssl_dh = # hidden, use -P to show it > ssl_key = # hidden, use -P to show it > userdb { > driver = prefetch > } > userdb { > args = /usr/local/cpanel/etc/dovecot/cpauthd-dict.conf > driver = dict > } > userdb { > args = /usr/local/cpanel/bin/dovecot-wrap > driver = checkpassword > } > protocol imap { > imap_capability = +NAMESPACE > imap_idle_notify_interval = 24 mins > imap_logout_format = in=%i, out=%o, bytes=%i/%o > mail_max_userip_connections = 20 > mail_plugins = acl quota imap_quota zlib imap_zlib quota_clone virtual > namespace sent { > hidden = yes > list = no > location = virtual:/usr/local/cpanel/etc/dovecot/virtual/sent:INDEX=~/mail/virtual/%u/sent > prefix = sent > separator = . > } > namespace spam { > hidden = yes > list = no > location = virtual:/usr/local/cpanel/etc/dovecot/virtual/spam:INDEX=~/mail/virtual/%u/spam > prefix = spam > separator = . > } > } > protocol pop3 { > mail_max_userip_connections = 3 > mail_plugins = quota quota quota_clone virtual zlib > namespace sent { > hidden = yes > list = no > location = virtual:/usr/local/cpanel/etc/dovecot/virtual/sent:INDEX=~/mail/virtual/%u/sent > prefix = sent > separator = . > } > namespace spam { > hidden = yes > list = no > location = virtual:/usr/local/cpanel/etc/dovecot/virtual/spam:INDEX=~/mail/virtual/%u/spam > prefix = spam > separator = . > } > pop3_logout_format = top=%t/%p, retr=%r/%b, del=%d/%m, size=%s, bytes=%i/%o > pop3_uidl_format = UID%u-%v > } > protocol lmtp { > mail_plugins = quota quota_clone zlib > postmaster_address = root > quota_full_tempfail = no > } > protocol lda { > mail_plugins = quota quota_clone zlib > postmaster_address = root > quota_full_tempfail = no > } > local_name smoker-devautomerge-c7-1.dev.cpanel.net { > ssl_cert = </etc/dovecot/ssl/dovecot.crt > ssl_key = # hidden, use -P to show it > } > >
Hi Aki, Please see below: Thank you -Nick # gdb /usr/libexec/dovecot/lmtp ./3445 Using host libthread_db library "/lib64/libthread_db.so.1". Core was generated by `dovecot/lmtp'. Program terminated with signal 6, Aborted. #0 0x00007f297814d1f7 in raise () from /lib64/libc.so.6 Missing separate debuginfos, use: debuginfo-install dovecot-2.3.0-8.cp1162.x86_64 (gdb) back #0 0x00007f297814d1f7 in raise () from /lib64/libc.so.6 #1 0x00007f297814e8e8 in abort () from /lib64/libc.so.6 #2 0x00007f29785a3eab in default_fatal_finish (type=<optimized out>, status=status at entry=0) at failures.c:228 #3 0x00007f29785a3f9e in i_internal_fatal_handler (ctx=0x7ffe12c97a00, format=<optimized out>, args=<optimized out>) at failures.c:718 #4 0x00007f2978517190 in i_panic (format=format at entry=0x7f29785df6a8 "file %s: line %d (%s): assertion failed: (%s)") at failures.c:306 #5 0x00007f29785bf6bb in event_unref (_event=_event at entry=0x55ac2dab3ea8) at lib-event.c:148 #6 0x00007f29788a1b64 in mail_storage_service_user_unref (_user=_user at entry=0x55ac2dab35a8) at mail-storage-service.c:1649 #7 0x000055ac2cbd03fa in lmtp_local_rcpt_deinit (rcpt=0x55ac2dab3570) at lmtp-local.c:125 #8 0x000055ac2cbd08dc in lmtp_local_deinit (_local=_local at entry=0x55ac2daa00f0) at lmtp-local.c:83 #9 0x000055ac2cbcfa5a in client_state_reset (client=0x55ac2daa0028) at client.c:177 #10 0x00007f297852e96c in smtp_server_connection_reset_state (conn=0x55ac2daaee40) at smtp-server-connection.c:1249 #11 0x00007f297852c31d in smtp_server_command_completed (cmd=0x55ac2dab3998) at smtp-server-command.c:380 #12 0x00007f297852f848 in smtp_server_connection_next_reply (conn=0x55ac2daaee40) at smtp-server-connection.c:623 #13 smtp_server_connection_send_replies (conn=0x55ac2daaee40) at smtp-server-connection.c:682 #14 smtp_server_connection_output (conn=0x55ac2daaee40) at smtp-server-connection.c:728 #15 0x00007f29785cb990 in stream_send_io (fstream=0x55ac2da75dd0) at ostream-file.c:475 #16 0x00007f29785bb4d5 in io_loop_call_io (io=0x55ac2dab75b0) at ioloop.c:614 #17 0x00007f29785bcdaf in io_loop_handler_run_internal (ioloop=ioloop at entry=0x55ac2da6ac50) at ioloop-epoll.c:222 #18 0x00007f29785bb5d2 in io_loop_handler_run (ioloop=ioloop at entry=0x55ac2da6ac50) at ioloop.c:666 #19 0x00007f29785bb7f8 in io_loop_run (ioloop=0x55ac2da6ac50) at ioloop.c:639 #20 0x00007f2978539bb3 in master_service_run (service=0x55ac2da6aae0, callback=callback at entry=0x55ac2cbcf3d0 <client_connected>) at master-service.c:767 #21 0x000055ac2cbcf1a6 in main (argc=1, argv=0x55ac2da6a8a0) at main.c:159 (gdb) bt full #0 0x00007f297814d1f7 in raise () from /lib64/libc.so.6 No symbol table info available. #1 0x00007f297814e8e8 in abort () from /lib64/libc.so.6 No symbol table info available. #2 0x00007f29785a3eab in default_fatal_finish (type=<optimized out>, status=status at entry=0) at failures.c:228 backtrace = 0x55ac2da62088 "/usr/lib64/dovecot/libdovecot.so.0(+0xc8ee4) [0x7f29785a3ee4] -> /usr/lib64/dovecot/libdovecot.so.0(+0xc8f9e) [0x7f29785a3f9e] -> /usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7f2978517190] -> /usr"... recursed = 0 #3 0x00007f29785a3f9e in i_internal_fatal_handler (ctx=0x7ffe12c97a00, format=<optimized out>, args=<optimized out>) at failures.c:718 status = 0 #4 0x00007f2978517190 in i_panic (format=format at entry=0x7f29785df6a8 "file %s: line %d (%s): assertion failed: (%s)") at failures.c:306 ctx = {type = LOG_TYPE_PANIC, exit_status = 0, timestamp = 0x0, timestamp_usecs = 0, log_prefix = 0x0} args = {{gp_offset = 40, fp_offset = 48, overflow_arg_area = 0x7ffe12c97af0, reg_save_area = 0x7ffe12c97a30}} #5 0x00007f29785bf6bb in event_unref (_event=_event at entry=0x55ac2dab3ea8) at lib-event.c:148 event = 0x55ac2dab8180 __func__ = "event_unref" #6 0x00007f29788a1b64 in mail_storage_service_user_unref (_user=_user at entry=0x55ac2dab35a8) at mail-storage-service.c:1649 user = 0x55ac2dab3e08 __func__ = "mail_storage_service_user_unref" #7 0x000055ac2cbd03fa in lmtp_local_rcpt_deinit (rcpt=0x55ac2dab3570) at lmtp-local.c:125 No locals. #8 0x000055ac2cbd08dc in lmtp_local_deinit (_local=_local at entry=0x55ac2daa00f0) at lmtp-local.c:83 rcptp_end = 0x55ac2da9fbb8 local = 0x55ac2dab8000 rcptp = 0x55ac2da9fbb0 #9 0x000055ac2cbcfa5a in client_state_reset (client=0x55ac2daa0028) at client.c:177 No locals. #10 0x00007f297852e96c in smtp_server_connection_reset_state (conn=0x55ac2daaee40) at smtp-server-connection.c:1249 No locals. #11 0x00007f297852c31d in smtp_server_command_completed (cmd=0x55ac2dab3998) at smtp-server-command.c:380 hook_completed = <optimized out> #12 0x00007f297852f848 in smtp_server_connection_next_reply (conn=0x55ac2daaee40) at smtp-server-connection.c:623 cmd = 0x55ac2dab3998 error = 0x0 i = <optimized out> #13 smtp_server_connection_send_replies (conn=0x55ac2daaee40) at smtp-server-connection.c:682 No locals. #14 smtp_server_connection_output (conn=0x55ac2daaee40) at smtp-server-connection.c:728 ret = 1 #15 0x00007f29785cb990 in stream_send_io (fstream=0x55ac2da75dd0) at ostream-file.c:475 ostream = 0x55ac2da75e60 use_cork = true ret = <optimized out> #16 0x00007f29785bb4d5 in io_loop_call_io (io=0x55ac2dab75b0) at ioloop.c:614 ioloop = 0x55ac2da6ac50 t_id = 2 __func__ = "io_loop_call_io" ---Type <return> to continue, or q <return> to quit--- #17 0x00007f29785bcdaf in io_loop_handler_run_internal (ioloop=ioloop at entry=0x55ac2da6ac50) at ioloop-epoll.c:222 ctx = 0x55ac2da761d0 events = <optimized out> list = 0x55ac2da9c650 io = <optimized out> tv = {tv_sec = 2147483, tv_usec = 0} events_count = <optimized out> msecs = <optimized out> ret = 1 i = 0 call = <optimized out> __func__ = "io_loop_handler_run_internal" #18 0x00007f29785bb5d2 in io_loop_handler_run (ioloop=ioloop at entry=0x55ac2da6ac50) at ioloop.c:666 __func__ = "io_loop_handler_run" #19 0x00007f29785bb7f8 in io_loop_run (ioloop=0x55ac2da6ac50) at ioloop.c:639 __func__ = "io_loop_run" #20 0x00007f2978539bb3 in master_service_run (service=0x55ac2da6aae0, callback=callback at entry=0x55ac2cbcf3d0 <client_connected>) at master-service.c:767 No locals. #21 0x000055ac2cbcf1a6 in main (argc=1, argv=0x55ac2da6a8a0) at main.c:159 set_roots = {0x7f2978843fc0 <smtp_submit_setting_parser_info>, 0x7f2978da4820 <lda_setting_parser_info>, 0x55ac2cdd4560 <lmtp_setting_parser_info>, 0x0} service_flags = <optimized out> storage_service_flags = <optimized out> tmp_base_dir = 0x55ac2da62040 "\003" c = <optimized out> error = 0x0 (gdb) p current_global_event $1 = (struct event *) 0x55ac2dab8180 (gdb) p event No symbol "event" in current context. (gdb) p *current_global_event $2 = {event_passthrough = {append_log_prefix = 0x7f29785bf830 <event_passthrough_set_append_log_prefix>, replace_log_prefix = 0x7f29785bf860 <event_passthrough_replace_log_prefix>, set_name = 0x7f29785bf8d0 <event_passthrough_set_name>, set_source = 0x7f29785bf930 <event_passthrough_set_source>, set_always_log_source = 0x7f29785bf970 <event_passthrough_set_always_log_source>, add_categories = 0x7f29785bfb00 <event_passthrough_add_categories>, add_category = 0x7f29785bfb70 <event_passthrough_add_category>, add_fields = 0x7f29785bfe80 <event_passthrough_add_fields>, add_str = 0x7f29785bfcc0 <event_passthrough_add_str>, add_int = 0x7f29785bfd40 <event_passthrough_add_int>, add_timeval = 0x7f29785bfdd0 <event_passthrough_add_timeval>, event = 0x7f29785bedd0 <event_passthrough_event>}, prev = 0x55ac2dac51a0, next = 0x55ac2da75be0, refcount = 0, pool = 0x55ac2dab8070, parent = 0x0, id = 2, log_prefix = 0x0, log_prefix_from_system_pool = false, log_prefix_replace = false, passthrough = false, forced_debug = false, always_log_source = false, sending_debug_log = false, id_sent_to_stats = false, call_free = false, tv_created = {tv_sec = 1517200513, tv_usec = 468845}, tv_last_sent = {tv_sec = 0, tv_usec = 0}, source_filename = 0x7f297894f95a "mail-storage-service.c", source_linenum = 1333, sending_name = 0x0, categories = {arr = {buffer = 0x0, element_size = 0}, v = 0x0, v_modifiable = 0x0}, fields = {arr = {buffer = 0x55ac2dab8278, element_size = 48}, v = 0x55ac2dab8278, v_modifiable = 0x55ac2dab8278}}> On Jan 27, 2018, at 4:40 PM, Aki Tuomi <aki.tuomi at dovecot.fi> wrote: > > Hi! > > This is a bug in the new event code. Can you get a full gdb backtrace? > > Aki > >> On January 27, 2018 at 6:58 AM "J. Nick Koston" <nick at cpanel.net> wrote: >> >> >> Hi, >> >> We are seeing a frequent assertion failure on LMTP delivery with 2.3.0. This only appears to happen on CentOS/RHEL 7. >> >> Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) >> Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Error: Raw backtrace: /usr/lib64/dovecot/libdovecot.so.0(+0xc8ec4) [0x7f226d036ec4] -> /usr/lib64/dovecot/libdovecot.so.0(+0xc8f7e) [0x7f226d036f7e] -> /usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7f226cfaa190] -> /usr/lib64/dovecot/libdovecot.so.0(+0xe469b) [0x7f226d05269b] -> /usr/lib64/dovecot/libdovecot-storage.so.0(mail_storage_service_user_unref+0xb4) [0x7f226d334b64] -> dovecot/lmtp(+0x63fa) [0x56438f3673fa] -> dovecot/lmtp(lmtp_local_deinit+0x5c) [0x56438f3678dc] -> dovecot/lmtp(client_state_reset+0x1a) [0x56438f366a5a] -> /usr/lib64/dovecot/libdovecot.so.0(smtp_server_connection_reset_state+0x3c) [0x7f226cfc196c] -> /usr/lib64/dovecot/libdovecot.so.0(smtp_server_command_completed+0x4d) [0x7f226cfbf31d] -> /usr/lib64/dovecot/libdovecot.so.0(+0x54848) [0x7f226cfc2848] -> /usr/lib64/dovecot/libdovecot.so.0(+0xf0970) [0x7f226d05e970] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0x65) [0x7f226d04e4b5] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x10f) [0x7f226d04fd8f] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x52) [0x7f226d04e5b2] -> /usr/lib64/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7f226d04e7d8] -> /usr/lib64/dovecot/libdovecot.so.0(master_service_run+0x13) [0x7f226cfccbb3] -> dovecot/lmtp(main+0x226) [0x56438f3661a6] -> /lib64/libc.so.6(__libc_start_main+0xf5) [0x7f226cbccc05] -> dovecot/lmtp(+0x52d5) [0x56438f3662d5] >> Jan 24 08:30:58 smoker-devautomerge-c7-1 dovecot: lmtp(29540): Fatal: master: service(lmtp): child 29540 killed with signal 6 (core dumps disabled) >> >> >> From the exim log: >> 'LOG: MAIN cwd=/usr/local/cpanel 3 args: /usr/sbin/sendmail -v bob at l56ob1nf.cptestLOG: MAIN <= root at smoker-devautomerge-c7-2.dev.cpanel.net U=root P=local S=3146285 T="Ignore this test from Cpanel-Email-Message-Retrieve.t"LOG: MAIN cwd=/var/spool/exim 4 args: /usr/sbin/exim -v -Mc 1eemjH-0006uB-C2delivering 1eemjH-0006uB-C2 LMTP<< 220 smoker-devautomerge-c7-2.dev.cpanel.net Dovecot ready. LMTP>> LHLO smoker-devautomerge-c7-2.dev.cpanel.net >> LMTP<< 250-smoker-devautomerge-c7-2.dev.cpanel.net LMTP<< 250-8BITMIME LMTP<< 250-CHUNKING LMTP<< 250-ENHANCEDSTATUSCODES LMTP<< 250-PIPELINING LMTP<< 250-STARTTLS LMTP<< 250-VRFY LMTP<< 250 XCLIENT ADDR PORT PROTO HELO LOGIN TTL TIMEOUT LMTP>> MAIL FROM:<root at smoker-devautomerge-c7-2.dev.cpanel.net> >> LMTP<< 250 2.1.0 OK LMTP>> RCPT TO:<bob at l56ob1nf.cptest> >> LMTP<< 250 2.1.5 OK LMTP>> DATA >> LMTP<< 354 OK LMTP>> writing message and terminating "." LMTP>> QUIT >> LOG: MAIN == bob at l56ob1nf.cptest R=virtual_user T=dovecot_virtual_delivery defer (-1): LMTP connection closed after end of data?, >> >> >> >> Jan 24 10:00:55 smoker-devautomerge-c7-1 dovecot: lmtp(30217): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) >> Jan 24 10:00:55 smoker-devautomerge-c7-1 dovecot: lmtp(30233): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) >> Jan 24 10:05:15 smoker-devautomerge-c7-1 dovecot: lmtp(32618): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) >> Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(375): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) >> Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(403): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) >> Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(424): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) >> Jan 24 10:05:52 smoker-devautomerge-c7-1 dovecot: lmtp(430): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) >> Jan 24 12:32:16 smoker-devautomerge-c7-1 dovecot: lmtp(4435): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) >> Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15232): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) >> Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15238): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) >> Jan 24 12:54:53 smoker-devautomerge-c7-1 dovecot: lmtp(15244): Panic: file lib-event.c: line 148 (event_unref): assertion failed: (event != current_global_event) >> >> doveconf -n: >> >> # 2.3.0 (c8b89eb): /etc/dovecot/dovecot.conf >> # OS: Linux 3.10.0-693.2.2.el7.x86_64 x86_64 CentOS Linux release 7.4.1708 (Core) >> auth_cache_size = 1 M >> auth_mechanisms = plain login >> auth_username_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$-=?^_{}~./@+%" >> disable_plaintext_auth = no >> first_valid_uid = 201 >> lda_mailbox_autocreate = yes >> lmtp_rcpt_check_quota = yes >> lmtp_save_to_detail_mailbox = yes >> lmtp_user_concurrency_limit = 4 >> mail_access_groups = dovecot >> mail_plugins = quota quota_clone zlib >> mail_prefetch_count = 20 >> maildir_very_dirty_syncs = yes >> namespace inbox { >> inbox = yes >> location = >> mailbox Archive { >> auto = create >> special_use = \Archive >> } >> mailbox Archives { >> auto = no >> special_use = \Archive >> } >> mailbox Drafts { >> auto = subscribe >> special_use = \Drafts >> } >> mailbox Junk { >> auto = create >> special_use = \Junk >> } >> mailbox Sent { >> auto = subscribe >> special_use = \Sent >> } >> mailbox "Sent Messages" { >> auto = no >> special_use = \Sent >> } >> mailbox Trash { >> auto = subscribe >> special_use = \Trash >> } >> mailbox spam { >> auto = create >> special_use = \Junk >> } >> prefix = INBOX. >> separator = . >> type = private >> } >> passdb { >> args = /usr/local/cpanel/etc/dovecot/cpauthd-dict.conf >> driver = dict >> result_failure = return-fail >> } >> passdb { >> args = /usr/local/cpanel/bin/dovecot-wrap >> driver = checkpassword >> skip = authenticated >> } >> plugin { >> acl = vfile:cache_secs=86400 >> quota_exceeded_message = Mailbox is full / Blocks limit exceeded / Inode limit exceeded >> } >> protocols = lmtp imap pop3 >> service auth { >> unix_listener auth-client { >> mode = 0666 >> } >> } >> service config { >> vsz_limit = 2 G >> } >> service dict { >> unix_listener dict { >> group = dovecot >> mode = 0660 >> } >> } >> service imap-login { >> client_limit = 500 >> inet_listener imap { >> address = *,:: >> } >> inet_listener imaps { >> address = *,:: >> } >> process_limit = 50 >> process_min_avail = 2 >> service_count = 0 >> vsz_limit = 128 M >> } >> service imap { >> process_limit = 512 >> vsz_limit = 512 M >> } >> service lmtp { >> client_limit = 1 >> process_limit = 500 >> unix_listener lmtp { >> group = mail >> mode = 0660 >> user = mailnull >> } >> vsz_limit = 512 M >> } >> service managesieve-login { >> client_limit = 500 >> process_limit = 50 >> process_min_avail = 2 >> service_count = 0 >> vsz_limit = 128 M >> } >> service managesieve { >> process_limit = 512 >> vsz_limit = 512 M >> } >> service pop3-login { >> client_limit = 500 >> inet_listener pop3 { >> address = *,:: >> } >> inet_listener pop3s { >> address = *,:: >> } >> process_limit = 50 >> process_min_avail = 2 >> service_count = 0 >> vsz_limit = 128 M >> } >> service pop3 { >> process_limit = 512 >> vsz_limit = 512 M >> } >> service quota-status { >> executable = quota-status -p postfix >> unix_listener quota-status { >> mode = 0666 >> } >> } >> ssl_cert = </etc/dovecot/ssl/dovecot.crt >> ssl_cipher_list = ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256 >> ssl_dh = # hidden, use -P to show it >> ssl_key = # hidden, use -P to show it >> userdb { >> driver = prefetch >> } >> userdb { >> args = /usr/local/cpanel/etc/dovecot/cpauthd-dict.conf >> driver = dict >> } >> userdb { >> args = /usr/local/cpanel/bin/dovecot-wrap >> driver = checkpassword >> } >> protocol imap { >> imap_capability = +NAMESPACE >> imap_idle_notify_interval = 24 mins >> imap_logout_format = in=%i, out=%o, bytes=%i/%o >> mail_max_userip_connections = 20 >> mail_plugins = acl quota imap_quota zlib imap_zlib quota_clone virtual >> namespace sent { >> hidden = yes >> list = no >> location = virtual:/usr/local/cpanel/etc/dovecot/virtual/sent:INDEX=~/mail/virtual/%u/sent >> prefix = sent >> separator = . >> } >> namespace spam { >> hidden = yes >> list = no >> location = virtual:/usr/local/cpanel/etc/dovecot/virtual/spam:INDEX=~/mail/virtual/%u/spam >> prefix = spam >> separator = . >> } >> } >> protocol pop3 { >> mail_max_userip_connections = 3 >> mail_plugins = quota quota quota_clone virtual zlib >> namespace sent { >> hidden = yes >> list = no >> location = virtual:/usr/local/cpanel/etc/dovecot/virtual/sent:INDEX=~/mail/virtual/%u/sent >> prefix = sent >> separator = . >> } >> namespace spam { >> hidden = yes >> list = no >> location = virtual:/usr/local/cpanel/etc/dovecot/virtual/spam:INDEX=~/mail/virtual/%u/spam >> prefix = spam >> separator = . >> } >> pop3_logout_format = top=%t/%p, retr=%r/%b, del=%d/%m, size=%s, bytes=%i/%o >> pop3_uidl_format = UID%u-%v >> } >> protocol lmtp { >> mail_plugins = quota quota_clone zlib >> postmaster_address = root >> quota_full_tempfail = no >> } >> protocol lda { >> mail_plugins = quota quota_clone zlib >> postmaster_address = root >> quota_full_tempfail = no >> } >> local_name smoker-devautomerge-c7-1.dev.cpanel.net { >> ssl_cert = </etc/dovecot/ssl/dovecot.crt >> ssl_key = # hidden, use -P to show it >> } >> >> >