Hello,
I have a new director ring I am setting up on centos 7 with dovecot
2.3.1. I haven't been able to replecate this in testing, but as soon as
I start pushing production traffic to the new ring I see dozens of these in the
logs:
Apr 18 00:34:00 d.director.imapd.sonic.net kernel: imap-login[163107]: segfault
at 10 ip 00007ff625698dd5sp 00007ffe4b77bb28 error 4 in
libdovecot.so.0.0.0[7ff6255bf000+16e000]
My config:
# 2.3.1 (c5a5c0c82): /etc/dovecot/dovecot.conf
# OS: Linux 3.10.0-693.21.1.el7.x86_64 x86_64 CentOS Linux release 7.4.1708
(Core)
# Hostname: c.director.imapd.sonic.net
auth_master_user_separator = *
auth_username_format = %Ln
auth_verbose = yes
default_client_limit = 20000
director_mail_servers = 64.142.111.75 64.142.111.43 64.142.111.76 64.142.111.39
64.142.111.81
director_servers = c.director.imapd.sonic.net d.director.imapd.sonic.net
disable_plaintext_auth = no
doveadm_port = 1842
login_trusted_networks = 64.142.18.0/24 64.142.105.2 64.142.111.67 64.142.111.36
64.142.105.3 64.142.111.35 64.142.111.68
namespace inbox {
inbox = yes
location mailbox Drafts {
special_use = \Drafts
}
mailbox Junk {
special_use = \Junk
}
mailbox Sent {
special_use = \Sent
}
mailbox "Sent Messages" {
special_use = \Sent
}
mailbox Trash {
special_use = \Trash
}
prefix }
passdb {
args = /etc/dovecot/master-users
driver = passwd-file
master = yes
pass = yes
}
passdb {
args = proxy=y nopassword=y
driver = static
}
service anvil {
client_limit = 40103
}
service auth {
client_limit = 41704
}
service director {
fifo_listener login/proxy-notify {
mode = 0666
}
inet_listener {
port = 9321
}
unix_listener login/director {
mode = 0666
}
}
service imap-login {
executable = imap-login director
process_limit = 20000
process_min_avail = 32
}
service imap {
process_limit = 20480
}
service ipc {
unix_listener ipc {
user = dovecot
}
}
service pop3-login {
executable = pop3-login director
process_limit = 20000
process_min_avail = 32
}
ssl_cert = </etc/dovecot/ssl/imap.sonic.net.pem
ssl_cipher_list =
ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
ssl_dh = # hidden, use -P to show it
ssl_key = # hidden, use -P to show it
syslog_facility = local0
protocol imap {
imap_max_line_length = 128 k
ssl_cert = </etc/dovecot/ssl/imap.sonic.net.pem
ssl_key = # hidden, use -P to show it
}
protocol pop3 {
ssl_cert = </etc/dovecot/ssl/pop.sonic.net.pem
ssl_key = # hidden, use -P to show it
}
backtrace:
Stack trace of thread 56084:
#0 0x00007fe529be2dd5 i_stream_get_root_io (libdovecot.so.0)
#1 0x00007fe529be2e39 i_stream_set_input_pending
(libdovecot.so.0)
#2 0x00007fe527415a59 openssl_iostream_bio_sync
(libssl_iostream_openssl.so)
#3 0x00007fe527415c2a openssl_iostream_more
(libssl_iostream_openssl.so)
#4 0x00007fe527415c6b openssl_iostream_destroy
(libssl_iostream_openssl.so)
#5 0x00007fe529e8be72 client_unref (libdovecot-login.so.0)
#6 0x00007fe529e90441 login_proxy_free_full
(libdovecot-login.so.0)
#7 0x00007fe529e90e07 login_proxy_finished
(libdovecot-login.so.0)
#8 0x00007fe529becfb5 io_loop_call_io (libdovecot.so.0)
#9 0x00007fe529bee89f io_loop_handler_run_internal
(libdovecot.so.0)
#10 0x00007fe529bed0b2 io_loop_handler_run (libdovecot.so.0)
#11 0x00007fe529bed2d8 io_loop_run (libdovecot.so.0)
#12 0x00007fe529b69673 master_service_run (libdovecot.so.0)
#13 0x00007fe529e93532 login_binary_run (libdovecot-login.so.0)
#14 0x00007fe529767c05 __libc_start_main (libc.so.6)
#15 0x00005567b256050a _start (imap-login)
Grant Keller
System Operations
grant.keller at sonic.com
On 24.04.2018 00:40, Grant Keller wrote:> Hello, > > I have a new director ring I am setting up on centos 7 with dovecot > 2.3.1. I haven't been able to replecate this in testing, but as soon as > I start pushing production traffic to the new ring I see dozens of these in the > logs: > Apr 18 00:34:00 d.director.imapd.sonic.net kernel: imap-login[163107]: segfault at 10 ip 00007ff625698dd5sp 00007ffe4b77bb28 error 4 in libdovecot.so.0.0.0[7ff6255bf000+16e000] > > My config: > # 2.3.1 (c5a5c0c82): /etc/dovecot/dovecot.conf > # OS: Linux 3.10.0-693.21.1.el7.x86_64 x86_64 CentOS Linux release 7.4.1708 (Core) > # Hostname: c.director.imapd.sonic.net > auth_master_user_separator = * > auth_username_format = %Ln > auth_verbose = yes > default_client_limit = 20000 > director_mail_servers = 64.142.111.75 64.142.111.43 64.142.111.76 64.142.111.39 64.142.111.81 > director_servers = c.director.imapd.sonic.net d.director.imapd.sonic.net > disable_plaintext_auth = no > doveadm_port = 1842 > login_trusted_networks = 64.142.18.0/24 64.142.105.2 64.142.111.67 64.142.111.36 64.142.105.3 64.142.111.35 64.142.111.68 > namespace inbox { > inbox = yes > location > mailbox Drafts { > special_use = \Drafts > } > mailbox Junk { > special_use = \Junk > } > mailbox Sent { > special_use = \Sent > } > mailbox "Sent Messages" { > special_use = \Sent > } > mailbox Trash { > special_use = \Trash > } > prefix > } > passdb { > args = /etc/dovecot/master-users > driver = passwd-file > master = yes > pass = yes > } > passdb { > args = proxy=y nopassword=y > driver = static > } > service anvil { > client_limit = 40103 > } > service auth { > client_limit = 41704 > } > service director { > fifo_listener login/proxy-notify { > mode = 0666 > } > inet_listener { > port = 9321 > } > unix_listener login/director { > mode = 0666 > } > } > service imap-login { > executable = imap-login director > process_limit = 20000 > process_min_avail = 32 > } > service imap { > process_limit = 20480 > } > service ipc { > unix_listener ipc { > user = dovecot > } > } > service pop3-login { > executable = pop3-login director > process_limit = 20000 > process_min_avail = 32 > } > ssl_cert = </etc/dovecot/ssl/imap.sonic.net.pem > ssl_cipher_list = ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA > ssl_dh = # hidden, use -P to show it > ssl_key = # hidden, use -P to show it > syslog_facility = local0 > protocol imap { > imap_max_line_length = 128 k > ssl_cert = </etc/dovecot/ssl/imap.sonic.net.pem > ssl_key = # hidden, use -P to show it > } > protocol pop3 { > ssl_cert = </etc/dovecot/ssl/pop.sonic.net.pem > ssl_key = # hidden, use -P to show it > } > > backtrace: > Stack trace of thread 56084: > #0 0x00007fe529be2dd5 i_stream_get_root_io (libdovecot.so.0) > #1 0x00007fe529be2e39 i_stream_set_input_pending (libdovecot.so.0) > #2 0x00007fe527415a59 openssl_iostream_bio_sync (libssl_iostream_openssl.so) > #3 0x00007fe527415c2a openssl_iostream_more (libssl_iostream_openssl.so) > #4 0x00007fe527415c6b openssl_iostream_destroy (libssl_iostream_openssl.so) > #5 0x00007fe529e8be72 client_unref (libdovecot-login.so.0) > #6 0x00007fe529e90441 login_proxy_free_full (libdovecot-login.so.0) > #7 0x00007fe529e90e07 login_proxy_finished (libdovecot-login.so.0) > #8 0x00007fe529becfb5 io_loop_call_io (libdovecot.so.0) > #9 0x00007fe529bee89f io_loop_handler_run_internal (libdovecot.so.0) > #10 0x00007fe529bed0b2 io_loop_handler_run (libdovecot.so.0) > #11 0x00007fe529bed2d8 io_loop_run (libdovecot.so.0) > #12 0x00007fe529b69673 master_service_run (libdovecot.so.0) > #13 0x00007fe529e93532 login_binary_run (libdovecot-login.so.0) > #14 0x00007fe529767c05 __libc_start_main (libc.so.6) > #15 0x00005567b256050a _start (imap-login) > > > > Grant Keller > System Operations > grant.keller at sonic.comHi! Can you provide bt full output? Aki
Sorry about that, here it is:
(gdb) bt full
#0 i_stream_get_root_io (stream=0x0) at istream.c:911
No locals.
#1 0x00007f963a47de39 in i_stream_set_input_pending (stream=<optimized
out>, pending=pending at entry=true)
at istream.c:923
No locals.
#2 0x00007f9637cb0a59 in openssl_iostream_bio_input
(type=OPENSSL_IOSTREAM_SYNC_TYPE_HANDSHAKE,
ssl_io=0x5615d14844d0) at iostream-openssl.c:498
data = 0x7f963a4d00bd <uchar_nul> ""
bytes = 17339
ret = <optimized out>
bytes_read = true
size = 0
#3 openssl_iostream_bio_sync (ssl_io=ssl_io at entry=0x5615d14844d0,
type=OPENSSL_IOSTREAM_SYNC_TYPE_HANDSHAKE) at iostream-openssl.c:510
ret = false
#4 0x00007f9637cb0c2a in openssl_iostream_more (ssl_io=0x5615d14844d0,
type=type at entry=OPENSSL_IOSTREAM_SYNC_TYPE_HANDSHAKE) at
iostream-openssl.c:524
ret = <optimized out>
#5 0x00007f9637cb2f6c in o_stream_ssl_flush (stream=0x5615d14847d0) at
ostream-openssl.c:128
sstream = 0x5615d14847d0
plain_output = 0x5615d146d570
ret = <optimized out>
#6 0x00007f963a4960fe in o_stream_flush (stream=stream at entry=0x5615d1484870)
at ostream.c:200
_stream = 0x5615d14847d0
ret = 1
__func__ = "o_stream_flush"
#7 0x00007f963a4961a0 in o_stream_close_full (stream=0x5615d1484870,
close_parents=<optimized out>)
at ostream.c:53
No locals.
#8 0x00007f963a496243 in o_stream_destroy (stream=stream at
entry=0x5615d149af90) at ostream.c:75
No locals.
#9 0x00007f963a72acbc in login_proxy_free_final (proxy=0x5615d149af50) at
login-proxy.c:416
---Type <return> to continue, or q <return> to quit---
__func__ = "login_proxy_free_final"
#10 0x00007f963a72b427 in login_proxy_free_full (_proxy=_proxy at
entry=0x7ffea4c6c578,
reason=0x5615d14280c0 "Disconnected by server(0s idle, in=2015,
out=10272)",
delayed=delayed at entry=true) at login-proxy.c:521
proxy = 0x5615d149af50
client = 0x5615d1469d08
ipstr = <optimized out>
delay_ms = <optimized out>
__func__ = "login_proxy_free_full"
#11 0x00007f963a72be07 in login_proxy_free_delayed (reason=<optimized
out>, _proxy=0x7ffea4c6c578)
at login-proxy.c:541
No locals.
#12 login_proxy_free_errstr (server=true, errstr=<optimized out>,
_proxy=0x7ffea4c6c578)
at login-proxy.c:129
proxy = 0x5615d149af50
reason = 0x5615d1428088
#13 login_proxy_finished (side=<optimized out>, status=<optimized
out>, proxy=0x0) at login-proxy.c:619
errstr = <optimized out>
server_side = true
#14 0x00007f963a487fb5 in io_loop_call_io (io=0x5615d149b190) at ioloop.c:674
ioloop = 0x5615d1430d70
t_id = 2
__func__ = "io_loop_call_io"
#15 0x00007f963a48989f in io_loop_handler_run_internal (ioloop=ioloop at
entry=0x5615d1430d70)
at ioloop-epoll.c:222
ctx = 0x5615d14612e0
events = <optimized out>
list = 0x5615d1462150
io = <optimized out>
tv = {tv_sec = 59, tv_usec = 664792}
events_count = <optimized out>
msecs = <optimized out>
---Type <return> to continue, or q <return> to quit---
ret = 1
i = 0
call = <optimized out>
__func__ = "io_loop_handler_run_internal"
#16 0x00007f963a4880b2 in io_loop_handler_run (ioloop=ioloop at
entry=0x5615d1430d70) at ioloop.c:726
__func__ = "io_loop_handler_run"
#17 0x00007f963a4882d8 in io_loop_run (ioloop=0x5615d1430d70) at ioloop.c:699
__func__ = "io_loop_run"
#18 0x00007f963a404673 in master_service_run (service=0x5615d1430c00,
callback=callback at entry=0x7f963a72dd70 <client_connected>) at
master-service.c:767
No locals.
#19 0x00007f963a72e532 in login_binary_run (binary=<optimized out>,
argc=2, argv=0x5615d14308c0)
at main.c:549
set_pool = 0x5615d1431ef0
login_socket = 0x5615d14308eb "director"
c = <optimized out>
#20 0x00007f963a002c05 in __libc_start_main (main=0x5615cfbe84d0 <main>,
argc=2, ubp_av=0x7ffea4c6c898,
init=<optimized out>, fini=<optimized out>,
rtld_fini=<optimized out>, stack_end=0x7ffea4c6c888)
at ../csu/libc-start.c:274
result = <optimized out>
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {0, -1836909874292919703,
94651679671521,
140731662911632, 0, 0, 1836542576478188137,
1850284778656821865}, mask_was_saved = 0}},
priv = {pad = {0x0, 0x0, 0x7f963a9454c3 <_dl_init+275>,
0x7f963ab59150}, data = {prev = 0x0,
cleanup = 0x0, canceltype = 982799555}}}
not_first_call = <optimized out>
#21 0x00005615cfbe850a in _start ()
No symbol table info available.
(gdb)
Quoting Aki Tuomi (2018-04-23 23:55:24)>
>
> On 24.04.2018 00:40, Grant Keller wrote:
> > Hello,
> >
> > I have a new director ring I am setting up on centos 7 with dovecot
> > 2.3.1. I haven't been able to replecate this in testing, but as
soon as
> > I start pushing production traffic to the new ring I see dozens of
these in the
> > logs:
> > Apr 18 00:34:00 d.director.imapd.sonic.net kernel: imap-login[163107]:
segfault at 10 ip 00007ff625698dd5sp 00007ffe4b77bb28 error 4 in
libdovecot.so.0.0.0[7ff6255bf000+16e000]
> >
> > My config:
> > # 2.3.1 (c5a5c0c82): /etc/dovecot/dovecot.conf
> > # OS: Linux 3.10.0-693.21.1.el7.x86_64 x86_64 CentOS Linux release
7.4.1708 (Core)
> > # Hostname: c.director.imapd.sonic.net
> > auth_master_user_separator = *
> > auth_username_format = %Ln
> > auth_verbose = yes
> > default_client_limit = 20000
> > director_mail_servers = 64.142.111.75 64.142.111.43 64.142.111.76
64.142.111.39 64.142.111.81
> > director_servers = c.director.imapd.sonic.net
d.director.imapd.sonic.net
> > disable_plaintext_auth = no
> > doveadm_port = 1842
> > login_trusted_networks = 64.142.18.0/24 64.142.105.2 64.142.111.67
64.142.111.36 64.142.105.3 64.142.111.35 64.142.111.68
> > namespace inbox {
> > inbox = yes
> > location > > mailbox Drafts {
> > special_use = \Drafts
> > }
> > mailbox Junk {
> > special_use = \Junk
> > }
> > mailbox Sent {
> > special_use = \Sent
> > }
> > mailbox "Sent Messages" {
> > special_use = \Sent
> > }
> > mailbox Trash {
> > special_use = \Trash
> > }
> > prefix > > }
> > passdb {
> > args = /etc/dovecot/master-users
> > driver = passwd-file
> > master = yes
> > pass = yes
> > }
> > passdb {
> > args = proxy=y nopassword=y
> > driver = static
> > }
> > service anvil {
> > client_limit = 40103
> > }
> > service auth {
> > client_limit = 41704
> > }
> > service director {
> > fifo_listener login/proxy-notify {
> > mode = 0666
> > }
> > inet_listener {
> > port = 9321
> > }
> > unix_listener login/director {
> > mode = 0666
> > }
> > }
> > service imap-login {
> > executable = imap-login director
> > process_limit = 20000
> > process_min_avail = 32
> > }
> > service imap {
> > process_limit = 20480
> > }
> > service ipc {
> > unix_listener ipc {
> > user = dovecot
> > }
> > }
> > service pop3-login {
> > executable = pop3-login director
> > process_limit = 20000
> > process_min_avail = 32
> > }
> > ssl_cert = </etc/dovecot/ssl/imap.sonic.net.pem
> > ssl_cipher_list =
ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
> > ssl_dh = # hidden, use -P to show it
> > ssl_key = # hidden, use -P to show it
> > syslog_facility = local0
> > protocol imap {
> > imap_max_line_length = 128 k
> > ssl_cert = </etc/dovecot/ssl/imap.sonic.net.pem
> > ssl_key = # hidden, use -P to show it
> > }
> > protocol pop3 {
> > ssl_cert = </etc/dovecot/ssl/pop.sonic.net.pem
> > ssl_key = # hidden, use -P to show it
> > }
> >
> > backtrace:
> > Stack trace of thread 56084:
> > #0 0x00007fe529be2dd5 i_stream_get_root_io
(libdovecot.so.0)
> > #1 0x00007fe529be2e39 i_stream_set_input_pending
(libdovecot.so.0)
> > #2 0x00007fe527415a59 openssl_iostream_bio_sync
(libssl_iostream_openssl.so)
> > #3 0x00007fe527415c2a openssl_iostream_more
(libssl_iostream_openssl.so)
> > #4 0x00007fe527415c6b openssl_iostream_destroy
(libssl_iostream_openssl.so)
> > #5 0x00007fe529e8be72 client_unref
(libdovecot-login.so.0)
> > #6 0x00007fe529e90441 login_proxy_free_full
(libdovecot-login.so.0)
> > #7 0x00007fe529e90e07 login_proxy_finished
(libdovecot-login.so.0)
> > #8 0x00007fe529becfb5 io_loop_call_io
(libdovecot.so.0)
> > #9 0x00007fe529bee89f io_loop_handler_run_internal
(libdovecot.so.0)
> > #10 0x00007fe529bed0b2 io_loop_handler_run
(libdovecot.so.0)
> > #11 0x00007fe529bed2d8 io_loop_run (libdovecot.so.0)
> > #12 0x00007fe529b69673 master_service_run
(libdovecot.so.0)
> > #13 0x00007fe529e93532 login_binary_run
(libdovecot-login.so.0)
> > #14 0x00007fe529767c05 __libc_start_main (libc.so.6)
> > #15 0x00005567b256050a _start (imap-login)
> >
> >
> >
> > Grant Keller
> > System Operations
> > grant.keller at sonic.com
>
> Hi!
>
> Can you provide bt full output?
>
> Aki
>
Grant Keller
System Operations
grant.keller at sonic.com