Reproduce of error (Replicator: Panic: data stack: Out of memory):
=================================================================
When replay on any mail I will got OOM.
I try add in service replicate vsz_limit = 0 but that not help (after
proces dovecot/replicator eat all ram I got OOM)
Log error:
=========
Jun 03 09:38:59 Warning:
imap(ivan at myserv.local)<2533034><h7EQN4bgaJ4FvZdE>:
replication(ivan at myserv.local): Sync failure: Timeout in 2 secs
Jun 03 09:39:03 Panic: replicator: data stack: Out of memory when
allocating 4294967336 bytes
Jun 03 09:39:03 Error: replicator: Raw backtrace:
/usr/lib/dovecot/libdovecot.so.0(backtrace_append+0x42) [0x7feef08f0582]
-> /usr/lib/dovecot/libdovecot.so.0(backtrace_get+0x1e) [0x7feef08f069e]
-> /usr/lib/dovecot/libdovecot.so.0(+0x1022fb) [0x7feef08fd2fb] ->
/usr/lib/dovecot/libdovecot.so.0(+0x102391) [0x7feef08fd391] ->
/usr/lib/dovecot/libdovecot.so.0(+0x55589) [0x7feef0850589] ->
/usr/lib/dovecot/libdovecot.so.0(+0x54d52) [0x7feef084fd52] ->
/usr/lib/dovecot/libdovecot.so.0(+0xfb808) [0x7feef08f6808] ->
/usr/lib/dovecot/libdovecot.so.0(+0x122938) [0x7feef091d938] ->
/usr/lib/dovecot/libdovecot.so.0(+0xf76bf) [0x7feef08f26bf] ->
/usr/lib/dovecot/libdovecot.so.0(+0xf777e) [0x7feef08f277e] ->
/usr/lib/dovecot/libdovecot.so.0(buffer_append+0x61) [0x7feef08f2a21] ->
dovecot/replicator(replicator_queue_push+0x15a) [0x55953ece9b7a] ->
dovecot/replicator(+0x62b6) [0x55953ece92b6] ->
dovecot/replicator(+0x594e) [0x55953ece894e] ->
dovecot/replicator(+0x5b2c) [0x55953ece8b2c] ->
/usr/lib/dovecot/libdovecot.so.0(io_loop_call_io+0x69) [0x7feef0913529]
-> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x132)
[0x7feef0914c12] ->
/usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run+0x50)
[0x7feef09135d0] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_run+0x40)
[0x7feef0913790] ->
/usr/lib/dovecot/libdovecot.so.0(master_service_run+0x13)
[0x7feef0886353] -> dovecot/replicator(main+0x18d) [0x55953ece7cbd] ->
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xea) [0x7feef0651d0a]
-> dovecot/replicator(_start+0x2a) [0x55953ece7d7a]
Jun 03 09:39:03 Fatal: replicator: master: service(replicator): child
2532886 killed with signal 6 (core dumps disabled -
https://dovecot.org/bugreport.html#coredumps)
My configuration:
=========
# 2.3.19 (b3ad6004dc): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.5.19 (4eae2f79)
# OS: Linux 5.10.0-13-amd64 x86_64 Debian 11.3 ext4
# Hostname: mail.myserv.local
auth_mechanisms = plain login
auth_verbose = yes
debug_log_path = /var/log/dovecot.debug
default_client_limit = 5000
default_process_limit = 1000
default_vsz_limit = 2 G
deliver_log_format = msgid=%m, subject=%s, from=%f, size=%p(%w), %$
dict {
quota = pgsql:/etc/dovecot/dovecot-dict-sql.conf.ext
}
doveadm_password = # hidden, use -P to show it
hostname = myserv.local
lda_mailbox_autocreate = yes
lda_mailbox_autosubscribe = yes
listen = *
log_path = /var/log/dovecot.log
mail_home = /var/mail/vhosts/%d/mail/%n
mail_location =
maildir:/var/mail/vhosts/%d/mail/%n:INDEX=/var/mail/vhosts/%d/indexes/%n
mail_plugins = " quota fts fts_solr notify replication"
mail_privileged_group = vmail
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope
encoded-character vacation subaddress comparator-i;ascii-numeric
relational regex imap4flags copy include variables body enotify
environment mailbox date index ihave duplicate mime foreverypart
extracttext imapsieve vnd.dovecot.imapsieve
namespace inbox {
inbox = yes
location mailbox Drafts {
special_use = \Drafts
}
mailbox Junk {
special_use = \Junk
}
mailbox Sent {
special_use = \Sent
}
mailbox "Sent Messages" {
special_use = \Sent
}
mailbox Trash {
special_use = \Trash
}
prefix separator = /
}
passdb {
args = /etc/dovecot/dovecot-sql.conf.ext
driver = sql
}
plugin {
autocreate = Sent
autocreate2 = Drafts
autocreate3 = Junk
autocreate4 = Trash
autosubscribe = Sent
autosubscribe2 = Drafts
autosubscribe3 = Junk
autosubscribe4 = Trash
expire = Trash
expire2 = Trash/*
expire3 = Spam
fts = solr
fts_autoindex = yes
fts_solr = url=http://localhost:8983/solr/dovecot/
imapsieve_mailbox1_before = file:/etc/dovecot/sieve/report-spam.sieve
imapsieve_mailbox1_causes = COPY
imapsieve_mailbox1_name = Junk
imapsieve_mailbox2_before = file:/etc/dovecot/sieve/report-ham.sieve
imapsieve_mailbox2_causes = COPY
imapsieve_mailbox2_from = Junk
imapsieve_mailbox2_name = *
mail_replica = tcp:imap.myserv2.local:12345
quota = dict:Mailbox::proxy::quota
quota_rule = *:storage=1G
quota_warning = storage=95%% quota-warning 95 %u
quota_warning2 = storage=80%% quota-warning 80 %u
replication_sync_timeout = 2
sieve =
file:/var/mail/vhosts/%d/sieve/%n;active=/var/mail/vhosts/%d/sieve/%n/.dovecot.sieve
sieve_after = /etc/dovecot/sieve/after.d/
sieve_global_extensions = +vnd.dovecot.pipe +vnd.dovecot.environment
sieve_max_actions = 32
sieve_max_redirects = 4
sieve_max_script_size = 1M
sieve_pipe_bin_dir = /etc/dovecot/sieve
sieve_plugins = sieve_imapsieve sieve_extprograms
sieve_quota_max_scripts = 50
}
postmaster_address = postmaster@%d
protocols = " imap lmtp sieve pop3"
service aggregator {
fifo_listener replication-notify-fifo {
user = vmail
}
unix_listener replication-notify {
user = vmail
}
}
service auth-worker {
user = vmail
}
service auth {
unix_listener /var/spool/postfix/private/auth {
group = postfix
mode = 0666
user = postfix
}
}
service dict {
unix_listener dict {
user = vmail
}
}
service doveadm {
inet_listener {
port = 12345
ssl = no
}
}
service imap-login {
process_min_avail = 4
service_count = 1
}
service lmtp {
unix_listener /var/spool/postfix/private/dovecot-lmtp {
group = postfix
mode = 0600
user = postfix
}
}
service quota-warning {
executable = script /var/lib/dovecot/quota-warning.sh
unix_listener quota-warning {
user = vmail
}
user = root
}
service replicator {
process_min_avail = 1
unix_listener replicator-doveadm {
mode = 0600
user = vmail
}
}
ssl = required
ssl_cert = </etc/letsencrypt/live/myserv.local/fullchain.pem
ssl_client_ca_dir = /etc/ssl/certs
ssl_key = # hidden, use -P to show it
ssl_min_protocol = TLSv1.1
userdb {
args = /etc/dovecot/dovecot-sql.conf.ext
driver = sql
}
protocol lmtp {
mail_plugins = " quota fts fts_solr notify replication sieve"
}
protocol !indexer-worker {
mail_vsize_bg_after_count = 100
}
protocol lda {
mail_plugins = " quota fts fts_solr notify replication sieve"
}
protocol imap {
mail_max_userip_connections = 20
mail_plugins = " quota fts fts_solr notify replication imap_quota
imap_sieve"
}
protocol sieve {
mail_max_userip_connections = 20
managesieve_max_line_length = 64 k
}
protocol pop3 {
mail_max_userip_connections = 20
mail_plugins = " quota fts fts_solr notify replication quota"
}
Paul Kudla (SCOM.CA Internet Services Inc.)
2022-Jun-03 13:17 UTC
Replicator: Panic: data stack: Out of memory
Ok a little more help :
vsz_limit = 0 --> means unlimited ram for allocation, change this/try 2g
etc pending avaliable ram.
I use :
service replicator {
client_limit = 0
drop_priv_before_exec = no
idle_kill = 4294967295s
process_limit = 1
process_min_avail = 0
service_count = 0
unix_listener replicator-doveadm {
mode = 0600
user = vmail
}
vsz_limit = 8192M
}
next :
replication_max_conns = 100
think of this as a socket that gets opened everytime you connect to the
other server
100 means it will handle 100 seperate replication user requests
at a time (which is way more then sufficent)
if the connections overflows it just means the replication que will
carry on when resources opens up thus not killing the server resources.
If a que overflows because too much other stuff is going on it will just
reque and go again.
also note :
service aggregator {
process_limit = 1000
#vsz_limit = 1g
fifo_listener replication-notify-fifo {
user = vmail
group = vmail
mode = 0666
}
}
Happy Friday !!!
Thanks - paul
Paul Kudla
Scom.ca Internet Services <http://www.scom.ca>
004-1009 Byron Street South
Whitby, Ontario - Canada
L1N 4S3
Toronto 416.642.7266
Main?1.866.411.7266
Fax?1.888.892.7266
Email?paul at scom.ca
On 6/3/2022 5:13 AM, Ivan Juri?i? wrote:>
> Reproduce of error (Replicator: Panic: data stack: Out of memory):
> =================================================================>
> When replay on any mail I will got OOM.
>
> I try add in service replicate vsz_limit = 0 but that not help (after
> proces dovecot/replicator eat all ram I got OOM)
>
> Log error:
> =========>
> Jun 03 09:38:59 Warning:
> imap(ivan at myserv.local)<2533034><h7EQN4bgaJ4FvZdE>:
> replication(ivan at myserv.local): Sync failure: Timeout in 2 secs
> Jun 03 09:39:03 Panic: replicator: data stack: Out of memory when
> allocating 4294967336 bytes
> Jun 03 09:39:03 Error: replicator: Raw backtrace:
> /usr/lib/dovecot/libdovecot.so.0(backtrace_append+0x42) [0x7feef08f0582]
> -> /usr/lib/dovecot/libdovecot.so.0(backtrace_get+0x1e) [0x7feef08f069e]
> -> /usr/lib/dovecot/libdovecot.so.0(+0x1022fb) [0x7feef08fd2fb] ->
> /usr/lib/dovecot/libdovecot.so.0(+0x102391) [0x7feef08fd391] ->
> /usr/lib/dovecot/libdovecot.so.0(+0x55589) [0x7feef0850589] ->
> /usr/lib/dovecot/libdovecot.so.0(+0x54d52) [0x7feef084fd52] ->
> /usr/lib/dovecot/libdovecot.so.0(+0xfb808) [0x7feef08f6808] ->
> /usr/lib/dovecot/libdovecot.so.0(+0x122938) [0x7feef091d938] ->
> /usr/lib/dovecot/libdovecot.so.0(+0xf76bf) [0x7feef08f26bf] ->
> /usr/lib/dovecot/libdovecot.so.0(+0xf777e) [0x7feef08f277e] ->
> /usr/lib/dovecot/libdovecot.so.0(buffer_append+0x61) [0x7feef08f2a21] ->
> dovecot/replicator(replicator_queue_push+0x15a) [0x55953ece9b7a] ->
> dovecot/replicator(+0x62b6) [0x55953ece92b6] ->
> dovecot/replicator(+0x594e) [0x55953ece894e] ->
> dovecot/replicator(+0x5b2c) [0x55953ece8b2c] ->
> /usr/lib/dovecot/libdovecot.so.0(io_loop_call_io+0x69) [0x7feef0913529]
> -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x132)
> [0x7feef0914c12] ->
> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run+0x50)
> [0x7feef09135d0] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_run+0x40)
> [0x7feef0913790] ->
> /usr/lib/dovecot/libdovecot.so.0(master_service_run+0x13)
> [0x7feef0886353] -> dovecot/replicator(main+0x18d) [0x55953ece7cbd]
->
> /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xea) [0x7feef0651d0a]
> -> dovecot/replicator(_start+0x2a) [0x55953ece7d7a]
> Jun 03 09:39:03 Fatal: replicator: master: service(replicator): child
> 2532886 killed with signal 6 (core dumps disabled -
> https://dovecot.org/bugreport.html#coredumps)
>
> My configuration:
> =========>
> # 2.3.19 (b3ad6004dc): /etc/dovecot/dovecot.conf
> # Pigeonhole version 0.5.19 (4eae2f79)
> # OS: Linux 5.10.0-13-amd64 x86_64 Debian 11.3 ext4
> # Hostname: mail.myserv.local
> auth_mechanisms = plain login
> auth_verbose = yes
> debug_log_path = /var/log/dovecot.debug
> default_client_limit = 5000
> default_process_limit = 1000
> default_vsz_limit = 2 G
> deliver_log_format = msgid=%m, subject=%s, from=%f, size=%p(%w), %$
> dict {
> ? quota = pgsql:/etc/dovecot/dovecot-dict-sql.conf.ext
> }
> doveadm_password = # hidden, use -P to show it
> hostname = myserv.local
> lda_mailbox_autocreate = yes
> lda_mailbox_autosubscribe = yes
> listen = *
> log_path = /var/log/dovecot.log
> mail_home = /var/mail/vhosts/%d/mail/%n
> mail_location =
> maildir:/var/mail/vhosts/%d/mail/%n:INDEX=/var/mail/vhosts/%d/indexes/%n
> mail_plugins = " quota fts fts_solr notify replication"
> mail_privileged_group = vmail
> managesieve_notify_capability = mailto
> managesieve_sieve_capability = fileinto reject envelope
> encoded-character vacation subaddress comparator-i;ascii-numeric
> relational regex imap4flags copy include variables body enotify
> environment mailbox date index ihave duplicate mime foreverypart
> extracttext imapsieve vnd.dovecot.imapsieve
> namespace inbox {
> ? inbox = yes
> ? location > ? mailbox Drafts {
> ??? special_use = \Drafts
> ? }
> ? mailbox Junk {
> ??? special_use = \Junk
> ? }
> ? mailbox Sent {
> ??? special_use = \Sent
> ? }
> ? mailbox "Sent Messages" {
> ??? special_use = \Sent
> ? }
> ? mailbox Trash {
> ??? special_use = \Trash
> ? }
> ? prefix > ? separator = /
> }
> passdb {
> ? args = /etc/dovecot/dovecot-sql.conf.ext
> ? driver = sql
> }
> plugin {
> ? autocreate = Sent
> ? autocreate2 = Drafts
> ? autocreate3 = Junk
> ? autocreate4 = Trash
> ? autosubscribe = Sent
> ? autosubscribe2 = Drafts
> ? autosubscribe3 = Junk
> ? autosubscribe4 = Trash
> ? expire = Trash
> ? expire2 = Trash/*
> ? expire3 = Spam
> ? fts = solr
> ? fts_autoindex = yes
> ? fts_solr = url=http://localhost:8983/solr/dovecot/
> ? imapsieve_mailbox1_before = file:/etc/dovecot/sieve/report-spam.sieve
> ? imapsieve_mailbox1_causes = COPY
> ? imapsieve_mailbox1_name = Junk
> ? imapsieve_mailbox2_before = file:/etc/dovecot/sieve/report-ham.sieve
> ? imapsieve_mailbox2_causes = COPY
> ? imapsieve_mailbox2_from = Junk
> ? imapsieve_mailbox2_name = *
> ? mail_replica = tcp:imap.myserv2.local:12345
> ? quota = dict:Mailbox::proxy::quota
> ? quota_rule = *:storage=1G
> ? quota_warning = storage=95%% quota-warning 95 %u
> ? quota_warning2 = storage=80%% quota-warning 80 %u
> ? replication_sync_timeout = 2
> ? sieve =
>
file:/var/mail/vhosts/%d/sieve/%n;active=/var/mail/vhosts/%d/sieve/%n/.dovecot.sieve
>
> ? sieve_after = /etc/dovecot/sieve/after.d/
> ? sieve_global_extensions = +vnd.dovecot.pipe +vnd.dovecot.environment
> ? sieve_max_actions = 32
> ? sieve_max_redirects = 4
> ? sieve_max_script_size = 1M
> ? sieve_pipe_bin_dir = /etc/dovecot/sieve
> ? sieve_plugins = sieve_imapsieve sieve_extprograms
> ? sieve_quota_max_scripts = 50
> }
> postmaster_address = postmaster@%d
> protocols = " imap lmtp sieve pop3"
> service aggregator {
> ? fifo_listener replication-notify-fifo {
> ??? user = vmail
> ? }
> ? unix_listener replication-notify {
> ??? user = vmail
> ? }
> }
> service auth-worker {
> ? user = vmail
> }
> service auth {
> ? unix_listener /var/spool/postfix/private/auth {
> ??? group = postfix
> ??? mode = 0666
> ??? user = postfix
> ? }
> }
> service dict {
> ? unix_listener dict {
> ??? user = vmail
> ? }
> }
> service doveadm {
> ? inet_listener {
> ??? port = 12345
> ??? ssl = no
> ? }
> }
> service imap-login {
> ? process_min_avail = 4
> ? service_count = 1
> }
> service lmtp {
> ? unix_listener /var/spool/postfix/private/dovecot-lmtp {
> ??? group = postfix
> ??? mode = 0600
> ??? user = postfix
> ? }
> }
> service quota-warning {
> ? executable = script /var/lib/dovecot/quota-warning.sh
> ? unix_listener quota-warning {
> ??? user = vmail
> ? }
> ? user = root
> }
> service replicator {
> ? process_min_avail = 1
> ? unix_listener replicator-doveadm {
> ??? mode = 0600
> ??? user = vmail
> ? }
> }
> ssl = required
> ssl_cert = </etc/letsencrypt/live/myserv.local/fullchain.pem
> ssl_client_ca_dir = /etc/ssl/certs
> ssl_key = # hidden, use -P to show it
> ssl_min_protocol = TLSv1.1
> userdb {
> ? args = /etc/dovecot/dovecot-sql.conf.ext
> ? driver = sql
> }
> protocol lmtp {
> ? mail_plugins = " quota fts fts_solr notify replication sieve"
> }
> protocol !indexer-worker {
> ? mail_vsize_bg_after_count = 100
> }
> protocol lda {
> ? mail_plugins = " quota fts fts_solr notify replication sieve"
> }
> protocol imap {
> ? mail_max_userip_connections = 20
> ? mail_plugins = " quota fts fts_solr notify replication imap_quota
> imap_sieve"
> }
> protocol sieve {
> ? mail_max_userip_connections = 20
> ? managesieve_max_line_length = 64 k
> }
> protocol pop3 {
> ? mail_max_userip_connections = 20
> ? mail_plugins = " quota fts fts_solr notify replication quota"
> }
>