Replicator: Panic: data stack: Out of memory

Paul Kudla (SCOM.CA Internet Services Inc.)
Fri, 03 Jun 2022 13:17:54 -0000

Ok a little more help :

vsz_limit = 0 --> means unlimited ram for allocation, change this/try 2g
etc pending avaliable ram.


I use :

service replicator {
   client_limit = 0
   drop_priv_before_exec = no
   idle_kill = 4294967295s
   process_limit = 1
   process_min_avail = 0
   service_count = 0
         unix_listener replicator-doveadm {
     mode = 0600
     user = vmail
   }
   vsz_limit = 8192M
}







next :

replication_max_conns = 100

think of this as a socket that gets opened everytime you connect to the
other server

100 means it will handle 100 seperate replication user requests

at a time (which is way more then sufficent)

if the connections overflows it just means the replication que will
carry on when resources opens up thus not killing the server resources.

If a que overflows because too much other stuff is going on it will just
reque and go again.

also note :

service aggregator {
   process_limit = 1000
   #vsz_limit = 1g
   fifo_listener replication-notify-fifo {
     user = vmail
     group = vmail
     mode = 0666
   }

}







Happy Friday !!!
Thanks - paul

Paul Kudla


Scom.ca Internet Services <http://www.scom.ca>
004-1009 Byron Street South
Whitby, Ontario - Canada
L1N 4S3

Toronto 416.642.7266
Main?1.866.411.7266
Fax?1.888.892.7266
Email?paul at scom.ca

On 6/3/2022 5:13 AM, Ivan Juri?i? wrote:
>
> Reproduce of error (Replicator: Panic: data stack: Out of memory):
> =================================================================>
> When replay on any mail I will got OOM.
>
> I try add in service replicate vsz_limit = 0 but that not help (after
> proces dovecot/replicator eat all ram I got OOM)
>
> Log error:
> =========>
> Jun 03 09:38:59 Warning:
> imap(ivan at myserv.local)<2533034><h7EQN4bgaJ4FvZdE>:
> replication(ivan at myserv.local): Sync failure: Timeout in 2 secs
> Jun 03 09:39:03 Panic: replicator: data stack: Out of memory when
> allocating 4294967336 bytes
> Jun 03 09:39:03 Error: replicator: Raw backtrace:
> /usr/lib/dovecot/libdovecot.so.0(backtrace_append+0x42) [0x7feef08f0582]
> -> /usr/lib/dovecot/libdovecot.so.0(backtrace_get+0x1e) [0x7feef08f069e]
> -> /usr/lib/dovecot/libdovecot.so.0(+0x1022fb) [0x7feef08fd2fb] ->
> /usr/lib/dovecot/libdovecot.so.0(+0x102391) [0x7feef08fd391] ->
> /usr/lib/dovecot/libdovecot.so.0(+0x55589) [0x7feef0850589] ->
> /usr/lib/dovecot/libdovecot.so.0(+0x54d52) [0x7feef084fd52] ->
> /usr/lib/dovecot/libdovecot.so.0(+0xfb808) [0x7feef08f6808] ->
> /usr/lib/dovecot/libdovecot.so.0(+0x122938) [0x7feef091d938] ->
> /usr/lib/dovecot/libdovecot.so.0(+0xf76bf) [0x7feef08f26bf] ->
> /usr/lib/dovecot/libdovecot.so.0(+0xf777e) [0x7feef08f277e] ->
> /usr/lib/dovecot/libdovecot.so.0(buffer_append+0x61) [0x7feef08f2a21] ->
> dovecot/replicator(replicator_queue_push+0x15a) [0x55953ece9b7a] ->
> dovecot/replicator(+0x62b6) [0x55953ece92b6] ->
> dovecot/replicator(+0x594e) [0x55953ece894e] ->
> dovecot/replicator(+0x5b2c) [0x55953ece8b2c] ->
> /usr/lib/dovecot/libdovecot.so.0(io_loop_call_io+0x69) [0x7feef0913529]
> -> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x132)
> [0x7feef0914c12] ->
> /usr/lib/dovecot/libdovecot.so.0(io_loop_handler_run+0x50)
> [0x7feef09135d0] -> /usr/lib/dovecot/libdovecot.so.0(io_loop_run+0x40)
> [0x7feef0913790] ->
> /usr/lib/dovecot/libdovecot.so.0(master_service_run+0x13)
> [0x7feef0886353] -> dovecot/replicator(main+0x18d) [0x55953ece7cbd]
->
> /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xea) [0x7feef0651d0a]
> -> dovecot/replicator(_start+0x2a) [0x55953ece7d7a]
> Jun 03 09:39:03 Fatal: replicator: master: service(replicator): child
> 2532886 killed with signal 6 (core dumps disabled -
> https://dovecot.org/bugreport.html#coredumps)
>
> My configuration:
> =========>
> # 2.3.19 (b3ad6004dc): /etc/dovecot/dovecot.conf
> # Pigeonhole version 0.5.19 (4eae2f79)
> # OS: Linux 5.10.0-13-amd64 x86_64 Debian 11.3 ext4
> # Hostname: mail.myserv.local
> auth_mechanisms = plain login
> auth_verbose = yes
> debug_log_path = /var/log/dovecot.debug
> default_client_limit = 5000
> default_process_limit = 1000
> default_vsz_limit = 2 G
> deliver_log_format = msgid=%m, subject=%s, from=%f, size=%p(%w), %$
> dict {
>  ? quota = pgsql:/etc/dovecot/dovecot-dict-sql.conf.ext
> }
> doveadm_password = # hidden, use -P to show it
> hostname = myserv.local
> lda_mailbox_autocreate = yes
> lda_mailbox_autosubscribe = yes
> listen = *
> log_path = /var/log/dovecot.log
> mail_home = /var/mail/vhosts/%d/mail/%n
> mail_location =
> maildir:/var/mail/vhosts/%d/mail/%n:INDEX=/var/mail/vhosts/%d/indexes/%n
> mail_plugins = " quota fts fts_solr notify replication"
> mail_privileged_group = vmail
> managesieve_notify_capability = mailto
> managesieve_sieve_capability = fileinto reject envelope
> encoded-character vacation subaddress comparator-i;ascii-numeric
> relational regex imap4flags copy include variables body enotify
> environment mailbox date index ihave duplicate mime foreverypart
> extracttext imapsieve vnd.dovecot.imapsieve
> namespace inbox {
>  ? inbox = yes
>  ? location >  ? mailbox Drafts {
>  ??? special_use = \Drafts
>  ? }
>  ? mailbox Junk {
>  ??? special_use = \Junk
>  ? }
>  ? mailbox Sent {
>  ??? special_use = \Sent
>  ? }
>  ? mailbox "Sent Messages" {
>  ??? special_use = \Sent
>  ? }
>  ? mailbox Trash {
>  ??? special_use = \Trash
>  ? }
>  ? prefix >  ? separator = /
> }
> passdb {
>  ? args = /etc/dovecot/dovecot-sql.conf.ext
>  ? driver = sql
> }
> plugin {
>  ? autocreate = Sent
>  ? autocreate2 = Drafts
>  ? autocreate3 = Junk
>  ? autocreate4 = Trash
>  ? autosubscribe = Sent
>  ? autosubscribe2 = Drafts
>  ? autosubscribe3 = Junk
>  ? autosubscribe4 = Trash
>  ? expire = Trash
>  ? expire2 = Trash/*
>  ? expire3 = Spam
>  ? fts = solr
>  ? fts_autoindex = yes
>  ? fts_solr = url=http://localhost:8983/solr/dovecot/
>  ? imapsieve_mailbox1_before = file:/etc/dovecot/sieve/report-spam.sieve
>  ? imapsieve_mailbox1_causes = COPY
>  ? imapsieve_mailbox1_name = Junk
>  ? imapsieve_mailbox2_before = file:/etc/dovecot/sieve/report-ham.sieve
>  ? imapsieve_mailbox2_causes = COPY
>  ? imapsieve_mailbox2_from = Junk
>  ? imapsieve_mailbox2_name = *
>  ? mail_replica = tcp:imap.myserv2.local:12345
>  ? quota = dict:Mailbox::proxy::quota
>  ? quota_rule = *:storage=1G
>  ? quota_warning = storage=95%% quota-warning 95 %u
>  ? quota_warning2 = storage=80%% quota-warning 80 %u
>  ? replication_sync_timeout = 2
>  ? sieve =
>
file:/var/mail/vhosts/%d/sieve/%n;active=/var/mail/vhosts/%d/sieve/%n/.dovecot.sieve
>
>  ? sieve_after = /etc/dovecot/sieve/after.d/
>  ? sieve_global_extensions = +vnd.dovecot.pipe +vnd.dovecot.environment
>  ? sieve_max_actions = 32
>  ? sieve_max_redirects = 4
>  ? sieve_max_script_size = 1M
>  ? sieve_pipe_bin_dir = /etc/dovecot/sieve
>  ? sieve_plugins = sieve_imapsieve sieve_extprograms
>  ? sieve_quota_max_scripts = 50
> }
> postmaster_address = postmaster@%d
> protocols = " imap lmtp sieve pop3"
> service aggregator {
>  ? fifo_listener replication-notify-fifo {
>  ??? user = vmail
>  ? }
>  ? unix_listener replication-notify {
>  ??? user = vmail
>  ? }
> }
> service auth-worker {
>  ? user = vmail
> }
> service auth {
>  ? unix_listener /var/spool/postfix/private/auth {
>  ??? group = postfix
>  ??? mode = 0666
>  ??? user = postfix
>  ? }
> }
> service dict {
>  ? unix_listener dict {
>  ??? user = vmail
>  ? }
> }
> service doveadm {
>  ? inet_listener {
>  ??? port = 12345
>  ??? ssl = no
>  ? }
> }
> service imap-login {
>  ? process_min_avail = 4
>  ? service_count = 1
> }
> service lmtp {
>  ? unix_listener /var/spool/postfix/private/dovecot-lmtp {
>  ??? group = postfix
>  ??? mode = 0600
>  ??? user = postfix
>  ? }
> }
> service quota-warning {
>  ? executable = script /var/lib/dovecot/quota-warning.sh
>  ? unix_listener quota-warning {
>  ??? user = vmail
>  ? }
>  ? user = root
> }
> service replicator {
>  ? process_min_avail = 1
>  ? unix_listener replicator-doveadm {
>  ??? mode = 0600
>  ??? user = vmail
>  ? }
> }
> ssl = required
> ssl_cert = </etc/letsencrypt/live/myserv.local/fullchain.pem
> ssl_client_ca_dir = /etc/ssl/certs
> ssl_key = # hidden, use -P to show it
> ssl_min_protocol = TLSv1.1
> userdb {
>  ? args = /etc/dovecot/dovecot-sql.conf.ext
>  ? driver = sql
> }
> protocol lmtp {
>  ? mail_plugins = " quota fts fts_solr notify replication sieve"
> }
> protocol !indexer-worker {
>  ? mail_vsize_bg_after_count = 100
> }
> protocol lda {
>  ? mail_plugins = " quota fts fts_solr notify replication sieve"
> }
> protocol imap {
>  ? mail_max_userip_connections = 20
>  ? mail_plugins = " quota fts fts_solr notify replication imap_quota
> imap_sieve"
> }
> protocol sieve {
>  ? mail_max_userip_connections = 20
>  ? managesieve_max_line_length = 64 k
> }
> protocol pop3 {
>  ? mail_max_userip_connections = 20
>  ? mail_plugins = " quota fts fts_solr notify replication quota"
> }
>