I'm running dovecots replicator successfully on 33 mailboxes. There is
only one mailbox (the biggest one, 167000 mails) that is not replicating
successfully.
After 10 minutes the "source" starts logging these messages:
dovecot: dsync-local(foo at example.com): Error: dsync(imap.netsend.nl):
I/O has stalled, no activity for 600 seconds
dovecot: dsync-local(foo at example.com): Error: Timeout during
state=sync_mails (send=mail_requests recv=changes)
dovecot: dsync-local(foo at example.com): Error: Remote command process
isn't dying, killing it
dovecot: dsync-local(foo at example.com): Error: kill(17459, SIGKILL)
failed: Operation not permitted
dovecot: doveadm: Error: Pseudo-terminal will not be allocated because
stdin is not a terminal.
dovecot: doveadm: Error: dsync-remote(foo at example.com): Error: Couldn't
lock //.dovecot-sync.lock: Timed out after 30 seconds
dovecot: doveadm: Error: Pseudo-terminal will not be allocated because
stdin is not a terminal.
dovecot: doveadm: Error: dsync-remote(foo at example.com): Error: Couldn't
lock //.dovecot-sync.lock: Timed out after 30 seconds
The 30 seconds timeout keeps repeating, while the doveadm process on the
destination keeps on going for hours and hours.
Destination imap.netsend.nl is creating a lot of hardlinks (at least one
hardlink per message) in .temp.... files. This takes about 1,5 hour,
after that the number of hardlinks decline again. But it looks like the
connection with source is already lost.
I've patched the source to overcome the timeout by changing it from 10
minutes to 3 hours[1] but now I'm running into a memory error:
dovecot: dsync-local(foo at example.com): Error: dsync(imap.netsend.nl):
I/O has stalled, no activity for 10800 seconds
dovecot: dsync-local(foo at example.com): Error: Timeout during
state=sync_mails (send=done recv=mails)
dovecot: dsync-local(foo at example.com): Error: mremap_anon(211677184)
failed: Cannot allocate memory
The allocate memory error shows up every 10 seconds.
Next I'll try to look into the memory allocation code, but maybe someone
already has a better idea?
Thanks.
-Tim
Config on source:
# 2.2.15: /etc/dovecot/dovecot.conf
# OS: OpenBSD 5.7 amd64
auth_debug = yes
auth_mechanisms = plain login
auth_verbose = yes
dsync_remote_cmd = /usr/bin/ssh imap.netsend.nl
first_valid_gid = 30000
first_valid_uid = 30000
last_valid_gid = 30100
last_valid_uid = 30100
mail_debug = yes
mail_home = /srv/vmail/%d/%n
mail_location = maildir:~/Maildir
mail_plugins = " notify replication"
passdb {
args = /passwd
driver = passwd-file
}
passdb {
args = /passwd_smtponly
driver = passwd-file
}
plugin {
mail_replica = remoteprefix:imap.netsend.nl
}
protocols = imap lmtp
service aggregator {
fifo_listener replication-notify-fifo {
mode = 0666
}
unix_listener replication-notify {
mode = 0666
}
}
service auth-worker {
chroot = /etc/mail
unix_listener /etc/mail/auth-worker {
mode = 0600
user = $default_internal_user
}
user = $default_internal_user
}
service auth {
chroot = /etc/mail
unix_listener /var/spool/postfix/private/dovecot-auth {
group = _postfix
mode = 0600
user = _postfix
}
}
service imap-login {
inet_listener imap {
port = 0
}
process_min_avail = 1
}
service lmtp {
unix_listener /var/spool/postfix/private/dovecot-lmtp {
group = _postfix
mode = 0600
user = _postfix
}
}
service replicator {
process_min_avail = 1
unix_listener replicator-doveadm {
mode = 0600
}
}
ssl = required
ssl_cert = </etc/ssl/private/2015/chain.crt
ssl_cipher_list =
ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:DHE-RSA-AES256-SHA256:ECDHE-RSA-AES256-SHA:DHE-RSA-AES256-SHA;
ssl_dh_parameters_length = 2048
ssl_key = </etc/ssl/private/the.key
ssl_prefer_server_ciphers = yes
ssl_protocols = !SSLv2 !SSLv3
userdb {
args = /passwd
default_fields = uid=30000 gid=30000 home=/srv/vmail/%d/%n/./
driver = passwd-file
}
valid_chroot_dirs = /srv/vmail
verbose_ssl = yes
protocol imap {
mail_max_userip_connections = 20
}
config on destination imap.netsend.nl:
# 2.2.15: /etc/dovecot/dovecot.conf
# OS: OpenBSD 5.7 amd64
auth_debug = yes
auth_mechanisms = plain login
auth_verbose = yes
first_valid_gid = 30000
first_valid_uid = 30000
last_valid_gid = 30100
last_valid_uid = 30100
mail_debug = yes
mail_home = /srv/vmail/%d/%n
mail_location = sdbox:~/dbox
passdb {
args = /passwd
driver = passwd-file
}
passdb {
args = /passwd_smtponly
driver = passwd-file
}
protocols = imap lmtp
service auth-worker {
chroot = /etc/mail
unix_listener /etc/mail/auth-worker {
mode = 0600
user = $default_internal_user
}
user = $default_internal_user
}
service auth {
chroot = /etc/mail
unix_listener /var/spool/postfix/private/dovecot-auth {
group = _postfix
mode = 0600
user = _postfix
}
}
service imap-login {
inet_listener imap {
port = 0
}
process_min_avail = 1
}
service lmtp {
unix_listener /var/spool/postfix/private/dovecot-lmtp {
group = _postfix
mode = 0600
user = _postfix
}
}
ssl = required
ssl_cert = </etc/ssl/private/2015/chain.crt
ssl_cipher_list =
ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:DHE-RSA-AES256-SHA256:ECDHE-RSA-AES256-SHA:DHE-RSA-AES256-SHA;
ssl_dh_parameters_length = 2048
ssl_key = </etc/ssl/private/the.key
ssl_prefer_server_ciphers = yes
ssl_protocols = !SSLv2 !SSLv3
userdb {
args = /passwd
default_fields = uid=30000 gid=30000 home=/srv/vmail/%d/%n/./
driver = passwd-file
}
valid_chroot_dirs = /srv/vmail
verbose_ssl = yes
protocol imap {
mail_max_userip_connections = 20
}
[1]
--- src/doveadm/dsync/dsync-ibc-stream.c 2014-09-29
13:07:07.000000000 +0200
+++ src/doveadm/dsync/dsync-ibc-stream.c2 2015-06-15
12:28:56.000000000 +0200
@@ -24,7 +24,7 @@
#include <stdlib.h>
-#define DSYNC_IBC_STREAM_TIMEOUT_MSECS (60*10*1000)
+#define DSYNC_IBC_STREAM_TIMEOUT_MSECS (60*180*1000)
#define DSYNC_IBC_STREAM_OUTBUF_THROTTLE_SIZE (1024*128)
#define DSYNC_PROTOCOL_VERSION_MAJOR 3
