Apostolis Hardalias
2020-Jul-03 11:06 UTC
Mail replication fails between v2.2.27 and v2.3.4.1
Hello,
I have two installations of dovecot configured to replicate mailboxes
between them. recently, i upgraded the operating system on one of them
(mx2.example.com) and now i'm running one installation on version 2.2.27
(debian stretch) and another on version 2.3.4.1 (mx1.example.com debian
buster).
My setup includes 3 shared namespaces that point to the mailboxes of 3
accounts. these accounts are setup to use maildir format.
After upgrading one of the hosts, i started having issues with failed
replications on those mailboxes.
I tried running the replication manually and repeatedly by issuing the
following on mx1.example.com:
doveadv -d sync -u exception_bucket at example.com -d -l 30 -n -u
and the logs on mx2.example.com included the following lines consistently:
Jul 02 14:21:58 dsync-server(exception_bucket at example.com): Debug: brain
S: Change during sync: Mailbox GUID 2d32711ddfd27250390e0000a712b6e3 was
lost
Jul 02 14:21:58 dsync-server(exception_bucket at example.com): Debug: brain
S: We don't have mailbox 2d32711ddfd27250390e0000a712b6e3
I double checked for the existance of the mailbox with that guid (that
guid points to the inbox) and the mailbox existed on both servers but,
the contents of the mailbox on the two servers where different (missing
e-mails on both servers).
I attempted running a force-resync command as follows but with no luck.
doveadm -d force-resync -u exception_bucket at example.com inbox
but the issue persisted. admittedly, i didn't run the command on both
servers because i read it doesn't replicate after figuring a workaround.
The next thing i tried was attempting replication without the -n flag,
given that you mention that there are some issues replicating shared
namespaces. That didn't fix the replication status either.
After looking around, i came onto the solution on this thread:
https://dovecot.org/pipermail/dovecot/2017-october/109620.html
which proposed setting mailbox_list_index = no.
This was the default setting on version 2.2.27, so i changed that
setting on mx2.example.com which was running version 2.3.4.1.
This allowed the replication to work but i don't think it is a setting i
want in my configuration.
Is there something i can do to re-enable dovecot to check index files
instead of mailboxes? Could that be an issue when replicating between
those two different versions?
I'm attaching the configuration files for both the mailservers, a
separate configuration file is included since mail_replica settings are
set from there, and a link to the compressed log file i pulled during
the operation.
https://filebin.net/d9yygnq8jg74unwb/replication_debug.log.bz2?t=vv2o81ka
Thank you in advance,
Apostolis Hardalias
-------------- next part --------------
driver = mysql
connect = host=127.0.0.1 dbname=mailer user=mailer password=***reducted***
default_pass_scheme = PLAIN-MD5
user_query = SELECT '/home/vmail/%d/%n' as home, 'vmail' as uid,
'vmail' as gid, mailbox as mail, 'tcps:mx1.example.com:12345' as
mail_replica FROM virtual_users LEFT JOIN virtual_domains ON
virtual_users.domain_id = virtual_domains.id WHERE
virtual_users.user='%n' AND virtual_domains.name='%d';
password_query = SELECT password FROM virtual_users LEFT JOIN virtual_domains ON
virtual_users.domain_id = virtual_domains.id WHERE
virtual_users.user='%n' AND virtual_domains.name='%d';
iterate_query = SELECT CONCAT(u.user, '@', d.name) AS user FROM
virtual_users as u LEFT JOIN virtual_domains AS d on u.domain_id = d.id;
-------------- next part --------------
# 2.2.27 (c0f36b0): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.4.16 (fed8554)
# OS: Linux 4.9.0-11-amd64 x86_64 Debian 9.11
auth_mechanisms = plain login
default_vsz_limit = 512 M
doveadm_password = # hidden, use -P to show it
first_valid_uid = 100
haproxy_trusted_networks = 192.168.0.0/24
mail_location = mdbox:~/mdbox:ALT=/srv/mail/alt/%d/%n
mail_log_prefix = "%Ls[%p]: user=<%u>, "
mail_plugins = " acl notify replication acl stats"
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character
vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy
include variables body enotify environment mailbox date index ihave duplicate
mime foreverypart extracttext spamtest spamtestplus
namespace {
location =
maildir:/home/vmail/example.com/exception_bucket/Maildir:INDEXPVT=~/exceptions
prefix = Exceptions.
subscriptions = no
type = public
}
namespace {
location = maildir:/home/vmail/example.com/support/Maildir:INDEXPVT=~/support
at example.com
prefix = Support.
subscriptions = no
type = public
}
namespace {
location = maildir:/home/vmail/example.com/staging/Maildir:INDEXPVT=~/staging
at example.com
prefix = Staging.
subscriptions = no
type = public
}
namespace inbox {
inbox = yes
location =
mailbox Drafts {
auto = subscribe
special_use = \Drafts
}
mailbox Sent {
auto = subscribe
special_use = \Sent
}
mailbox Spam {
auto = subscribe
special_use = \Junk
}
mailbox Trash {
auto = subscribe
special_use = \Trash
}
prefix =
separator = .
type = private
}
passdb {
args = /etc/dovecot/dovecot-sql.conf
driver = sql
}
plugin {
acl = vfile
antispam_backend = spool2dir
antispam_spam = spam;Spam;junk;Junk
antispam_spool2dir_notspam = /var/spool/spamtrainer/%%lu-%u-%%05lu-ham
antispam_spool2dir_spam = /var/spool/spamtrainer/%%lu-%u-%%05lu-spam
antispam_trash = trash;Trash;Deleted Items;Deleted Messages
mail_log_events = delete
mail_log_fields = uid box msgid from subject
sieve = ~/.dovecot.sieve
sieve_before = /home/vmail/global.sieve
sieve_default = /home/vmail/global.sieve
sieve_dir = ~/sieve
sieve_duplicate_default_period = 1h
sieve_duplicate_max_period = 1d
sieve_execute_bin_dir = /usr/local/lib/dovecot/sieve-execute
sieve_extensions = +spamtest +spamtestplus
sieve_global_extensions = +vnd.dovecot.execute
sieve_plugins = sieve_extprograms
sieve_spamtest_max_header = X-Spam-Status: required=([0-9.]+)
sieve_spamtest_status_header = X-Spam-Status: score=(-?[0-9.]+)
sieve_spamtest_status_type = score
stats_refresh = 30secs
stats_track_cmds = yes
}
protocols = " imap sieve"
service aggregator {
fifo_listener replication-notify-fifo {
user = vmail
}
unix_listener replication-notify {
user = vmail
}
}
service anvil {
client_limit = 2000
}
service auth {
client_limit = 2000
unix_listener /var/spool/postfix/private/auth {
group = postfix
mode = 0660
user = postfix
}
unix_listener auth-userdb {
group = vmail
mode = 0777
}
}
service doveadm {
inet_listener {
port = 12345
ssl = yes
}
vsz_limit = 1500 M
}
service imap-login {
inet_listener imap_haproxy {
haproxy = yes
port = 20143
}
inet_listener imaps_haproxy {
haproxy = yes
port = 20993
ssl = yes
}
process_limit = 2048
process_min_avail = 5
}
service imap-postlogin {
executable = script-login /usr/local/sbin/ldap-postlogin
user = $default_internal_user
}
service imap {
executable = imap imap-postlogin
process_limit = 2048
}
service replicator {
process_min_avail = 1
unix_listener replicator-doveadm {
mode = 0600
user = vmail
}
}
service stats {
fifo_listener stats-mail {
mode = 0600
user = vmail
}
fifo_listener stats-user {
mode = 0600
user = vmail
}
}
ssl_cert = </etc/ssl/certs/star.example.com.crt
ssl_client_ca_dir = /etc/ssl/certs
ssl_key = # hidden, use -P to show it
stats_carbon_name = metamail
stats_carbon_server = 192.168.0.12:2003
userdb {
args = /etc/dovecot/dovecot-sql.conf
driver = sql
}
protocol lda {
log_path = /var/log/dovecot/deliver.log
mail_plugins = sieve replication notify
postmaster_address = postmaster at example.com
}
protocol imap {
mail_max_userip_connections = 500
mail_plugins = " acl notify replication acl stats antispam mail_log
imap_stats"
}
-------------- next part --------------
# 2.3.4.1 (f79e8e7e4): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.5.4 ()
# OS: Linux 4.19.0-9-amd64 x86_64 Debian 10.4
# Hostname: mx2.example.com
auth_mechanisms = plain login
default_vsz_limit = 512 M
doveadm_password = # hidden, use -P to show it
first_valid_uid = 100
haproxy_trusted_networks = 192.168.0.0/24
login_greeting = Dovecot ready.
mail_location = mdbox:~/mdbox:ALT=false/%d/%n
mail_plugins = " notify replication acl old_stats"
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character
vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy
include variables body enotify environment mailbox date index ihave duplicate
mime foreverypart extracttext spamtest spamtestplus
namespace {
location =
maildir:/home/vmail/example.com/exception_bucket/Maildir:INDEXPVT=~/exceptions
prefix = Exceptions.
subscriptions = no
type = public
}
namespace {
location = maildir:/home/vmail/example.com/support/Maildir:INDEXPVT=~/support
at example.com
prefix = Support.
subscriptions = no
type = public
}
namespace {
location = maildir:/home/vmail/example.com/staging/Maildir:INDEXPVT=~/staging
at example.com
prefix = Staging.
subscriptions = no
type = public
}
namespace inbox {
inbox = yes
location =
mailbox Drafts {
auto = subscribe
special_use = \Drafts
}
mailbox Sent {
auto = subscribe
special_use = \Sent
}
mailbox Spam {
auto = subscribe
special_use = \Junk
}
mailbox Trash {
auto = subscribe
special_use = \Trash
}
prefix =
separator = .
type = private
}
old_stats_carbon_name = koromail
old_stats_carbon_server = 192.168.0.12:2003
passdb {
args = /etc/dovecot/dovecot-sql.conf
driver = sql
}
plugin {
acl = vfile
antispam_backend = spool2dir
antispam_spam = spam;Spam;junk;Junk
antispam_spool2dir_notspam = /var/spool/spamtrainer/%%lu-%u-%%05lu-ham
antispam_spool2dir_spam = /var/spool/spamtrainer/%%lu-%u-%%05lu-spam
antispam_trash = trash;Trash;Deleted Items;Deleted Messages
mail_log_events = delete
mail_log_fields = uid box msgid from subject
old_stats_refresh = 30s
old_stats_track_cmds = yes
sieve = ~/.dovecot.sieve
sieve_before = /home/vmail/global.sieve
sieve_default = /home/vmail/global.sieve
sieve_dir = ~/sieve
sieve_duplicate_default_period = 1h
sieve_duplicate_max_period = 1d
sieve_execute_bin_dir = /usr/local/lib/dovecot/sieve-execute
sieve_extensions = +spamtest +spamtestplus
sieve_global_extensions = +vnd.dovecot.execute
sieve_plugins = sieve_extprograms
sieve_spamtest_max_header = X-Spam-Status: required=([0-9.]+)
sieve_spamtest_status_header = X-Spam-Status: score=(-?[0-9.]+)
sieve_spamtest_status_type = score
}
protocols = " imap sieve"
service aggregator {
fifo_listener replication-notify-fifo {
user = vmail
}
unix_listener replication-notify {
user = vmail
}
}
service anvil {
client_limit = 2000
}
service auth {
client_limit = 2000
unix_listener /var/spool/postfix/private/auth {
group = postfix
mode = 0660
user = postfix
}
unix_listener auth-userdb {
group = vmail
mode = 0777
}
}
service doveadm {
inet_listener {
port = 12345
ssl = yes
}
vsz_limit = 1500 M
}
service imap-login {
inet_listener imap_haproxy {
haproxy = yes
port = 20143
}
inet_listener imaps_haproxy {
haproxy = yes
port = 20993
ssl = yes
}
process_limit = 2048
process_min_avail = 5
service_count = 0
}
service imap-postlogin {
executable = script-login /usr/local/sbin/ldap-postlogin
user = $default_internal_user
}
service imap {
executable = imap imap-postlogin
process_limit = 2048
}
service old-stats {
fifo_listener old-stats-mail {
mode = 0600
user = vmail
}
fifo_listener old-stats-user {
mode = 0600
user = vmail
}
}
service replicator {
process_min_avail = 1
unix_listener replicator-doveadm {
mode = 0600
user = vmail
}
}
ssl_cert = </etc/ssl/certs/star.example.com.crt
ssl_client_ca_dir = /etc/ssl/certs
ssl_dh = # hidden, use -P to show it
ssl_key = # hidden, use -P to show it
userdb {
args = /etc/dovecot/dovecot-sql.conf
driver = sql
}
protocol lda {
log_path = /var/log/dovecot/deliver.log
mail_plugins = sieve replication notify
postmaster_address = postmaster at example.com
}
protocol imap {
mail_plugins = " notify replication acl old_stats antispam mail_log
imap_old_stats"
}
Possibly Parallel Threads
Wisdom of the Ancients
