We're using Dovecot 2.2.4 and mdbox storage with compression.
I noticed yesterday that at least one of the accounts has unusual files in
it's mail/storage directory. This account has approximately 17.5G of
compressed mail, across about 750 storage files (m.###) using 8G of storage.
Starting on Dec 11, storage files m.409 through m.731 have not only their m.###
file, but also a m.###.broken file. At the same time .temp.<unix
time>.<unique id>.<hostname> files also started showing up...
...
-rw--w---- 1 user123 mail 3225 Dec 12 04:38 m.708
-rw--w---- 1 user123 mail 167106 Dec 12 03:35 m.708.broken
-rw--w---- 1 user123 mail 46155 Dec 12 04:38 m.709
-rw--w---- 1 user123 mail 177267 Dec 12 03:40 m.709.broken
-rw--w---- 1 user123 mail 20965889 Nov 26 19:22 m.71
-rw--w---- 1 user123 mail 133248 Dec 12 04:38 m.710
-rw--w---- 1 user123 mail 264361 Dec 12 03:40 m.710.broken
-rw--w---- 1 user123 mail 103407 Dec 12 04:38 m.711
-rw--w---- 1 user123 mail 627736 Dec 12 03:40 m.711.broken
-rw--w---- 1 user123 mail 5132 Dec 12 04:38 m.712
-rw--w---- 1 user123 mail 136244 Dec 12 03:40 m.712.broken
-rw--w---- 1 user123 mail 187224 Dec 12 04:38 m.713
-rw--w---- 1 user123 mail 351104 Dec 12 03:45 m.713.broken
-rw--w---- 1 user123 mail 435793 Dec 12 04:38 m.714
-rw--w---- 1 user123 mail 566906 Dec 12 03:45 m.714.broken
-rw--w---- 1 user123 mail 60916 Dec 12 04:38 m.715
-rw--w---- 1 user123 mail 192028 Dec 12 03:50 m.715.broken
-rw--w---- 1 user123 mail 38051 Dec 12 04:38 m.716
-rw--w---- 1 user123 mail 169163 Dec 12 03:50 m.716.broken
-rw--w---- 1 user123 mail 226067 Dec 12 04:38 m.717
-rw--w---- 1 user123 mail 389948 Dec 12 04:00 m.717.broken
...
-rw--w---- 1 user123 mail 20967506 Dec 2 17:32 m.96
-rw--w---- 1 user123 mail 20967131 Dec 2 18:40 m.97
-rw--w---- 1 user123 mail 20969078 Dec 2 18:38 m.98
-rw--w---- 1 user123 mail 20963978 Dec 2 19:23 m.99
-rw--w---- 1 user123 mail 294968 Dec 11 20:05
.temp.1386813944.P9142Q0M123686.myhost
-rw--w---- 1 user123 mail 294968 Dec 11 20:05
.temp.1386813967.P9213Q0M387733.myhost
-rw--w---- 1 user123 mail 294968 Dec 11 20:05
.temp.1386813970.P9231Q0M371190.myhost
-rw--w---- 1 user123 mail 327736 Dec 11 20:05
.temp.1386813972.P9173Q0M932070.myhost
-rw--w---- 1 user123 mail 327736 Dec 11 20:05
.temp.1386813972.P9239Q0M114754.myhost
-rw--w---- 1 user123 mail 327736 Dec 11 20:05
.temp.1386813983.P9261Q0M640646.myhost
-rw--w---- 1 user123 mail 1048632 Dec 11 20:06
.temp.1386814038.P9515Q0M530600.myhost
I'm not really seeing anything that stands out in the maillog during this
time.
What caused this problem? How concerned should I be about possible lost email?
This is a production environment.
Below is the dovecot -n output.
--
Michael Smith
# dovecot -n
# 2.2.4: /etc/dovecot/dovecot.conf
# OS: Linux 2.6.32-279.19.1.el6.x86_64 x86_64 CentOS release 6.3 (Final)
auth_debug = yes
auth_debug_passwords = yes
auth_default_realm = mydomain.com
auth_master_user_separator = *
auth_mechanisms = plain login
auth_verbose = yes
auth_verbose_passwords = plain
disable_plaintext_auth = no
dotlock_use_excl = no
lda_mailbox_autocreate = yes
lock_method = dotlock
mail_access_groups = mail
mail_debug = yes
mail_fsync = always
mail_location = mdbox:~/mail:INDEX=~/index
mail_plugins = quota zlib
mail_privileged_group = mail
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character
vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy
include variables body enotify environment mailbox date
mbox_read_locks = dotlock
mbox_write_locks = dotlock
mdbox_rotate_interval = 1 weeks
mdbox_rotate_size = 20 M
mmap_disable = yes
namespace {
inbox = yes
location mailbox Drafts {
auto = subscribe
special_use = \Drafts
}
mailbox Junk {
auto = subscribe
special_use = \Junk
}
mailbox Sent {
auto = subscribe
special_use = \Sent
}
mailbox Trash {
auto = subscribe
special_use = \Trash
}
prefix type = private
}
passdb {
args = /etc/dovecot/dovecot-sql-master.conf.ext
driver = sql
master = yes
pass = yes
}
passdb {
args = /etc/dovecot/dovecot-sql.conf.ext
driver = sql
}
plugin {
autosubscribe = Trash
autosubscribe2 = Junk
autosubscribe3 = Drafts
autosubscribe4 = Sent
autosubscribe5 = INBOX
mail_max_userip_connections = 10
managesieve_max_line_length = 65536
quota = dict:User quota::file:%h/dovecot.quota
quota2_rule = Trash:storage=+10%%
quota3_rule = Junk:storage=+20%%
quota_rule = *:storage=100M:messages=100000
recipient_delimiter = +
sieve_before = /var/opt/mail/global.sieve/
zlib_save = gz
zlib_save_level = 9
}
pop3_reuse_xuidl = yes
protocols = imap pop3 sieve lmtp
service auth {
inet_listener auth {
port = 113
}
unix_listener auth-userdb {
user = nobody
}
}
service lmtp {
inet_listener lmtp {
port = 24
}
process_min_avail = 10
service_count = 1
}
service managesieve-login {
inet_listener sieve {
port = 4190
}
process_min_avail = 0
service_count = 1
vsz_limit = 64 M
}
ssl = no
ssl_cert = </etc/pki/dovecot/certs/dovecot.pem
ssl_key = </etc/pki/dovecot/private/dovecot.pem
userdb {
args = /etc/dovecot/dovecot-sql.conf.ext
driver = sql
}
protocol lda {
mail_plugins = quota zlib sieve
}
protocol imap {
mail_plugins = quota zlib imap_quota
}
protocol sieve {
mail_max_userip_connections = 10
mail_plugins = quota zlib
managesieve_max_line_length = 65536
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character
vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy
include variables body enotify environment mailbox date
}
protocol lmtp {
mail_plugins = quota zlib sieve
}
On 13.12.2013, at 17.49, Michael Smith (DF) <msmith at datafoundry.com> wrote:> We're using Dovecot 2.2.4 and mdbox storage with compression. > > I noticed yesterday that at least one of the accounts has unusual files in it's mail/storage directory. This account has approximately 17.5G of compressed mail, across about 750 storage files (m.###) using 8G of storage. > > Starting on Dec 11, storage files m.409 through m.731 have not only their m.### file, but also a m.###.broken file. At the same time .temp.<unix time>.<unique id>.<hostname> files also started showing up... > ... > -rw--w---- 1 user123 mail 3225 Dec 12 04:38 m.708 > -rw--w---- 1 user123 mail 167106 Dec 12 03:35 m.708.broken > -rw--w---- 1 user123 mail 46155 Dec 12 04:38 m.709 > -rw--w---- 1 user123 mail 177267 Dec 12 03:40 m.709.broken..> What caused this problem?Something caused mdbox rebuilding, which triggered this bug: http://hg.dovecot.org/dovecot-2.2/rev/f965670a7b69> How concerned should I be about possible lost email? This is a production environment.It's very likely there is some lost emails in the *.broken files. Fix would be to 1) upgrade to v2.2.6 or newer 2) take a backup of the mdbox 3) move *.broken to their original names. 4) doveadm force-resync -u user at domain INBOX The main problem here is that after Dovecot fixed e.g. m.1234 file and copied the original to m.1234.broken, it could still have added some new mails to m.1234 file and by replacing it those mails would get lost. This one is a bit tricky to check and to fix.. One possibility would be to use "doveadm dump" for the m.1234 and m.1234.broken files and verify that the .broken file has all the GUIDs that m.1234 has (and more). Except it's possible that user would have intentionally deleted some of those mails, so if you bring some mails back user might have to re-delete some of them.