Hi, we upgraded our servers from version 2.2.13 to 2.2.27. After the upgrade we notice a 10x increase in traffic with the nfs storage an errors like this in the logfile: Nov 12 09:48:16 mail dovecot: imap(...): Error: Corrupted index cache file /.../dovecot.index.cache: invalid record size Nov 12 09:48:16 mail dovecot: imap(...): Error: unlink(/.../dovecot.index.cache) failed: No such file or directory (in mail-cache.c:29) Nov 12 09:48:16 mail dovecot: imap(...): Error: Corrupted index cache file /.../dovecot.index.cache: invalid record size Nov 12 09:48:16 mail dovecot: imap(...): Error: Broken file /.../dovecot-uidlist line 8: Invalid data: Is this normal ? Will the activity wind down ? Can we do something, like deleting the old dovecot.index* or dovecot-uidlist files from maildirs, or doing an doveadm force-refresh for all inboxes ? Thank you ! Here's my configuration: # 2.2.27 (c0f36b0): /etc/dovecot/dovecot.conf # Pigeonhole version 0.4.16 (fed8554) # OS: Linux 4.9.0-8-amd64 x86_64 Debian 9.6 nfs auth_failure_delay = 15 secs auth_mechanisms = plain login auth_verbose = yes auth_worker_max_count = 256 default_client_limit = 40000 default_process_limit = 512 dict { lastlogin = mysql:/etc/dovecot/mysql/dovecot-dict-lastlogin.conf quotadict = mysql:/etc/dovecot/mysql/dovecot-dict-quota.conf } disable_plaintext_auth = no first_valid_uid = 100 log_timestamp = "%Y-%m-%d %H:%M:%S " login_log_format_elements = user=<%u> method=%m rip=%r lip=%l pid=%p %c mail_fsync = always mail_location = maildir:*/home/virtual/*%d/%u mail_max_userip_connections = 16 mail_nfs_index = yes mail_nfs_storage = yes mail_plugins = zlib quota mail_log notify mail_privileged_group = mail mailbox_list_index = yes maildir_stat_dirs = yes managesieve_notify_capability = mailto managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date ihave mmap_disable = yes namespace inbox { inbox = yes location mailbox Drafts { special_use = \Drafts } mailbox Junk { special_use = \Junk } mailbox Sent { auto = subscribe special_use = \Sent } mailbox "Sent Messages" { special_use = \Sent } mailbox Spam { auto = subscribe special_use = \Junk } mailbox Trash { auto = subscribe special_use = \Trash } mailbox virtual/All { special_use = \All } prefix } passdb { args = /etc/dovecot/mysql/dovecot-sql.conf driver = sql } plugin { last_login_dict = proxy::lastlogin last_login_key = last-login/%u mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename mail_log_fields = uid box msgid size quota = dict:user::proxy::quotadict quota_rule2 = Trash:ignore quota_rule3 = Spam:ignore quota_status_nouser = DUNNO quota_status_overquota = 552 5.2.2 The email account that you tried to reach is over quota quota_status_success = DUNNO quota_warning = storage=95%% quota-warning 95 %u quota_warning2 = storage=80%% quota-warning 80 %u sieve = ~/.dovecot.sieve sieve_before = /etc/dovecot/sieve/default.sieve sieve_dir = ~/sieve stats_refresh = 30 secs stats_track_cmds = yes trash = /etc/dovecot/dovecot-trash.conf zlib_save = gz zlib_save_level = 6 } protocols = imap sieve pop3 lmtp service anvil { client_limit = 20000 unix_listener anvil-auth-penalty { mode = 00 } } service auth { client_limit = 20000 unix_listener /var/spool/postfix/private/auth { group = postfix mode = 0660 user = postfix } unix_listener auth-master { group = mail mode = 0660 user = vmail } user = root vsz_limit = 1 G } service dict { unix_listener dict { mode = 0660 user = vmail } } service imap-login { client_limit = 4096 executable = /usr/lib/dovecot/imap-login inet_listener imap { address = * port = 143 } inet_listener imaps { address = * port = 993 } process_limit = 4096 process_min_avail = 24 service_count = 100 } service imap { executable = /usr/lib/dovecot/imap process_limit = 4096 process_min_avail = 24 vsz_limit = 1 G } service lmtp { process_min_avail = 24 unix_listener /var/spool/postfix/private/dovecot-lmtp { group = postfix mode = 0600 user = postfix } vsz_limit = 1 G } service managesieve-login { client_limit = 4096 inet_listener sieve { port = 4190 } process_limit = 4096 process_min_avail = 24 service_count = 1 } service managesieve { process_limit = 4096 vsz_limit = 1 G } service pop3-login { client_limit = 4096 executable = /usr/lib/dovecot/pop3-login inet_listener pop3 { address = * port = 0 } inet_listener pop3s { address = * port = 995 } process_limit = 4096 process_min_avail = 24 service_count = 100 } service pop3 { executable = /usr/lib/dovecot/pop3 process_limit = 4096 process_min_avail = 24 vsz_limit = 1 G } service quota-status { client_limit = 4096 executable = /usr/lib/dovecot/quota-status -p postfix process_min_avail = 24 unix_listener /var/spool/postfix/private/quota-status { group = postfix mode = 0660 user = postfix } } service quota-warning { executable = script /usr/local/bin/quota-warning.sh unix_listener quota-warning { mode = 0660 user = vmail } user = vmail } userdb { args = /etc/dovecot/mysql/dovecot-sql.conf driver = sql } protocol imap { imap_idle_notify_interval = 4 mins imap_max_line_length = 64 k mail_max_userip_connections = 16 mail_plugins = zlib quota mail_log notify trash imap_quota } protocol pop3 { mail_max_userip_connections = 16 mail_plugins = zlib quota mail_log notify pop3_client_workarounds = outlook-no-nuls oe-ns-eoh pop3_fast_size_lookups = yes pop3_no_flag_updates = yes pop3_uidl_format = %08Xu%08Xv } protocol lda { auth_socket_path = /var/run/dovecot/auth-master mail_plugins = zlib quota mail_log notify sieve sendmail_path = /usr/lib/sendmail syslog_facility = mail } protocol sieve { mail_max_userip_connections = 16 managesieve_max_compile_errors = 5 } protocol lmtp { mail_plugins = zlib quota mail_log notify sieve syslog_facility = mail } -------------- next part -------------- An HTML attachment was scrubbed... URL: <https://dovecot.org/pipermail/dovecot/attachments/20181114/5225b134/attachment-0001.html>
It should eventually wind down once all the problems are fixed. Of course if it does not happen, you can always run force-resync for the problem users. Aki On 14.11.2018 10.08, Adrian M wrote:> Hi, > we upgraded our servers from version 2.2.13 to 2.2.27. After the > upgrade we notice a 10x increase in traffic with the nfs storage an > errors like this in the logfile: > > Nov 12 09:48:16 mail dovecot: imap(...): Error: Corrupted index cache > file /.../dovecot.index.cache: invalid record size > Nov 12 09:48:16 mail dovecot: imap(...): Error: > unlink(/.../dovecot.index.cache) failed: No such file or directory (in > mail-cache.c:29) > Nov 12 09:48:16 mail dovecot: imap(...): Error: Corrupted index cache > file /.../dovecot.index.cache: invalid record size > Nov 12 09:48:16 mail dovecot: imap(...): Error: Broken file > /.../dovecot-uidlist line 8: Invalid data: > > Is this normal ? > Will the activity wind down ? > Can we do something, like deleting the old dovecot.index* or > dovecot-uidlist files from maildirs, or doing an doveadm force-refresh > for all inboxes ? > > Thank you ! > > Here's my configuration: > # 2.2.27 (c0f36b0): /etc/dovecot/dovecot.conf > # Pigeonhole version 0.4.16 (fed8554) > # OS: Linux 4.9.0-8-amd64 x86_64 Debian 9.6 nfs > auth_failure_delay = 15 secs > auth_mechanisms = plain login > auth_verbose = yes > auth_worker_max_count = 256 > default_client_limit = 40000 > default_process_limit = 512 > dict { > lastlogin = mysql:/etc/dovecot/mysql/dovecot-dict-lastlogin.conf > quotadict = mysql:/etc/dovecot/mysql/dovecot-dict-quota.conf > } > disable_plaintext_auth = no > first_valid_uid = 100 > log_timestamp = "%Y-%m-%d %H:%M:%S " > login_log_format_elements = user=<%u> method=%m rip=%r lip=%l pid=%p %c > mail_fsync = always > mail_location = maildir://home/virtual//%d/%u > mail_max_userip_connections = 16 > mail_nfs_index = yes > mail_nfs_storage = yes > mail_plugins = zlib quota mail_log notify > mail_privileged_group = mail > mailbox_list_index = yes > maildir_stat_dirs = yes > managesieve_notify_capability = mailto > managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date ihave > mmap_disable = yes > namespace inbox { > inbox = yes > location = > mailbox Drafts { > special_use = \Drafts > } > mailbox Junk { > special_use = \Junk > } > mailbox Sent { > auto = subscribe > special_use = \Sent > } > mailbox "Sent Messages" { > special_use = \Sent > } > mailbox Spam { > auto = subscribe > special_use = \Junk > } > mailbox Trash { > auto = subscribe > special_use = \Trash > } > mailbox virtual/All { > special_use = \All > } > prefix = > } > passdb { > args = /etc/dovecot/mysql/dovecot-sql.conf > driver = sql > } > plugin { > last_login_dict = proxy::lastlogin > last_login_key = last-login/%u > mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename > mail_log_fields = uid box msgid size > quota = dict:user::proxy::quotadict > quota_rule2 = Trash:ignore > quota_rule3 = Spam:ignore > quota_status_nouser = DUNNO > quota_status_overquota = 552 5.2.2 The email account that you tried to reach is over quota > quota_status_success = DUNNO > quota_warning = storage=95%% quota-warning 95 %u > quota_warning2 = storage=80%% quota-warning 80 %u > sieve = ~/.dovecot.sieve > sieve_before = /etc/dovecot/sieve/default.sieve > sieve_dir = ~/sieve > stats_refresh = 30 secs > stats_track_cmds = yes > trash = /etc/dovecot/dovecot-trash.conf > zlib_save = gz > zlib_save_level = 6 > } > protocols = imap sieve pop3 lmtp > service anvil { > client_limit = 20000 > unix_listener anvil-auth-penalty { > mode = 00 > } > } > service auth { > client_limit = 20000 > unix_listener /var/spool/postfix/private/auth { > group = postfix > mode = 0660 > user = postfix > } > unix_listener auth-master { > group = mail > mode = 0660 > user = vmail > } > user = root > vsz_limit = 1 G > } > service dict { > unix_listener dict { > mode = 0660 > user = vmail > } > } > service imap-login { > client_limit = 4096 > executable = /usr/lib/dovecot/imap-login > inet_listener imap { > address = * > port = 143 > } > inet_listener imaps { > address = * > port = 993 > } > process_limit = 4096 > process_min_avail = 24 > service_count = 100 > } > service imap { > executable = /usr/lib/dovecot/imap > process_limit = 4096 > process_min_avail = 24 > vsz_limit = 1 G > } > service lmtp { > process_min_avail = 24 > unix_listener /var/spool/postfix/private/dovecot-lmtp { > group = postfix > mode = 0600 > user = postfix > } > vsz_limit = 1 G > } > service managesieve-login { > client_limit = 4096 > inet_listener sieve { > port = 4190 > } > process_limit = 4096 > process_min_avail = 24 > service_count = 1 > } > service managesieve { > process_limit = 4096 > vsz_limit = 1 G > } > service pop3-login { > client_limit = 4096 > executable = /usr/lib/dovecot/pop3-login > inet_listener pop3 { > address = * > port = 0 > } > inet_listener pop3s { > address = * > port = 995 > } > process_limit = 4096 > process_min_avail = 24 > service_count = 100 > } > service pop3 { > executable = /usr/lib/dovecot/pop3 > process_limit = 4096 > process_min_avail = 24 > vsz_limit = 1 G > } > service quota-status { > client_limit = 4096 > executable = /usr/lib/dovecot/quota-status -p postfix > process_min_avail = 24 > unix_listener /var/spool/postfix/private/quota-status { > group = postfix > mode = 0660 > user = postfix > } > } > service quota-warning { > executable = script /usr/local/bin/quota-warning.sh > unix_listener quota-warning { > mode = 0660 > user = vmail > } > user = vmail > } > > userdb { > args = /etc/dovecot/mysql/dovecot-sql.conf > driver = sql > } > protocol imap { > imap_idle_notify_interval = 4 mins > imap_max_line_length = 64 k > mail_max_userip_connections = 16 > mail_plugins = zlib quota mail_log notify trash imap_quota > } > protocol pop3 { > mail_max_userip_connections = 16 > mail_plugins = zlib quota mail_log notify > pop3_client_workarounds = outlook-no-nuls oe-ns-eoh > pop3_fast_size_lookups = yes > pop3_no_flag_updates = yes > pop3_uidl_format = %08Xu%08Xv > } > protocol lda { > auth_socket_path = /var/run/dovecot/auth-master > mail_plugins = zlib quota mail_log notify sieve > sendmail_path = /usr/lib/sendmail > syslog_facility = mail > } > protocol sieve { > mail_max_userip_connections = 16 > managesieve_max_compile_errors = 5 > } > protocol lmtp { > mail_plugins = zlib quota mail_log notify sieve > syslog_facility = mail > }-------------- next part -------------- An HTML attachment was scrubbed... URL: <https://dovecot.org/pipermail/dovecot/attachments/20181114/6c636203/attachment.html>
Thank you ! I was little concerned that the following settings are not in line with the new version: mail_nfs_index = yes mail_nfs_storage = yes mail_fsync = always mailbox_list_index = yes maildir_stat_dirs = yes mmap_disable = yes On Wed, Nov 14, 2018 at 10:19 AM Aki Tuomi <aki.tuomi at open-xchange.com> wrote:> It should eventually wind down once all the problems are fixed. Of course > if it does not happen, you can always run force-resync for the problem > users. > > Aki > On 14.11.2018 10.08, Adrian M wrote: > > Hi, > we upgraded our servers from version 2.2.13 to 2.2.27. After the upgrade > we notice a 10x increase in traffic with the nfs storage an errors like > this in the logfile: > > Nov 12 09:48:16 mail dovecot: imap(...): Error: Corrupted index cache file > /.../dovecot.index.cache: invalid record size > Nov 12 09:48:16 mail dovecot: imap(...): Error: > unlink(/.../dovecot.index.cache) failed: No such file or directory (in > mail-cache.c:29) > Nov 12 09:48:16 mail dovecot: imap(...): Error: Corrupted index cache file > /.../dovecot.index.cache: invalid record size > Nov 12 09:48:16 mail dovecot: imap(...): Error: Broken file > /.../dovecot-uidlist line 8: Invalid data: > > Is this normal ? > Will the activity wind down ? > Can we do something, like deleting the old dovecot.index* or > dovecot-uidlist files from maildirs, or doing an doveadm force-refresh for > all inboxes ? > > Thank you ! > > Here's my configuration: > > # 2.2.27 (c0f36b0): /etc/dovecot/dovecot.conf > # Pigeonhole version 0.4.16 (fed8554) > # OS: Linux 4.9.0-8-amd64 x86_64 Debian 9.6 nfs > auth_failure_delay = 15 secs > auth_mechanisms = plain login > auth_verbose = yes > auth_worker_max_count = 256 > default_client_limit = 40000 > default_process_limit = 512 > dict { > lastlogin = mysql:/etc/dovecot/mysql/dovecot-dict-lastlogin.conf > quotadict = mysql:/etc/dovecot/mysql/dovecot-dict-quota.conf > } > disable_plaintext_auth = no > first_valid_uid = 100 > log_timestamp = "%Y-%m-%d %H:%M:%S " > login_log_format_elements = user=<%u> method=%m rip=%r lip=%l pid=%p %c > mail_fsync = always > mail_location = maildir:*/home/virtual/*%d/%u > mail_max_userip_connections = 16 > mail_nfs_index = yes > mail_nfs_storage = yes > mail_plugins = zlib quota mail_log notify > mail_privileged_group = mail > mailbox_list_index = yes > maildir_stat_dirs = yes > managesieve_notify_capability = mailto > managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date ihave > mmap_disable = yes > namespace inbox { > inbox = yes > location > mailbox Drafts { > special_use = \Drafts > } > mailbox Junk { > special_use = \Junk > } > mailbox Sent { > auto = subscribe > special_use = \Sent > } > mailbox "Sent Messages" { > special_use = \Sent > } > mailbox Spam { > auto = subscribe > special_use = \Junk > } > mailbox Trash { > auto = subscribe > special_use = \Trash > } > mailbox virtual/All { > special_use = \All > } > prefix > } > passdb { > args = /etc/dovecot/mysql/dovecot-sql.conf > driver = sql > } > plugin { > last_login_dict = proxy::lastlogin > last_login_key = last-login/%u > mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename > mail_log_fields = uid box msgid size > quota = dict:user::proxy::quotadict > quota_rule2 = Trash:ignore > quota_rule3 = Spam:ignore > quota_status_nouser = DUNNO > quota_status_overquota = 552 5.2.2 The email account that you tried to reach is over quota > quota_status_success = DUNNO > quota_warning = storage=95%% quota-warning 95 %u > quota_warning2 = storage=80%% quota-warning 80 %u > sieve = ~/.dovecot.sieve > sieve_before = /etc/dovecot/sieve/default.sieve > sieve_dir = ~/sieve > stats_refresh = 30 secs > stats_track_cmds = yes > trash = /etc/dovecot/dovecot-trash.conf > zlib_save = gz > zlib_save_level = 6 > } > protocols = imap sieve pop3 lmtp > service anvil { > client_limit = 20000 > unix_listener anvil-auth-penalty { > mode = 00 > } > } > service auth { > client_limit = 20000 > unix_listener /var/spool/postfix/private/auth { > group = postfix > mode = 0660 > user = postfix > } > unix_listener auth-master { > group = mail > mode = 0660 > user = vmail > } > user = root > vsz_limit = 1 G > } > service dict { > unix_listener dict { > mode = 0660 > user = vmail > } > } > service imap-login { > client_limit = 4096 > executable = /usr/lib/dovecot/imap-login > inet_listener imap { > address = * > port = 143 > } > inet_listener imaps { > address = * > port = 993 > } > process_limit = 4096 > process_min_avail = 24 > service_count = 100 > } > service imap { > executable = /usr/lib/dovecot/imap > process_limit = 4096 > process_min_avail = 24 > vsz_limit = 1 G > } > service lmtp { > process_min_avail = 24 > unix_listener /var/spool/postfix/private/dovecot-lmtp { > group = postfix > mode = 0600 > user = postfix > } > vsz_limit = 1 G > } > service managesieve-login { > client_limit = 4096 > inet_listener sieve { > port = 4190 > } > process_limit = 4096 > process_min_avail = 24 > service_count = 1 > } > service managesieve { > process_limit = 4096 > vsz_limit = 1 G > } > service pop3-login { > client_limit = 4096 > executable = /usr/lib/dovecot/pop3-login > inet_listener pop3 { > address = * > port = 0 > } > inet_listener pop3s { > address = * > port = 995 > } > process_limit = 4096 > process_min_avail = 24 > service_count = 100 > } > service pop3 { > executable = /usr/lib/dovecot/pop3 > process_limit = 4096 > process_min_avail = 24 > vsz_limit = 1 G > } > service quota-status { > client_limit = 4096 > executable = /usr/lib/dovecot/quota-status -p postfix > process_min_avail = 24 > unix_listener /var/spool/postfix/private/quota-status { > group = postfix > mode = 0660 > user = postfix > } > } > service quota-warning { > executable = script /usr/local/bin/quota-warning.sh > unix_listener quota-warning { > mode = 0660 > user = vmail > } > user = vmail > } > > userdb { > args = /etc/dovecot/mysql/dovecot-sql.conf > driver = sql > } > protocol imap { > imap_idle_notify_interval = 4 mins > imap_max_line_length = 64 k > mail_max_userip_connections = 16 > mail_plugins = zlib quota mail_log notify trash imap_quota > } > protocol pop3 { > mail_max_userip_connections = 16 > mail_plugins = zlib quota mail_log notify > pop3_client_workarounds = outlook-no-nuls oe-ns-eoh > pop3_fast_size_lookups = yes > pop3_no_flag_updates = yes > pop3_uidl_format = %08Xu%08Xv > } > protocol lda { > auth_socket_path = /var/run/dovecot/auth-master > mail_plugins = zlib quota mail_log notify sieve > sendmail_path = /usr/lib/sendmail > syslog_facility = mail > } > protocol sieve { > mail_max_userip_connections = 16 > managesieve_max_compile_errors = 5 > } > protocol lmtp { > mail_plugins = zlib quota mail_log notify sieve > syslog_facility = mail > } > >-------------- next part -------------- An HTML attachment was scrubbed... URL: <https://dovecot.org/pipermail/dovecot/attachments/20181114/6b2d6d9e/attachment.html>