Hanno Stock
2019-Oct-01 17:03 UTC
Lost mails with sieve "duplicate" extension in case of failure
Hi!
we just experienced the following issue:
We have a global sieve_before script using the duplicate extension.
require ["duplicate", "imap4flags"];
if duplicate {
discard;
}
Now the storage of our mailbox store got full and lmtp deliveries
started to fail.
This resulted in logs like this:
Sep 25 16:14:56 mail-rc1 dovecot: lmtp(USERNAME): Error:
o_stream_send_istream(/var/mail/vhosts/mail.COMPANY.com/USERNAME/.dovecot/tmp/1569420896.M612482P5820.mail-rc1)
failed: No space left on device
Sep 25 16:14:56 mail-rc1 dovecot: lmtp(USERNAME): Error:
write(/var/mail/vhosts/mail.COMPANY.com/USERNAME/.dovecot/tmp/1569420896.M612482P5820.mail-rc1)
failed: No space left on device
Sep 25 16:14:56 mail-rc1 dovecot: lmtp(USERNAME): Error:
KT9HJGB2i128FgAATXMPig: sieve:
msgid=<redmine.journal-441799.20190925141453.94a97fd570706f2f at
mail.COMPANY.com>:
failed to store into mailbox 'INBOX': Internal error occurred. Refer to
server log for more information. [2019-09-25 16:14:56]
Sep 25 16:14:56 mail-rc1 dovecot: lmtp(USERNAME): Error:
KT9HJGB2i128FgAATXMPig: sieve: Execution of script
/mnt/data/var/mail/vhosts/mail.COMPANY.com/USERNAME/.sieve/.dovecot.sieve was
aborted due to temporary failure (user logfile
/mnt/data/var/mail/vhosts/mail.COMPANY.com/USERNAME/.sieve/.dovecot.sieve.log
may reveal additional details)
Sep 25 16:14:56 mail-rc1 postfix/lmtp[5851]: 4D27690AC:
to=<USERNAME at mail.COMPANY.com>, orig_to=<USER.NAME at
COMPANY.com>,
relay=mail-rc1.bvd.COMPANY.x[private/dovecot-lmtp], delay=1.3,
delays=1.3/0/0/0.01, dsn=4.2.0, status=deferred (host
mail-rc1.bvd.COMPANY.x[private/dovecot-lmtp] said: 451 4.2.0
<USERNAME at mail.COMPANY.com> Internal error occurred. Refer to server
log
for more information. [2019-09-25 16:14:56] (in reply to end of DATA
command))
So far, so good... A 451 response is what we wanted.
However on the next delivery attempt the following happens:
Sep 25 16:24:00 mail-rc1 postfix/lmtp[7040]: 4D27690AC:
to=<USERNAME at mail.COMPANY.com>, orig_to=<USER.NAME at
COMPANY.com>,
relay=mail-rc1.bvd.COMPANY.x[private/dovecot-lmtp], delay=545,
delays=545/0.09/0.02/0.01, dsn=2.0.0, status=sent (250 2.0.0
<USERNAME at mail.COMPANY.com> DBcCEYB4i11qGwAATXMPig Saved)
Sep 25 16:24:00 mail-rc1 dovecot: lmtp(USERNAME):
DBcCEYB4i11qGwAATXMPig: sieve:
msgid=<redmine.journal-441799.20190925141453.94a97fd570706f2f at
mail.COMPANY.com>:
marked message to be discarded if not explicitly delivered (discard action)
Sep 25 16:24:00 mail-rc1 postfix/lmtp[7040]: 4D27690AC:
to=<USERNAME at mail.COMPANY.com>, orig_to=<USER.NAME at
COMPANY.com>,
relay=mail-rc1.bvd.COMPANY.x[private/dovecot-lmtp], delay=545,
delays=545/0.09/0.02/0.01, dsn=2.0.0, status=sent (250 2.0.0
<USERNAME at mail.COMPANY.com> DBcCEYB4i11qGwAATXMPig Saved)
As the duplicate extension has already seen the mail, it is now
immediately discarded.
RFC 7352 however explicitly states:
Implementations MUST only update the internal duplicate-tracking list
when the Sieve script execution finishes successfully. If failing
script executions add the unique ID to the duplicate-tracking list,
all "duplicate" tests in the Sieve script would erroneously yield
"true" for the next delivery attempt of the same message.
Dovecot seems not to honour this specification.
Maybe this is because the "before_script" is not the script that
fails,
but the subsequent scripts fail?
Am I missing some configuration option that would propagate the
subsequent failure back to the before_script?
Or is this a general limitation of the before_script mechanism?
Any help appreciated how to alleviate this issue!
Best regards
Hanno
---
Filesystem: ext4
Dovecot config:
# 2.2.27 (c0f36b0): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.4.16 (fed8554)
# OS: Linux 4.9.0-11-amd64 x86_64 Debian 9.11
auth_cache_negative_ttl = 0
auth_cache_size = 1 M
auth_cache_ttl = 2 hours
auth_mechanisms = plain login
auth_username_format = %Ln
mail_gid = vmail
mail_home = /var/mail/vhosts/mail.COMPANY.com/%n
mail_location = maildir:~/.dovecot
mail_plugins = " acl"
mail_privileged_group = vmail
mail_uid = vmail
mailbox_list_index = yes
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope
encoded-character vacation subaddress comparator-i;ascii-numeric
relational regex imap4flags copy include variables body enotify
environment mailbox date index ihave duplicate mime foreverypart
extracttext editheader imapflags notify
namespace {
hidden = yes
list = no
location = maildir:~/.dovecot-top-namespace
prefix separator = .
subscriptions = yes
type = private
}
namespace {
location
maildir:/var/mail/vhosts/mail.COMPANY.com/%%n/.dovecot:INDEXPVT=/var/mail/vhosts/mail.COMPANY.com/%n/.dovecot/shared/%%n
prefix = shared.%%u.
separator = .
subscriptions = no
type = shared
}
namespace inbox {
inbox = yes
location mailbox Drafts {
special_use = \Drafts
}
mailbox Junk {
special_use = \Junk
}
mailbox Sent {
special_use = \Sent
}
mailbox "Sent Messages" {
special_use = \Sent
}
mailbox Trash {
special_use = \Trash
}
prefix = INBOX.
separator = .
}
passdb {
args = /etc/dovecot/dovecot-ldap.conf.ext
driver = ldap
}
plugin {
acl = vfile
acl_shared_dict = file:/var/lib/dovecot/db/shared-mailboxes.db
sieve
file:/var/mail/vhosts/mail.COMPANY.com/%n/.sieve/sieve;active=/var/mail/vhosts/mail.COMPANY.com/%n/.sieve/.dovecot.sieve
sieve_after = /var/mail/sieve/global-after.sieve
sieve_before = /var/mail/sieve/global-before.sieve
sieve_default = /var/mail/sieve/global.sieve
sieve_extensions = +imapflags +notify +editheader
sieve_global = /var/mail/sieve
}
protocols = imap lmtp sieve
service auth-worker {
user = vmail
}
service auth {
unix_listener /var/spool/postfix/private/auth {
group = postfix
mode = 0600
user = postfix
}
unix_listener auth-userdb {
mode = 0600
user = vmail
}
user = dovecot
}
service imap-login {
inet_listener imap {
port = 0
}
inet_listener imaps {
port = 993
ssl = yes
}
process_limit = 9000
}
service imap {
process_limit = 9000
}
service lmtp {
unix_listener /var/spool/postfix/private/dovecot-lmtp {
group = postfix
mode = 0600
user = postfix
}
}
service pop3-login {
inet_listener pop3 {
port = 0
}
inet_listener pop3s {
port = 0
}
}
ssl = required
ssl_cert = </etc/letsencrypt/live/mail.COMPANY.com/fullchain.pem
ssl_key = # hidden, use -P to show it
userdb {
args = /etc/dovecot/dovecot-ldap.conf.ext
driver = ldap
}
protocol lmtp {
mail_plugins = " sieve"
}
protocol lda {
mail_plugins = " sieve"
}
protocol imap {
mail_max_userip_connections = 30
mail_plugins = " acl imap_acl"
}
protocol sieve {
mail_debug = yes
}
Stephan Bosch
2019-Oct-01 17:54 UTC
Lost mails with sieve "duplicate" extension in case of failure
On 01/10/2019 19:03, Hanno Stock via dovecot wrote:> Hi! > > we just experienced the following issue: > > We have a global sieve_before script using the duplicate extension. > > require ["duplicate", "imap4flags"]; > if duplicate { > discard; > } > > Now the storage of our mailbox store got full and lmtp deliveries > started to fail. > > This resulted in logs like this: > > Sep 25 16:14:56 mail-rc1 dovecot: lmtp(USERNAME): Error: > o_stream_send_istream(/var/mail/vhosts/mail.COMPANY.com/USERNAME/.dovecot/tmp/1569420896.M612482P5820.mail-rc1) > failed: No space left on device > Sep 25 16:14:56 mail-rc1 dovecot: lmtp(USERNAME): Error: > write(/var/mail/vhosts/mail.COMPANY.com/USERNAME/.dovecot/tmp/1569420896.M612482P5820.mail-rc1) > failed: No space left on device > Sep 25 16:14:56 mail-rc1 dovecot: lmtp(USERNAME): Error: > KT9HJGB2i128FgAATXMPig: sieve: > msgid=<redmine.journal-441799.20190925141453.94a97fd570706f2f at mail.COMPANY.com>: > failed to store into mailbox 'INBOX': Internal error occurred. Refer to > server log for more information. [2019-09-25 16:14:56] > Sep 25 16:14:56 mail-rc1 dovecot: lmtp(USERNAME): Error: > KT9HJGB2i128FgAATXMPig: sieve: Execution of script > /mnt/data/var/mail/vhosts/mail.COMPANY.com/USERNAME/.sieve/.dovecot.sieve was > aborted due to temporary failure (user logfile > /mnt/data/var/mail/vhosts/mail.COMPANY.com/USERNAME/.sieve/.dovecot.sieve.log > may reveal additional details) > Sep 25 16:14:56 mail-rc1 postfix/lmtp[5851]: 4D27690AC: > to=<USERNAME at mail.COMPANY.com>, orig_to=<USER.NAME at COMPANY.com>, > relay=mail-rc1.bvd.COMPANY.x[private/dovecot-lmtp], delay=1.3, > delays=1.3/0/0/0.01, dsn=4.2.0, status=deferred (host > mail-rc1.bvd.COMPANY.x[private/dovecot-lmtp] said: 451 4.2.0 > <USERNAME at mail.COMPANY.com> Internal error occurred. Refer to server log > for more information. [2019-09-25 16:14:56] (in reply to end of DATA > command)) > > So far, so good... A 451 response is what we wanted. > > However on the next delivery attempt the following happens: > > Sep 25 16:24:00 mail-rc1 postfix/lmtp[7040]: 4D27690AC: > to=<USERNAME at mail.COMPANY.com>, orig_to=<USER.NAME at COMPANY.com>, > relay=mail-rc1.bvd.COMPANY.x[private/dovecot-lmtp], delay=545, > delays=545/0.09/0.02/0.01, dsn=2.0.0, status=sent (250 2.0.0 > <USERNAME at mail.COMPANY.com> DBcCEYB4i11qGwAATXMPig Saved) > Sep 25 16:24:00 mail-rc1 dovecot: lmtp(USERNAME): > DBcCEYB4i11qGwAATXMPig: sieve: > msgid=<redmine.journal-441799.20190925141453.94a97fd570706f2f at mail.COMPANY.com>: > marked message to be discarded if not explicitly delivered (discard action) > Sep 25 16:24:00 mail-rc1 postfix/lmtp[7040]: 4D27690AC: > to=<USERNAME at mail.COMPANY.com>, orig_to=<USER.NAME at COMPANY.com>, > relay=mail-rc1.bvd.COMPANY.x[private/dovecot-lmtp], delay=545, > delays=545/0.09/0.02/0.01, dsn=2.0.0, status=sent (250 2.0.0 > <USERNAME at mail.COMPANY.com> DBcCEYB4i11qGwAATXMPig Saved) > > As the duplicate extension has already seen the mail, it is now > immediately discarded. > > RFC 7352 however explicitly states: > > Implementations MUST only update the internal duplicate-tracking list > when the Sieve script execution finishes successfully. If failing > script executions add the unique ID to the duplicate-tracking list, > all "duplicate" tests in the Sieve script would erroneously yield > "true" for the next delivery attempt of the same message. > > Dovecot seems not to honour this specification. > > Maybe this is because the "before_script" is not the script that fails, > but the subsequent scripts fail? > Am I missing some configuration option that would propagate the > subsequent failure back to the before_script? > Or is this a general limitation of the before_script mechanism? > > Any help appreciated how to alleviate this issue!This is most definitely a bug. Regards, Stephan.> > Best regards > > Hanno > > --- > > Filesystem: ext4 > > Dovecot config: > > # 2.2.27 (c0f36b0): /etc/dovecot/dovecot.conf > # Pigeonhole version 0.4.16 (fed8554) > # OS: Linux 4.9.0-11-amd64 x86_64 Debian 9.11 > auth_cache_negative_ttl = 0 > auth_cache_size = 1 M > auth_cache_ttl = 2 hours > auth_mechanisms = plain login > auth_username_format = %Ln > mail_gid = vmail > mail_home = /var/mail/vhosts/mail.COMPANY.com/%n > mail_location = maildir:~/.dovecot > mail_plugins = " acl" > mail_privileged_group = vmail > mail_uid = vmail > mailbox_list_index = yes > managesieve_notify_capability = mailto > managesieve_sieve_capability = fileinto reject envelope > encoded-character vacation subaddress comparator-i;ascii-numeric > relational regex imap4flags copy include variables body enotify > environment mailbox date index ihave duplicate mime foreverypart > extracttext editheader imapflags notify > namespace { > hidden = yes > list = no > location = maildir:~/.dovecot-top-namespace > prefix > separator = . > subscriptions = yes > type = private > } > namespace { > location > maildir:/var/mail/vhosts/mail.COMPANY.com/%%n/.dovecot:INDEXPVT=/var/mail/vhosts/mail.COMPANY.com/%n/.dovecot/shared/%%n > prefix = shared.%%u. > separator = . > subscriptions = no > type = shared > } > namespace inbox { > inbox = yes > location > mailbox Drafts { > special_use = \Drafts > } > mailbox Junk { > special_use = \Junk > } > mailbox Sent { > special_use = \Sent > } > mailbox "Sent Messages" { > special_use = \Sent > } > mailbox Trash { > special_use = \Trash > } > prefix = INBOX. > separator = . > } > passdb { > args = /etc/dovecot/dovecot-ldap.conf.ext > driver = ldap > } > plugin { > acl = vfile > acl_shared_dict = file:/var/lib/dovecot/db/shared-mailboxes.db > sieve > file:/var/mail/vhosts/mail.COMPANY.com/%n/.sieve/sieve;active=/var/mail/vhosts/mail.COMPANY.com/%n/.sieve/.dovecot.sieve > sieve_after = /var/mail/sieve/global-after.sieve > sieve_before = /var/mail/sieve/global-before.sieve > sieve_default = /var/mail/sieve/global.sieve > sieve_extensions = +imapflags +notify +editheader > sieve_global = /var/mail/sieve > } > protocols = imap lmtp sieve > service auth-worker { > user = vmail > } > service auth { > unix_listener /var/spool/postfix/private/auth { > group = postfix > mode = 0600 > user = postfix > } > unix_listener auth-userdb { > mode = 0600 > user = vmail > } > user = dovecot > } > service imap-login { > inet_listener imap { > port = 0 > } > inet_listener imaps { > port = 993 > ssl = yes > } > process_limit = 9000 > } > service imap { > process_limit = 9000 > } > service lmtp { > unix_listener /var/spool/postfix/private/dovecot-lmtp { > group = postfix > mode = 0600 > user = postfix > } > } > service pop3-login { > inet_listener pop3 { > port = 0 > } > inet_listener pop3s { > port = 0 > } > } > ssl = required > ssl_cert = </etc/letsencrypt/live/mail.COMPANY.com/fullchain.pem > ssl_key = # hidden, use -P to show it > userdb { > args = /etc/dovecot/dovecot-ldap.conf.ext > driver = ldap > } > protocol lmtp { > mail_plugins = " sieve" > } > protocol lda { > mail_plugins = " sieve" > } > protocol imap { > mail_max_userip_connections = 30 > mail_plugins = " acl imap_acl" > } > protocol sieve { > mail_debug = yes > }
Stephan Bosch
2019-Oct-01 19:19 UTC
Lost mails with sieve "duplicate" extension in case of failure
On 01/10/2019 19:54, Stephan Bosch wrote:> > > On 01/10/2019 19:03, Hanno Stock via dovecot wrote: >> RFC 7352 however explicitly states: >> >> ??? Implementations MUST only update the internal duplicate-tracking >> list >> ??? when the Sieve script execution finishes successfully.? If failing >> ??? script executions add the unique ID to the duplicate-tracking list, >> ??? all "duplicate" tests in the Sieve script would erroneously yield >> ??? "true" for the next delivery attempt of the same message. >> >> Dovecot seems not to honour this specification. >> >> Maybe this is because the "before_script" is not the script that fails, >> but the subsequent scripts fail? >> Am I missing some configuration option that would propagate the >> subsequent failure back to the before_script? >> Or is this a general limitation of the before_script mechanism? >> >> Any help appreciated how to alleviate this issue! > > This is most definitely a bug.Yes, it is a design flaw in our implementation. Tracking internally as DOP-1449. Regards, Stephan.> >> -- >> >> Filesystem: ext4 >> >> Dovecot config: >> >> # 2.2.27 (c0f36b0): /etc/dovecot/dovecot.conf >> # Pigeonhole version 0.4.16 (fed8554) >> # OS: Linux 4.9.0-11-amd64 x86_64 Debian 9.11 >> auth_cache_negative_ttl = 0 >> auth_cache_size = 1 M >> auth_cache_ttl = 2 hours >> auth_mechanisms = plain login >> auth_username_format = %Ln >> mail_gid = vmail >> mail_home = /var/mail/vhosts/mail.COMPANY.com/%n >> mail_location = maildir:~/.dovecot >> mail_plugins = " acl" >> mail_privileged_group = vmail >> mail_uid = vmail >> mailbox_list_index = yes >> managesieve_notify_capability = mailto >> managesieve_sieve_capability = fileinto reject envelope >> encoded-character vacation subaddress comparator-i;ascii-numeric >> relational regex imap4flags copy include variables body enotify >> environment mailbox date index ihave duplicate mime foreverypart >> extracttext editheader imapflags notify >> namespace { >> ?? hidden = yes >> ?? list = no >> ?? location = maildir:~/.dovecot-top-namespace >> ?? prefix >> ?? separator = . >> ?? subscriptions = yes >> ?? type = private >> } >> namespace { >> ?? location >> maildir:/var/mail/vhosts/mail.COMPANY.com/%%n/.dovecot:INDEXPVT=/var/mail/vhosts/mail.COMPANY.com/%n/.dovecot/shared/%%n >> >> ?? prefix = shared.%%u. >> ?? separator = . >> ?? subscriptions = no >> ?? type = shared >> } >> namespace inbox { >> ?? inbox = yes >> ?? location >> ?? mailbox Drafts { >> ???? special_use = \Drafts >> ?? } >> ?? mailbox Junk { >> ???? special_use = \Junk >> ?? } >> ?? mailbox Sent { >> ???? special_use = \Sent >> ?? } >> ?? mailbox "Sent Messages" { >> ???? special_use = \Sent >> ?? } >> ?? mailbox Trash { >> ???? special_use = \Trash >> ?? } >> ?? prefix = INBOX. >> ?? separator = . >> } >> passdb { >> ?? args = /etc/dovecot/dovecot-ldap.conf.ext >> ?? driver = ldap >> } >> plugin { >> ?? acl = vfile >> ?? acl_shared_dict = file:/var/lib/dovecot/db/shared-mailboxes.db >> ?? sieve >> file:/var/mail/vhosts/mail.COMPANY.com/%n/.sieve/sieve;active=/var/mail/vhosts/mail.COMPANY.com/%n/.sieve/.dovecot.sieve >> >> ?? sieve_after = /var/mail/sieve/global-after.sieve >> ?? sieve_before = /var/mail/sieve/global-before.sieve >> ?? sieve_default = /var/mail/sieve/global.sieve >> ?? sieve_extensions = +imapflags +notify +editheader >> ?? sieve_global = /var/mail/sieve >> } >> protocols = imap lmtp sieve >> service auth-worker { >> ?? user = vmail >> } >> service auth { >> ?? unix_listener /var/spool/postfix/private/auth { >> ???? group = postfix >> ???? mode = 0600 >> ???? user = postfix >> ?? } >> ?? unix_listener auth-userdb { >> ???? mode = 0600 >> ???? user = vmail >> ?? } >> ?? user = dovecot >> } >> service imap-login { >> ?? inet_listener imap { >> ???? port = 0 >> ?? } >> ?? inet_listener imaps { >> ???? port = 993 >> ???? ssl = yes >> ?? } >> ?? process_limit = 9000 >> } >> service imap { >> ?? process_limit = 9000 >> } >> service lmtp { >> ?? unix_listener /var/spool/postfix/private/dovecot-lmtp { >> ???? group = postfix >> ???? mode = 0600 >> ???? user = postfix >> ?? } >> } >> service pop3-login { >> ?? inet_listener pop3 { >> ???? port = 0 >> ?? } >> ?? inet_listener pop3s { >> ???? port = 0 >> ?? } >> } >> ssl = required >> ssl_cert = </etc/letsencrypt/live/mail.COMPANY.com/fullchain.pem >> ssl_key =? # hidden, use -P to show it >> userdb { >> ?? args = /etc/dovecot/dovecot-ldap.conf.ext >> ?? driver = ldap >> } >> protocol lmtp { >> ?? mail_plugins = " sieve" >> } >> protocol lda { >> ?? mail_plugins = " sieve" >> } >> protocol imap { >> ?? mail_max_userip_connections = 30 >> ?? mail_plugins = " acl imap_acl" >> } >> protocol sieve { >> ?? mail_debug = yes >> } >