dovecot - Oct 2019 - imapsieve administrator scripts are not executed in the order they are defined

Dear dovecot developers,

I have an issue with the Pigeonhole IMAPSieve Plugin and the order in which
administrator scripts are executed.  Although I cannot find anything about the
order in which the scripts are executed, I would expect they are executed in
the order they are defined: the one defined by imapsieve_mailbox1_* before the
one defined by imapsieve_mailbox2_* in case both match.

I defined several administrator scripts; among them one that should be executed
when copying a mail to mailbox "Inbox", and one that should be
executed when
copying a mail from a mailbox "*.Spam" to any other mailbox.  No
matter in
which order I define the two scripts, the one related to "Inbox" is
always
executed first when I move a mail from "blah.Spam" to
"Inbox", see the snipped
from my logs below.

I guess there might be a bug in file imap-sieve-storage.c, function
imap_sieve_mailbox_rules_match(...): The comment in this function states
"Insert sorted by rule index", but in case the index of the rule to be
inserted is higher than any index already in the "rules" array, the
rule is
inserted in position 0 instead of being appended to the end, because
"insert_idx" is initialized with 0.

Since the rule for "*.Spam" is a pattern rule, it is the first rule to
be
inserted into "rules" (in
imap_sieve_mailbox_rules_match_patterns(...)).
Afterwards the rule for "Inbox" is always inserted at position 0 (in
imap_sieve_mailbox_rules_match(...)), no matter if its rule index is higher or
lower than the index of the other rule.

The following change could fix this:
 - unsigned int insert_idx = 0;
 + unsigned int insert_idx = array_count(rules);

Best regards,
  Frank


----- log -----
[...]
Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
Debug: imapsieve: mailbox INBOX: MOVE event
Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
Debug: sieve: Pigeonhole version 0.5.7.2 (7372921a) initializing
Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
Debug: sieve: Sieve Extprograms plugin for Pigeonhole version 0.5.7.2
(7372921a) loaded
Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
Debug: imapsieve: Static mailbox rule [1]: mailbox=`*.Spam' from=`*'
causes=(COPY) => before=`file:/usr/lib/dovecot/sieve/spamassassin-learn-
spam.sieve' after=(none)
Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
Debug: imapsieve: Static mailbox rule [2]: mailbox=`*' from=`*.Spam'
causes=(COPY) => before=`file:/usr/lib/dovecot/sieve/spamassassin-learn-
ham.sieve' after=(none)
Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
Debug: imapsieve: Static mailbox rule [3]: mailbox=`INBOX' from=`*'
causes=(COPY APPEND) => before=`file:~/.dovecot.sieve' after=(none)
Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
Debug: imapsieve: Matched static mailbox rule [2]
Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
Debug: imapsieve: Matched static mailbox rule [3]
Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
Debug: sieve: file storage: Using Sieve script path: /var/spool/mail/
user/.dovecot.sieve
Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
Debug: sieve: file script: Opened script `.dovecot' from `/var/spool/mail/
user/.dovecot.sieve'
Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
Debug: sieve: file storage: Using Sieve script path: /usr/lib/dovecot/sieve/
spamassassin-learn-ham.sieve
Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
Debug: sieve: file script: Opened script `spamassassin-learn-ham' from
`/usr/
lib/dovecot/sieve/spamassassin-learn-ham.sieve'
Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
Debug: sieve: Opening script 1 of 2 from
`/var/spool/mail/user/.dovecot.sieve'
Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
Debug: sieve: Loading script /var/spool/mail/user/.dovecot.sieve
[...]

----- dovecot -n -----
# 2.3.7.2 (3c910f64b): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.5.7.2 (7372921a)
# OS: Linux 5.1.18-gentoo x86_64 Gentoo Base System release 2.6 ext4
# Hostname: somehost.de
auth_username_format = %Ln
mail_debug = yes
mail_home = /var/spool/mail/%u
mail_location = maildir:/var/spool/mail/%u/mail
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character
vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy
include variables body enotify environment mailbox date index ihave duplicate
mime foreverypart extracttext vnd.dovec
ot.debug
namespace inbox {
 inbox = yes
 location  mailbox Drafts {
   special_use = \Drafts
 }
 mailbox Junk {
   special_use = \Junk
 }
 mailbox Sent {
   special_use = \Sent
 }
 mailbox "Sent Messages" {
   special_use = \Sent
 }
 mailbox Trash {
   special_use = \Trash
 }
 prefix }
passdb {
 args = scheme=CRYPT username_format=%u /etc/dovecot/users
 driver = passwd-file
}
plugin {
 imapsieve_mailbox1_before = file:/usr/lib/dovecot/sieve/spamassassin-learn-
spam.sieve
 imapsieve_mailbox1_causes = COPY
 imapsieve_mailbox1_name = *.Spam
 imapsieve_mailbox2_before = file:/usr/lib/dovecot/sieve/spamassassin-learn-
ham.sieve
 imapsieve_mailbox2_causes = COPY
 imapsieve_mailbox2_from = *.Spam
 imapsieve_mailbox2_name = *
 imapsieve_mailbox3_before = file:~/.dovecot.sieve
 imapsieve_mailbox3_causes = COPY APPEND
 imapsieve_mailbox3_name = Inbox
 mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename
append
 sieve = file:~/sieve;active=~/.dovecot.sieve
 sieve_execute_bin_dir = /usr/lib/dovecot/sieve
 sieve_extensions = +vnd.dovecot.debug
 sieve_filter_bin_dir = /usr/lib/dovecot/sieve
 sieve_filter_exec_timeout = 300s
 sieve_filter_input_eol = lf
 sieve_global = /usr/lib/dovecot/sieve
 sieve_global_extensions = +vnd.dovecot.pipe +vnd.dovecot.execute
+vnd.dovecot.filter
 sieve_pipe_bin_dir = /usr/lib/dovecot/sieve
 sieve_pipe_exec_timeout = 300s
 sieve_pipe_input_eol = lf
 sieve_plugins = sieve_extprograms
}
postmaster_address = postmaster at frankagainstthemachine.de
protocols = imap lmtp sieve sieve
service imap-login {
 inet_listener imap {
   port = 0
 }
}
service lmtp {
 unix_listener /var/spool/postfix/private/dovecot-lmtp {
   group = postfix
   mode = 0666
   user = postfix
 }
}
ssl = required
ssl_cert = </etc/ssl/dovecot/somehost.cert.pem
ssl_cipher_list = ALL:!DH:!kRSA:!SRP:!kDHd:!DSS:!aNULL:!eNULL:!EXPORT:!DES:!
3DES:!MD5:!PSK:!RC4:!ADH:!LOW at STRENGTH
ssl_dh = # hidden, use -P to show it
ssl_key = # hidden, use -P to show it
ssl_prefer_server_ciphers = yes
userdb {
 args = username_format=%u /etc/dovecot/users
 driver = passwd-file
}
verbose_ssl = yes
protocol lmtp {
 mail_plugins = " sieve"
}
protocol lda {
 mail_plugins = sieve
}
protocol imap {
 mail_plugins = " imap_sieve"
}
protocol sieve {
 mail_debug = yes
}

On 05/10/2019 00:43, frankagainstthemachine--- via dovecot
wrote:
> Dear dovecot developers,
>
> I have an issue with the Pigeonhole IMAPSieve Plugin and the order in which
> administrator scripts are executed.  Although I cannot find anything about
the
> order in which the scripts are executed, I would expect they are executed
in
> the order they are defined: the one defined by imapsieve_mailbox1_* before
the
> one defined by imapsieve_mailbox2_* in case both match.
>
> I defined several administrator scripts; among them one that should be
executed
> when copying a mail to mailbox "Inbox", and one that should be
executed when
> copying a mail from a mailbox "*.Spam" to any other mailbox.  No
matter in
> which order I define the two scripts, the one related to "Inbox"
is always
> executed first when I move a mail from "blah.Spam" to
"Inbox", see the snipped
> from my logs below.
>
> I guess there might be a bug in file imap-sieve-storage.c, function
> imap_sieve_mailbox_rules_match(...): The comment in this function states
> "Insert sorted by rule index", but in case the index of the rule
to be
> inserted is higher than any index already in the "rules" array,
the rule is
> inserted in position 0 instead of being appended to the end, because
> "insert_idx" is initialized with 0.
>
> Since the rule for "*.Spam" is a pattern rule, it is the first
rule to be
> inserted into "rules" (in
imap_sieve_mailbox_rules_match_patterns(...)).
> Afterwards the rule for "Inbox" is always inserted at position 0
(in
> imap_sieve_mailbox_rules_match(...)), no matter if its rule index is higher
or
> lower than the index of the other rule.
>
> The following change could fix this:
>   - unsigned int insert_idx = 0;
>   + unsigned int insert_idx = array_count(rules);
Confirmed. Tracking internally as DOP-1465.

Regards,

Stephan.
> Best regards,
>    Frank
>
>
> ----- log -----
> [...]
> Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
> Debug: imapsieve: mailbox INBOX: MOVE event
> Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
> Debug: sieve: Pigeonhole version 0.5.7.2 (7372921a) initializing
> Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
> Debug: sieve: Sieve Extprograms plugin for Pigeonhole version 0.5.7.2
> (7372921a) loaded
> Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
> Debug: imapsieve: Static mailbox rule [1]: mailbox=`*.Spam'
from=`*'
> causes=(COPY) => before=`file:/usr/lib/dovecot/sieve/spamassassin-learn-
> spam.sieve' after=(none)
> Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
> Debug: imapsieve: Static mailbox rule [2]: mailbox=`*'
from=`*.Spam'
> causes=(COPY) => before=`file:/usr/lib/dovecot/sieve/spamassassin-learn-
> ham.sieve' after=(none)
> Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
> Debug: imapsieve: Static mailbox rule [3]: mailbox=`INBOX' from=`*'
> causes=(COPY APPEND) => before=`file:~/.dovecot.sieve' after=(none)
> Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
> Debug: imapsieve: Matched static mailbox rule [2]
> Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
> Debug: imapsieve: Matched static mailbox rule [3]
> Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
> Debug: sieve: file storage: Using Sieve script path: /var/spool/mail/
> user/.dovecot.sieve
> Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
> Debug: sieve: file script: Opened script `.dovecot' from
`/var/spool/mail/
> user/.dovecot.sieve'
> Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
> Debug: sieve: file storage: Using Sieve script path:
/usr/lib/dovecot/sieve/
> spamassassin-learn-ham.sieve
> Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
> Debug: sieve: file script: Opened script `spamassassin-learn-ham' from
`/usr/
> lib/dovecot/sieve/spamassassin-learn-ham.sieve'
> Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
> Debug: sieve: Opening script 1 of 2 from
`/var/spool/mail/user/.dovecot.sieve'
> Oct  4 23:51:12 somehost dovecot[22918]:
imap(user)<22927><oLq3sRyU0t/AqLwV>:
> Debug: sieve: Loading script /var/spool/mail/user/.dovecot.sieve
> [...]
>
> ----- dovecot -n -----
> # 2.3.7.2 (3c910f64b): /etc/dovecot/dovecot.conf
> # Pigeonhole version 0.5.7.2 (7372921a)
> # OS: Linux 5.1.18-gentoo x86_64 Gentoo Base System release 2.6 ext4
> # Hostname: somehost.de
> auth_username_format = %Ln
> mail_debug = yes
> mail_home = /var/spool/mail/%u
> mail_location = maildir:/var/spool/mail/%u/mail
> managesieve_notify_capability = mailto
> managesieve_sieve_capability = fileinto reject envelope encoded-character
> vacation subaddress comparator-i;ascii-numeric relational regex imap4flags
copy
> include variables body enotify environment mailbox date index ihave
duplicate
> mime foreverypart extracttext vnd.dovec
> ot.debug
> namespace inbox {
>   inbox = yes
>   location >   mailbox Drafts {
>     special_use = \Drafts
>   }
>   mailbox Junk {
>     special_use = \Junk
>   }
>   mailbox Sent {
>     special_use = \Sent
>   }
>   mailbox "Sent Messages" {
>     special_use = \Sent
>   }
>   mailbox Trash {
>     special_use = \Trash
>   }
>   prefix > }
> passdb {
>   args = scheme=CRYPT username_format=%u /etc/dovecot/users
>   driver = passwd-file
> }
> plugin {
>   imapsieve_mailbox1_before =
file:/usr/lib/dovecot/sieve/spamassassin-learn-
> spam.sieve
>   imapsieve_mailbox1_causes = COPY
>   imapsieve_mailbox1_name = *.Spam
>   imapsieve_mailbox2_before =
file:/usr/lib/dovecot/sieve/spamassassin-learn-
> ham.sieve
>   imapsieve_mailbox2_causes = COPY
>   imapsieve_mailbox2_from = *.Spam
>   imapsieve_mailbox2_name = *
>   imapsieve_mailbox3_before = file:~/.dovecot.sieve
>   imapsieve_mailbox3_causes = COPY APPEND
>   imapsieve_mailbox3_name = Inbox
>   mail_log_events = delete undelete expunge copy mailbox_delete
mailbox_rename
> append
>   sieve = file:~/sieve;active=~/.dovecot.sieve
>   sieve_execute_bin_dir = /usr/lib/dovecot/sieve
>   sieve_extensions = +vnd.dovecot.debug
>   sieve_filter_bin_dir = /usr/lib/dovecot/sieve
>   sieve_filter_exec_timeout = 300s
>   sieve_filter_input_eol = lf
>   sieve_global = /usr/lib/dovecot/sieve
>   sieve_global_extensions = +vnd.dovecot.pipe +vnd.dovecot.execute
> +vnd.dovecot.filter
>   sieve_pipe_bin_dir = /usr/lib/dovecot/sieve
>   sieve_pipe_exec_timeout = 300s
>   sieve_pipe_input_eol = lf
>   sieve_plugins = sieve_extprograms
> }
> postmaster_address = postmaster at frankagainstthemachine.de
> protocols = imap lmtp sieve sieve
> service imap-login {
>   inet_listener imap {
>     port = 0
>   }
> }
> service lmtp {
>   unix_listener /var/spool/postfix/private/dovecot-lmtp {
>     group = postfix
>     mode = 0666
>     user = postfix
>   }
> }
> ssl = required
> ssl_cert = </etc/ssl/dovecot/somehost.cert.pem
> ssl_cipher_list =
ALL:!DH:!kRSA:!SRP:!kDHd:!DSS:!aNULL:!eNULL:!EXPORT:!DES:!
> 3DES:!MD5:!PSK:!RC4:!ADH:!LOW at STRENGTH
> ssl_dh = # hidden, use -P to show it
> ssl_key = # hidden, use -P to show it
> ssl_prefer_server_ciphers = yes
> userdb {
>   args = username_format=%u /etc/dovecot/users
>   driver = passwd-file
> }
> verbose_ssl = yes
> protocol lmtp {
>   mail_plugins = " sieve"
> }
> protocol lda {
>   mail_plugins = sieve
> }
> protocol imap {
>   mail_plugins = " imap_sieve"
> }
> protocol sieve {
>   mail_debug = yes
> }
>
>