dovecot - Oct 2015 - fts solr ignores new mailboxes until fts rescan

Hello,
on my debian wheezy installation, I implemented fts plugin with solr as
indexing system.
I noticed that if a user creates a new folder, that folder it is ignored by
dovecot indexing until a doveadm fts rescan is performed for that user.
 From that time on everything works ok on that folder.

As a test I set fts_autoindex=yes and put in that folder a brand new mail
never indexed before. in my logs indexer-worker says it indexed 0 messages

Obviously this is a major issue as users often create new folders, but a
search always returns empty? results.

My system is a debian wheezy with dovecot packages from backports? dovecot
2.2.13-11~bpo70+1.

Am I missing something? Any clue would be appreciated.

Thank you all in advance.

Bests
Giorgio Paolucci
============================
doveconf -n
# 2.2.13: /etc/dovecot/dovecot.conf
# OS: Linux 3.2.0-4-amd64 x86_64 Debian 7.8
auth_default_realm = xxx
disable_plaintext_auth = no
log_path = /var/log/dovecot/dovecot.log
mail_fsync = always
mail_nfs_index = yes
mail_nfs_storage = yes
mail_plugins = quota? listescape mail_log notify fts fts_solr
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character
vacation subaddress comparator-i;ascii-numeric relational regex imap4flags
copy include variables body enotify environment mailbox date ihave
mdbox_rotate_interval = 1 days
mdbox_rotate_size = 10 M
mmap_disable = yes
namespace inbox {
? inbox = yes
? location ? mailbox Drafts {
??? special_use = \Drafts
? }
? mailbox Junk {
??? special_use = \Junk
? }
? mailbox Sent {
??? special_use = \Sent
? }
? mailbox "Sent Messages" {
??? special_use = \Sent
? }
? mailbox Trash {
??? special_use = \Trash
? }
? prefix ? separator = /
}
passdb {
? args = /etc/dovecot/conf.d/dovecot-sql.conf.ext
? driver = sql
}
passdb {
? args = /etc/dovecot/conf.d/dovecot-sqlSSO.conf.ext
? driver = sql
}
plugin {
? fts = solr
? fts_solr = url=http://%n.%1n.solr.csia:8983/solr/
? mail_log_events = delete undelete expunge copy mailbox_delete
mailbox_rename
? mail_log_fields = uid box msgid size
? pop3_migration_mailbox = INBOX
? quota = dict:User quota::file:/SYSTEM/USERS/homes/%d/%1u/%u/%u.quota
? quota_rule = *:storage=7000M
? quota_rule2 = Trash:storage=+100M
? quota_warning = storage=95%% quota-warning 95 %u
? quota_warning2 = storage=80%% quota-warning 80 %u
? sieve = /SYSTEM/USERS/mailbox/%d/%1n/%u/.dovecotactive.sieve
? sieve_default = /etc/dovecot/default.sieve
? sieve_dir = /SYSTEM/USERS/mailbox/%d/%1n/%u/sieve
}
pop3c_host = 147.162.10.68
protocols = imap pop3 lmtp sieve
service auth {
? unix_listener /var/spool/postfix/private/auth {
??? group = postfix
??? mode = 0660
??? user = postfix
? }
? unix_listener auth-userdb {
??? group = vmail
??? mode = 0600
??? user = vmail
? }
}
service imap-login {
? inet_listener imaps {
??? ssl = no
? }
? process_min_avail = 20
? service_count = 1
}
service imap {
? executable = imap postlogin
? process_limit = 4096
}
service lmtp {
? executable = lmtp
? inet_listener lmtp {
??? port = 2223
? }
? process_min_avail = 15
}
service pop3-login {
? inet_listener pop3s {
??? ssl = no
? }
}
service pop3 {
? process_limit = 256
}
service postlogin {
? executable = script-login -d rawlog
? user = vmail
}
service quota-warning {
? executable = script /SYSTEM/DOVECOT/scripts/quota-warning.sh
? unix_listener quota-warning {
??? user = vmail
? }
? user = vmail
}
ssl = no
userdb {
? driver = prefetch
}
userdb {
? args = /etc/dovecot/conf.d/dovecot-sql.conf.ext
? driver = sql
}
userdb {
? driver = prefetch
}
userdb {
? args = /etc/dovecot/conf.d/dovecot-sqlSSO.conf.ext
? driver = sql
}
protocol lmtp {
? info_log_path = /tmp/dovecot-lmtp-info.log
? log_path = /tmp/dovecot-lmtp.log
? mail_plugins = quota? listescape mail_log notify sieve
}
protocol lda {
? mail_plugins = quota? listescape mail_log notify
}
protocol imap {
? imap_max_line_length = 128 k
? mail_max_userip_connections = 20
? mail_plugins = quota? listescape mail_log notify fts fts_solr
imap_quota
}
protocol doveadm {
? mail_plugins = quota? listescape mail_log notify fts fts_solr
pop3_migration
}
protocol pop3 {
? pop3_no_flag_updates = yes
? pop3_uidl_format = %10v.%09u

Hi,

On 2015-10-12 18:39, Giorgio Paolucci wrote:
> Hello,
> on my debian wheezy installation, I implemented fts plugin with solr as
> indexing system.
> I noticed that if a user creates a new folder, that folder it is 
> ignored by
> dovecot indexing until a doveadm fts rescan is performed for that user.
> From that time on everything works ok on that folder.
> 
> As a test I set fts_autoindex=yes and put in that folder a brand new 
> mail
> never indexed before. in my logs indexer-worker says it indexed 0 
> messages
> 
> Obviously this is a major issue as users often create new folders, but 
> a
> search always returns empty? results.
> 
> My system is a debian wheezy with dovecot packages from backports? 
> dovecot
> 2.2.13-11~bpo70+1.
> 
> Am I missing something? Any clue would be appreciated.
Am using 2.2.19 with fts_autoindex = yes and Maildir storage.

I create a folder and moved a message from inbox to the new folder and 
the log says it indexed 1 message in test, I don't have messages that 
where never indexed but i doubt this makes any difference.

I'm using Roundcube and tried the search with both Subject and Body 
searches.

Are the mails indexed when you perform a search?
How do you search?
> Thank you all in advance.
> 
> Bests
> Giorgio Paolucci
> ============================> 
> doveconf -n
> # 2.2.13: /etc/dovecot/dovecot.conf
> # OS: Linux 3.2.0-4-amd64 x86_64 Debian 7.8
> auth_default_realm = xxx
> disable_plaintext_auth = no
> log_path = /var/log/dovecot/dovecot.log
> mail_fsync = always
> mail_nfs_index = yes
> mail_nfs_storage = yes
> mail_plugins = quota? listescape mail_log notify fts fts_solr
> managesieve_notify_capability = mailto
> managesieve_sieve_capability = fileinto reject envelope 
> encoded-character
> vacation subaddress comparator-i;ascii-numeric relational regex 
> imap4flags
> copy include variables body enotify environment mailbox date ihave
> mdbox_rotate_interval = 1 days
> mdbox_rotate_size = 10 M
> mmap_disable = yes
> namespace inbox {
> ? inbox = yes
> ? location > ? mailbox Drafts {
> ??? special_use = \Drafts
> ? }
> ? mailbox Junk {
> ??? special_use = \Junk
> ? }
> ? mailbox Sent {
> ??? special_use = \Sent
> ? }
> ? mailbox "Sent Messages" {
> ??? special_use = \Sent
> ? }
> ? mailbox Trash {
> ??? special_use = \Trash
> ? }
> ? prefix > ? separator = /
> }
> passdb {
> ? args = /etc/dovecot/conf.d/dovecot-sql.conf.ext
> ? driver = sql
> }
> passdb {
> ? args = /etc/dovecot/conf.d/dovecot-sqlSSO.conf.ext
> ? driver = sql
> }
> plugin {
> ? fts = solr
> ? fts_solr = url=http://%n.%1n.solr.csia:8983/solr/
i have and fts_autoindex = yes and break-imap-search set, but afaik the 
lattter should only take effect on searches

regards
christian
> ? mail_log_events = delete undelete expunge copy mailbox_delete
> mailbox_rename
> ? mail_log_fields = uid box msgid size
> ? pop3_migration_mailbox = INBOX
> ? quota = dict:User quota::file:/SYSTEM/USERS/homes/%d/%1u/%u/%u.quota
> ? quota_rule = *:storage=7000M
> ? quota_rule2 = Trash:storage=+100M
> ? quota_warning = storage=95%% quota-warning 95 %u
> ? quota_warning2 = storage=80%% quota-warning 80 %u
> ? sieve = /SYSTEM/USERS/mailbox/%d/%1n/%u/.dovecotactive.sieve
> ? sieve_default = /etc/dovecot/default.sieve
> ? sieve_dir = /SYSTEM/USERS/mailbox/%d/%1n/%u/sieve
> }
> pop3c_host = 147.162.10.68
> protocols = imap pop3 lmtp sieve
> service auth {
> ? unix_listener /var/spool/postfix/private/auth {
> ??? group = postfix
> ??? mode = 0660
> ??? user = postfix
> ? }
> ? unix_listener auth-userdb {
> ??? group = vmail
> ??? mode = 0600
> ??? user = vmail
> ? }
> }
> service imap-login {
> ? inet_listener imaps {
> ??? ssl = no
> ? }
> ? process_min_avail = 20
> ? service_count = 1
> }
> service imap {
> ? executable = imap postlogin
> ? process_limit = 4096
> }
> service lmtp {
> ? executable = lmtp
> ? inet_listener lmtp {
> ??? port = 2223
> ? }
> ? process_min_avail = 15
> }
> service pop3-login {
> ? inet_listener pop3s {
> ??? ssl = no
> ? }
> }
> service pop3 {
> ? process_limit = 256
> }
> service postlogin {
> ? executable = script-login -d rawlog
> ? user = vmail
> }
> service quota-warning {
> ? executable = script /SYSTEM/DOVECOT/scripts/quota-warning.sh
> ? unix_listener quota-warning {
> ??? user = vmail
> ? }
> ? user = vmail
> }
> ssl = no
> userdb {
> ? driver = prefetch
> }
> userdb {
> ? args = /etc/dovecot/conf.d/dovecot-sql.conf.ext
> ? driver = sql
> }
> userdb {
> ? driver = prefetch
> }
> userdb {
> ? args = /etc/dovecot/conf.d/dovecot-sqlSSO.conf.ext
> ? driver = sql
> }
> protocol lmtp {
> ? info_log_path = /tmp/dovecot-lmtp-info.log
> ? log_path = /tmp/dovecot-lmtp.log
> ? mail_plugins = quota? listescape mail_log notify sieve
> }
> protocol lda {
> ? mail_plugins = quota? listescape mail_log notify
> }
> protocol imap {
> ? imap_max_line_length = 128 k
> ? mail_max_userip_connections = 20
> ? mail_plugins = quota? listescape mail_log notify fts fts_solr
> imap_quota
> }
> protocol doveadm {
> ? mail_plugins = quota? listescape mail_log notify fts fts_solr
> pop3_migration
> }
> protocol pop3 {
> ? pop3_no_flag_updates = yes
> ? pop3_uidl_format = %10v.%09u

Hi Christian,
may be the issue is related to this other post:
http://dovecot.org/pipermail/dovecot/2015-September/102094.html

Anyway, what I can tell you is that for every? user account,? for every
folders created after the last doveadm fts rescan for that user, dovecot
does not invoke indexer.
A new rescan fix the problem for all the folders created up to that moment.

I was wondering if I was missing something or if there was a work around to
force rescan only for new folders at the moment of their creation...

Bests.
Giorgio

Quoting Christian Kivalo <ml+dovecot at valo.at>:
> Hi,
>
> On 2015-10-12 18:39, Giorgio Paolucci wrote:
>> Hello,
>> on my debian wheezy installation, I implemented fts plugin with solr as
>> indexing system.
>> I noticed that if a user creates a new folder, that folder it is
>> ignored by
>> dovecot indexing until a doveadm fts rescan is performed for that user.
>> From that time on everything works ok on that folder.
>>
>> As a test I set fts_autoindex=yes and put in that folder a brand new
mail
>> never indexed before. in my logs indexer-worker says it indexed 0
>> messages
>>
>> Obviously this is a major issue as users often create new folders, but
a
>> search always returns empty? results.
>>
>> My system is a debian wheezy with dovecot packages from backports?
>> dovecot
>> 2.2.13-11~bpo70+1.
>>
>> Am I missing something? Any clue would be appreciated.
>
> Am using 2.2.19 with fts_autoindex = yes and Maildir storage.
>
> I create a folder and moved a message from inbox to the new folder and
> the log says it indexed 1 message in test, I don't have messages that
> where never indexed but i doubt this makes any difference.
>
> I'm using Roundcube and tried the search with both Subject and Body
> searches.
>
> Are the mails indexed when you perform a search?
> How do you search?
>
>> Thank you all in advance.
>>
>> Bests
>> Giorgio Paolucci
>> ============================>>
>> doveconf -n
>> # 2.2.13: /etc/dovecot/dovecot.conf
>> # OS: Linux 3.2.0-4-amd64 x86_64 Debian 7.8
>> auth_default_realm = xxx
>> disable_plaintext_auth = no
>> log_path = /var/log/dovecot/dovecot.log
>> mail_fsync = always
>> mail_nfs_index = yes
>> mail_nfs_storage = yes
>> mail_plugins = quota? listescape mail_log notify fts fts_solr
>> managesieve_notify_capability = mailto
>> managesieve_sieve_capability = fileinto reject envelope
encoded-character
>> vacation subaddress comparator-i;ascii-numeric relational regex
>> imap4flags
>> copy include variables body enotify environment mailbox date ihave
>> mdbox_rotate_interval = 1 days
>> mdbox_rotate_size = 10 M
>> mmap_disable = yes
>> namespace inbox {
>> ? inbox = yes
>> ? location >> ? mailbox Drafts {
>> ??? special_use = \Drafts
>> ? }
>> ? mailbox Junk {
>> ??? special_use = \Junk
>> ? }
>> ? mailbox Sent {
>> ??? special_use = \Sent
>> ? }
>> ? mailbox "Sent Messages" {
>> ??? special_use = \Sent
>> ? }
>> ? mailbox Trash {
>> ??? special_use = \Trash
>> ? }
>> ? prefix >> ? separator = /
>> }
>> passdb {
>> ? args = /etc/dovecot/conf.d/dovecot-sql.conf.ext
>> ? driver = sql
>> }
>> passdb {
>> ? args = /etc/dovecot/conf.d/dovecot-sqlSSO.conf.ext
>> ? driver = sql
>> }
>> plugin {
>> ? fts = solr
>> ? fts_solr = url=http://%n.%1n.solr.csia:8983/solr/
>
> i have and fts_autoindex = yes and break-imap-search set, but afaik the
> lattter should only take effect on searches
>
> regards
> christian
>
>> ? mail_log_events = delete undelete expunge copy mailbox_delete
>> mailbox_rename
>> ? mail_log_fields = uid box msgid size
>> ? pop3_migration_mailbox = INBOX
>> ? quota = dict:User quota::file:/SYSTEM/USERS/homes/%d/%1u/%u/%u.quota
>> ? quota_rule = *:storage=7000M
>> ? quota_rule2 = Trash:storage=+100M
>> ? quota_warning = storage=95%% quota-warning 95 %u
>> ? quota_warning2 = storage=80%% quota-warning 80 %u
>> ? sieve = /SYSTEM/USERS/mailbox/%d/%1n/%u/.dovecotactive.sieve
>> ? sieve_default = /etc/dovecot/default.sieve
>> ? sieve_dir = /SYSTEM/USERS/mailbox/%d/%1n/%u/sieve
>> }
>> pop3c_host = 147.162.10.68
>> protocols = imap pop3 lmtp sieve
>> service auth {
>> ? unix_listener /var/spool/postfix/private/auth {
>> ??? group = postfix
>> ??? mode = 0660
>> ??? user = postfix
>> ? }
>> ? unix_listener auth-userdb {
>> ??? group = vmail
>> ??? mode = 0600
>> ??? user = vmail
>> ? }
>> }
>> service imap-login {
>> ? inet_listener imaps {
>> ??? ssl = no
>> ? }
>> ? process_min_avail = 20
>> ? service_count = 1
>> }
>> service imap {
>> ? executable = imap postlogin
>> ? process_limit = 4096
>> }
>> service lmtp {
>> ? executable = lmtp
>> ? inet_listener lmtp {
>> ??? port = 2223
>> ? }
>> ? process_min_avail = 15
>> }
>> service pop3-login {
>> ? inet_listener pop3s {
>> ??? ssl = no
>> ? }
>> }
>> service pop3 {
>> ? process_limit = 256
>> }
>> service postlogin {
>> ? executable = script-login -d rawlog
>> ? user = vmail
>> }
>> service quota-warning {
>> ? executable = script /SYSTEM/DOVECOT/scripts/quota-warning.sh
>> ? unix_listener quota-warning {
>> ??? user = vmail
>> ? }
>> ? user = vmail
>> }
>> ssl = no
>> userdb {
>> ? driver = prefetch
>> }
>> userdb {
>> ? args = /etc/dovecot/conf.d/dovecot-sql.conf.ext
>> ? driver = sql
>> }
>> userdb {
>> ? driver = prefetch
>> }
>> userdb {
>> ? args = /etc/dovecot/conf.d/dovecot-sqlSSO.conf.ext
>> ? driver = sql
>> }
>> protocol lmtp {
>> ? info_log_path = /tmp/dovecot-lmtp-info.log
>> ? log_path = /tmp/dovecot-lmtp.log
>> ? mail_plugins = quota? listescape mail_log notify sieve
>> }
>> protocol lda {
>> ? mail_plugins = quota? listescape mail_log notify
>> }
>> protocol imap {
>> ? imap_max_line_length = 128 k
>> ? mail_max_userip_connections = 20
>> ? mail_plugins = quota? listescape mail_log notify fts fts_solr
>> imap_quota
>> }
>> protocol doveadm {
>> ? mail_plugins = quota? listescape mail_log notify fts fts_solr
>> pop3_migration
>> }
>> protocol pop3 {
>> ? pop3_no_flag_updates = yes? pop3_uidl_format = %10v.%09u
>
> ?
  --

? ? ?Ing. Giorgio Paolucci
? ? ?Universita' di Padova
? ? ?Centro Servizi Informatici di Ateneo
? ? ?Responsabile Infrastrutture,
? ? ?Networking e Sicurezza
? ? ?Tel.+39-049-8273711

--