dovecot - Jul 2012 - fts solr : out of memory

Hi Timo,
since I've upgraded to 2.1.8, solr indexer doesnt work correctly.
I see a lot of these errors in logs:

 Fatal: master: service(indexer-worker): child 9582 returned error 83
(Out of memory (service indexer-worker { vsz_limit=256 MB }, you may
need to increase it))

I've tried to set vsz_limit to a lot more, (around 4096 MB) but same
problem occurs.

thank for your help !
Regards
Tonio


Java(TM) SE Runtime Environment (build 1.6.0_31-b04)

doveconf -n
# 2.1.8: /etc/dovecot/dovecot.conf
# OS: Linux 3.2.13-xxxx-std-ipv6-64 x86_64 Debian wheezy/sid xfs
auth_cache_negative_ttl = 0
auth_cache_size = 10 M
auth_mechanisms = plain login
dict {
  quota = mysql:/etc/dovecot/dovecot-dict-quota-sql.conf
}
hostname = maildir2.spamguard.fr
log_timestamp = %Y-%m-%d %H:%M:%S
mail_gid = vmail
mail_location = sdbox:/home/virtual/%u
mail_plugins = quota fts fts_solr acl zlib
mail_privileged_group = vmail
mail_uid = vmail
managesieve_notify_capability = mailto
managesieve_sieve_capability = comparator-i;octet
comparator-i;ascii-casemap fileinto reject envelope encoded-character
vacation subaddress comparator-i;ascii-numeric relational regex
imap4flags copy include variables body enotify environment mailbox date
spamtest spamtestplus virustest
namespace {
  inbox = yes
  location   prefix   separator = /
  type = private
}
namespace {
  alias_for   hidden = yes
  inbox = no
  list = no
  location   prefix = INBOX/
  separator = /
  type = private
}
namespace {
  list = children
  location = sdbox:/home/virtual/%%u:INDEX=/home/virtual/%u/shared/%%u
  prefix = shared/%%u/
  separator = /
  subscriptions = no
  type = shared
}
passdb {
  args = /etc/dovecot/dovecot-sql.conf
  driver = sql
}
plugin {
  acl = vfile
  acl_shared_dict = file:/home/virtual/shared-%d-mailboxes.db
  autocreate = Spam
  autocreate2 = SpamFalse
  autocreate3 = SpamToLearn
  autocreate4 = Sent
  autosubscribe = Spam
  autosubscribe2 = SpamFalse
  autosubscribe3 = SpamToLearn
  autosubscribe4 = Sent
  fts = solr
  fts_solr = url=http://localhost:8081/solr/
  mail_log_events = delete undelete expunge copy mailbox_delete
mailbox_rename save mailbox_create
  mail_log_fields = uid box msgid size
  quota = dict:User quota::proxy::quota
  quota2 = dict:User quota::file:/home/virtual/%u/dovecot-quota
  quota_warning = storage=85%% quota-warning 85 %u
  sieve = ~/dovecot.sieve
  sieve_before = /home/virtual/sieve/
  sieve_dir = ~/sieve
  zlib_save = bz2
  zlib_save_level = 9
}
protocols = imap pop3 sieve lmtp
service auth {
  unix_listener /var/spool/postfix-in2/private/auth {
    group = mail
    mode = 0660
    user = postfix
  }
  unix_listener /var/spool/postfix-maildir2/private/auth {
    group = mail
    mode = 0660
    user = postfix
  }
  unix_listener /var/spool/postfix-msa/private/auth {
    group = mail
    mode = 0660
    user = postfix
  }
  unix_listener /var/spool/postfix-smtp06/private/auth {
    group = mail
    mode = 0660
    user = postfix
  }
  unix_listener /var/spool/postfix/private/auth {
    group = mail
    mode = 0660
    user = postfix
  }
  unix_listener auth-in2-master {
    mode = 0660
    user = vmail
  }
  unix_listener auth-maildir2-master {
    mode = 0660
    user = vmail
  }
  unix_listener auth-master {
    mode = 0660
    user = vmail
  }
  unix_listener auth-msa-master {
    mode = 0660
    user = vmail
  }
  unix_listener auth-out-master {
    mode = 0660
    user = vmail
  }
  unix_listener auth-smtp06-master {
    mode = 0660
    user = vmail
  }
  unix_listener auth-userdb {
    mode = 0660
    user = vmail
  }
}
service dict {
  unix_listener dict {
    mode = 0600
    user = vmail
  }
}
service imap-login {
  inet_listener imap {
    port = 143
  }
  inet_listener imaps {
    port = 993
  }
  process_limit = 256
  process_min_avail = 4
  service_count = 0
  vsz_limit = 256 M
}
service lmtp {
  inet_listener lmtp {
    address = 46.105.108.210
    port = 24
  }
  user = vmail
  vsz_limit = 368 M
}
service managesieve-login {
  inet_listener sieve {
    port = 4190
  }
  process_limit = 256
  vsz_limit = 64 M
}
service pop3-login {
  inet_listener pop3 {
    port = 110
  }
  inet_listener pop3s {
    port = 995
  }
  process_limit = 256
  process_min_avail = 4
  service_count = 0
  vsz_limit = 256 M
}
service quota-warning {
  executable = script /usr/bin/quota-warning.sh
  unix_listener quota-warning {
    mode = 0600
    user = vmail
  }
  user = vmail
}
ssl = required
ssl_ca = </etc/dovecot/ca-bundle.crt
ssl_cert = </etc/dovecot/mx2.eole-its.com.certchained.pem
ssl_key = </etc/postfix/tls/mx2.eole-its.com.key.pem
ssl_verify_client_cert = yes
userdb {
  args = /etc/dovecot/dovecot-sql.conf
  driver = sql
}
protocol imap {
  imap_client_workarounds = delay-newmail
  imap_max_line_length = 64 k
  mail_max_userip_connections = 60
  mail_plugins = quota imap_quota acl imap_acl autocreate mail_log
notify fts fts_solr zlib
}
protocol pop3 {
  mail_plugins = quota
  pop3_client_workarounds = outlook-no-nuls oe-ns-eoh
  pop3_uidl_format = %08Xu%08Xv
}
protocol sieve {
  managesieve_logout_format = bytes ( in=%i : out=%o )
}
protocol lda {
  info_log_path   log_path   mail_plugins = sieve quota autocreate zlib
  postmaster_address = postmaster at spamguard.fr
  quota_full_tempfail = yes
}
protocol lmtp {
  info_log_path   log_path   mail_plugins = sieve quota autocreate fts fts_solr
zlib
  postmaster_address = postmaster at spamguard.fr
  quota_full_tempfail = yes
}
local 46.105.108.210 {
  protocol imap {
    ssl = required
    ssl_cert = </etc/dovecot/imap.spamguard.fr.certchained2.pem
    ssl_key = </etc/dovecot/imap.spamguard.fr.key.pem
  }
}
local 87.98.168.xx {
  protocol imap {
    ssl = required
    ssl_cert = </etc/dovecot/mail.ovxx.fr.certchained.pem
    ssl_key = </etc/dovecot/mail.ovxx.fr.key.pem
  }
}

On 2012-07-28, at 11.21, tonio at starbridge.org wrote:
> Fatal: master: service(indexer-worker): child 9582 returned error 83
> (Out of memory (service indexer-worker { vsz_limit=256 MB }, you may
> need to increase it))
This looks very much like an issue I've been having with fts-lucene.
Indexing works (or at least completes without errors) in 2.1.7, but runs out of
memory in 2.1.8.

doveadm -v index -u username '*' crashes with:

doveadm(username): Fatal: pool_system_realloc(536870912): Out of memory
doveadm(username): Error: Raw backtrace:
/usr/lib/dovecot/libdovecot.so.0(+0x41d31) [0xb7699d31] ->
/usr/lib/dovecot/libdovecot.so.0(default_fatal_handler+0x47) [0xb7699e47] ->
/usr/lib/dovecot/libdovecot.so.0(i_fatal+0) [0xb766c958] ->
/usr/lib/dovecot/libdovecot.so.0(+0x5420e) [0xb76ac20e] ->
/usr/lib/dovecot/libdovecot.so.0(+0x3e358) [0xb7696358] ->
/usr/lib/dovecot/libdovecot.so.0(buffer_get_space_unsafe+0x72) [0xb76967f2]
-> /usr/lib/dovecot/libdovecot.so.0(buffer_append_space_unsafe+0x2b)
[0xb769691b] -> /usr/lib/dovecot/libdovecot.so.0(charset_to_utf8+0xe8)
[0xb768c3f8] ->
/usr/lib/dovecot/libdovecot.so.0(message_decoder_decode_next_block+0x3dd)
[0xb767fa4d] ->
/usr/lib/dovecot/modules/lib20_fts_plugin.so(fts_build_mail+0x11c) [0xb732881c]
-> /usr/lib/dovecot/modules/lib20_fts_plugin.so(+0xb256) [0xb732e256] ->
/usr/lib/dovecot/libdovecot-storage.so.0(mail_precache+0xf) [0xb774754f] ->
doveadm() [0x805736e] -> doveadm() [0x805422d] -> doveadm() [0x8054aad]
-> doveadm(doveadm_mail_try_run+0x12e) [0x8054fae] -> doveadm(main+0x3e4)
[0x805bf24] -> /lib/i686/cmov/libc.so.6(__libc_start_main+0xe6) [0xb7527ca6]
-> doveadm() [0x8053c11]

The problem occurs repeatably with specific messages. It looks as though the
crash happens in lib-charset/charset-iconv.c and the problem was introduced by
this change: http://hg.dovecot.org/dovecot-2.1/rev/0fde692cb565. If I remove
that one change from 2.1.8, I can index the affected messages again.

I've posted an example message that triggers the crash here:
https://www.atom.net/dovecot/testcase1.eml. That's a very contrived test
case, but I've encountered the same behaviour with a real message.

-- 

Matthew Powell                                          matthew at atom.net