dovecot - May 2012 - Dovecot usage values differ from actual disk usage

Hi,

in my dovecot setup the accounting database table shows wrong
values which do not correspond with the actual disk space used.

The disk usage says 2.6 Gigabytes:

mail01:~# du -sh /mail/dovecot/example.org/username
2.6G /mail/dovecot/example.org/username

While the doveadm quota get says 7 Gigabytes for the same user:

mail01:~# doveadm -f tab quota get -u username at example.org
Quota name Type Value Limit %
User quota STORAGE 7065208 - 0
User quota MESSAGE 282024 - 0

mysql> select * from dovecot.dovecot_usage where username="username at
example.org";
+----------------------+------------+----------+
| username             | storage    | messages |
+----------------------+------------+----------+
| username at example.org | 7234773131 | 282024   |
+----------------------+------------+----------+
1 row in set (0.00 sec)

What might be the reason for such huge differences (several gigabytes)
between storage value in dovecot_usage table and disk usage value?

After a quota recalc everything is correct:

mail01:~# doveadm quota recalc -u username at example.org
mail01:~# doveadm -f tab quota get -u username at example.org
Quota name Type Value Limit %
User quota STORAGE 2630434 - 0
User quota MESSAGE 100714 - 0

mysql> select * from dovecot.dovecot_usage where username="username at
example.org";
+----------------------+------------+----------+
| username             | storage    | messages |
+----------------------+------------+----------+
| username at example.org | 2693565239 | 100714   | 
+----------------------+------------+----------+
1 row in set (0.00 sec)

How can I keep the accounting data consistent with actual disk usage,
without running a recalc cronjob every few hours?

There's running a dovecot director and mailbox instance on each of our
servers
with the mails being stored on NFS and received from postfix via LMTP through
the director lmtp port 20024.

Configuration of director (dovecot-director.conf) and
mailbox (dovecot.conf) is attached.

Kind regards
Daniel
-------------- next part --------------
# 2.0.20: /etc/dovecot/dovecot.conf
# OS: Linux 2.6.32-40-server x86_64 Ubuntu 10.04.4 LTS 
auth_cache_negative_ttl = 0
auth_cache_size = 10 M
auth_cache_ttl = 1 mins
auth_verbose = yes
auth_verbose_passwords = sha1
deliver_log_format = mailbox: deliver: msgid=%m from=%f: %$
dict {
  quota = mysql:/etc/dovecot/conf.d/dovecot-dict-sql.conf.ext
}
disable_plaintext_auth = no
lda_mailbox_autocreate = yes
lda_mailbox_autosubscribe = yes
login_greeting = Mailbox
login_log_format = mailbox: login: %$: %s
login_trusted_networks = 10.129.3.0/24
mail_debug = yes
mail_gid = vmail
mail_home = /mail/dovecot/%d/%n
mail_location = mdbox:~/mail
mail_log_prefix = "mailbox: mail: %s(%u): "
mail_plugins = quota
mail_privileged_group = vmail
mail_uid = vmail
managesieve_implementation_string = Sieve
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character
vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy
include variables body enotify environment mailbox date ihave
mdbox_rotate_interval = 1 weeks
mdbox_rotate_size = 50 M
mmap_disable = yes
passdb {
  args = /etc/dovecot/conf.d/dovecot-sql.conf.ext
  driver = sql
}
plugin {
  quota = dict:User quota::proxy::quota
  quota_rule = *:storage=10G
  quota_rule2 = Trash:storage=+100M
  quota_warning = storage=95%% quota-warning 95 %u
  quota_warning2 = storage=80%% quota-warning 80 %u
  sieve = ~/.dovecot.sieve
  sieve_dir = ~/sieve
}
protocols = imap pop3 lmtp sieve
service dict {
  unix_listener dict {
    group = vmail
    mode = 0660
  }
}
service imap-login {
  inet_listener imap {
    port = 19143
  }
}
service lmtp {
  inet_listener lmtp {
    address = *
    port = 19024
  }
}
service managesieve-login {
  inet_listener sieve {
    port = 19200
  }
}
service pop3-login {
  inet_listener pop3 {
    port = 19110
  }
}
service quota-warning {
  executable = script /usr/local/bin/quota-warning
  unix_listener quota-warning {
    user = vmail
  }
  user = dovecot
}
ssl = no
userdb {
  driver = prefetch
}
userdb {
  args = /etc/dovecot/conf.d/dovecot-sql.conf.ext
  driver = sql
}
verbose_proctitle = yes
protocol imap {
  imap_client_workarounds = delay-newmail tb-extra-mailbox-sep
  mail_plugins = quota imap_quota
}
protocol lmtp {
  mail_plugins = quota sieve
}
-------------- next part --------------
# 2.0.20: /etc/dovecot-director/dovecot-director.conf
# OS: Linux 2.6.32-40-server x86_64 Ubuntu 10.04.4 LTS 
auth_verbose = yes
auth_verbose_passwords = sha1
base_dir = /var/run/dovecot-director
deliver_log_format = director: deliver: msgid=%m from=%f: %$
director_mail_servers = 10.129.3.193 10.129.3.192 10.129.3.191 10.129.3.190
director_servers = 10.129.3.193 10.129.3.192 10.129.3.191 10.129.3.190
lmtp_proxy = yes
login_greeting = Mail Balancer
login_log_format = director: login: %$: %s
login_trusted_networks = 10.129.3.0/24
mail_debug = yes
mail_gid = vmail
mail_home = /mail/dovecot/%d/%n
mail_location = mdbox:~/mail
mail_log_prefix = "director: mail: %s(%u): "
mail_privileged_group = vmail
mail_uid = vmail
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character
vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy
include variables body enotify environment mailbox date ihave
mmap_disable = yes
passdb {
  args = proxy=y nopassword=y user=%n at dovecotmail.%d
  driver = static
}
protocols = imap pop3 lmtp sieve
service auth {
  unix_listener auth-userdb {
    user = dovecot
  }
}
service director {
  fifo_listener login/proxy-notify {
    mode = 0666
  }
  inet_listener {
    port = 9090
  }
  unix_listener director-userdb {
    mode = 0600
  }
  unix_listener login/director {
    mode = 0666
  }
}
service imap-login {
  executable = imap-login director
  inet_listener imap {
    port = 20143
  }
  inet_listener imaps {
    port = 20993
    ssl = yes
  }
}
service lmtp {
  inet_listener lmtp {
    address = *
    port = 20024
  }
}
service managesieve-login {
  inet_listener sieve {
    port = 20200
  }
}
service pop3-login {
  executable = pop3-login director
  inet_listener pop3 {
    port = 20110
  }
  inet_listener pop3s {
    port = 20995
    ssl = yes
  }
}
ssl_cert = </etc/certs/wildcard.example.org.crt
ssl_key = </etc/certs/wildcard.example.org.key
verbose_proctitle = yes
protocol lmtp {
  auth_socket_path = director-userdb
  passdb {
    args = proxy=y nopassword=y port=19024
    driver = static
  }
}
protocol sieve {
  auth_socket_path = director-userdb
  passdb {
    args = proxy=y nopassword=y port=19200
    driver = static
  }
}
protocol imap {
  imap_client_workarounds = delay-newmail tb-extra-mailbox-sep
}

On Sun, 2012-05-13 at 02:07 +0200, Daniel Parthey wrote:
> in my dovecot setup the accounting database table shows wrong
> values which do not correspond with the actual disk space used.
Can you easily reproduce this with a test user? How?

If there's no easy way to manually reproduce this, try if you can
reproduce it by running imaptest against a test user:
http://imapwiki.org/ImapTest

In the tests does it make difference if you bypass director?

Hi Timo,

Timo Sirainen wrote:
> On Sun, 2012-05-13 at 02:07 +0200, Daniel Parthey wrote:
> 
> > in my dovecot setup the accounting database table shows wrong
> > values which do not correspond with the actual disk space used.
> 
> Can you easily reproduce this with a test user? How?
Maybe this is not the original problem, but here is some way to
reproduce a similar behaviour.

#####################
# Close Thunderbird #
#####################

#################
# Clear Mailbox #
#################

rm -rf /mail/dovecot/example.org/user1
sleep 1
doveadm quota recalc -u user1 at example.org

###############
# Check quota #
###############

# doveadm quota get -u user1 at example.org
Quota name                                                                      
Type    Value Limit %
User quota                                                                      
STORAGE     0     - 0
User quota                                                                      
MESSAGE     0     - 0

################
# Deliver mail #
################

{
cat <<EOF
LHLO localhost
MAIL FROM:<user2 at example.org>
RCPT TO:<user1 at example.org>
DATA
EOF
dd if=/dev/zero count=1 bs=1M | base64
cat <<EOF
.
QUIT
EOF
} | nc localhost 19024

###############
# Check quota #
###############

# doveadm quota get -u user1 at example.org
Quota name                                                                      
Type    Value Limit %
User quota                                                                      
STORAGE  1383     - 0
User quota                                                                      
MESSAGE     1     - 0

#######################
# Check mdbox storage #
#######################

du -sk /mail/dovecot/example.org/user1
1432    /mail/dovecot/example.org/user1

#####################
# Move Email around #
#####################

Move Email two times between Trash and INBOX (4 moves in total).
There is still only one physical message, just moved it around a bit.

#####################
# Check quota again #
#####################

# doveadm quota get -u user1 at example.org
Quota name                                                                      
Type    Value Limit %
User quota                                                                      
STORAGE  6918     - 0
User quota                                                                      
MESSAGE     5     - 0

Quota now displays 5 messages and 6 MB, even though there is only one message
with 1.5 MB.

#######################
# Check mdbox storage #
#######################

# du -sk /mail/dovecot/example.org/user1
1472    /mail/dovecot/example.org/user1

The actual disk usage did not increase, but the "usage value" did
increase
with every move operation.

As long as the client has its MUA open, the usage is incorrectly displayed
(and accounted), and does not correspond with actual disk usage.

#####################
# Close Thunderbird #
#####################

The mailboxes get expunged.

#####################
# Check quota again #
#####################

# doveadm quota get -u user1 at example.org
Quota name                                                                      
Type    Value Limit %
User quota                                                                      
STORAGE  1383     - 0
User quota                                                                      
MESSAGE     1     - 0

Usage value and message count is correct again, once thunderbird is closed.

#######################
# Check mdbox storage #
#######################

# du -sk /mail/dovecot/example.org/user1
1472    /mail/dovecot/example.org/user1

The actual disk usage did not change, which is OK.
> 
> In the tests does it make difference if you bypass director?
No. The behaviour is the same if I access the mailbox directly.

Regards,
Daniel