On 10/05/2015 5:47 PM, Timo Sirainen wrote:
>> How do I go about debugging this further? How do I go about
>> getting a backtrace on what is essentially a child process that
>> doesn't produce a core? (Note that my installation normally does
>> produce cores when it crashes, so there's something different about
>> these indexer processes..)
>
> One thing that often works is:
>
> mkdir /var/core; chmod 0777 /var/core echo "/var/core/%p" >
> /proc/sys/kernel/core_pattern
>
> If that doesn't help, you can also try: sysctl -w fs.suid_dumpable=2
> (and disable afterwards)
>
> Also if you can get indexer-worker to start without crashing (new
> mail delivery works?) you can attach gdb into it and wait for it to
> crash (gdb -p <indexer-worker pid>, cont).
Ok we have some (well, lots actually) of coredumps now:
tornado core # gdb /usr/libexec/dovecot/indexer-worker 26697
GNU gdb (Gentoo 7.9 vanilla) 7.9
Copyright (C) 2015 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later
<http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show
copying"
and "show warranty" for details.
This GDB was configured as "x86_64-pc-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://bugs.gentoo.org/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to
"word"...
Reading symbols from /usr/libexec/dovecot/indexer-worker...done.
Attaching to program: /usr/libexec/dovecot/indexer-worker, process 26697
ptrace: No such process.
[New LWP 26697]
warning: Could not load shared library symbols for linux-vdso.so.1.
Do you need "set solib-search-path" or "set sysroot"?
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `dovecot/indexer-worker'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x00007f4834fcc222 in fts_user_get_data_lang (user=0x16c7900)
at fts-user.c:302
302 fts-user.c: No such file or directory.
(gdb) info locals
hdr = 0x1701f08
key = {uid = 8553, type = FTS_BACKEND_BUILD_KEY_HDR, part = 0x1696860,
hdr_name = 0x1702120 "To", body_content_type = 0x0,
body_content_disposition = 0x0}
(gdb) bt full
#0 0x00007f4834fcc222 in fts_user_get_data_lang (user=0x16c7900)
at fts-user.c:302
fuser = 0x0
lang = 0x0
error = 0xf <error: Cannot access memory at address 0xf>
__FUNCTION__ = "fts_user_get_data_lang"
#1 0x00007f4834fc1bdb in fts_build_mail_header (ctx=0x7ffd5e021ca0,
block=0x7ffd5e021c80) at fts-build-mail.c:164
hdr = 0x1701f08
key = {uid = 8553, type = FTS_BACKEND_BUILD_KEY_HDR, part =
0x1696860,
hdr_name = 0x1702120 "To", body_content_type = 0x0,
body_content_disposition = 0x0}
#2 0x00007f4834fc2b68 in fts_build_mail_real (update_ctx=0x16f5130,
mail=0x17010f0) at fts-build-mail.c:540
ctx = {mail = 0x17010f0, update_ctx = 0x16f5130, content_type =
0x0,
content_disposition = 0x0, body_parser = 0x0, word_buf = 0x0,
pending_input = 0x0, cur_user_lang = 0x0}
input = 0x1701990
parser = 0x1701b20
decoder = 0x1701ef0
raw_block = {part = 0x1696860, hdr = 0x1702050, data = 0x0,
size = 0}
block = {part = 0x1696860, hdr = 0x1701f08, data =
0x7ffd5e021cd8 "",
size = 0}
prev_part = 0x1696860
parts = 0x7f4835cc94c0 <static_system_pool>
skip_body = false
body_part = false
body_added = false
binary_body = 192
ret = 1
__FUNCTION__ = "fts_build_mail_real"
#3 0x00007f4834fc2dcf in fts_build_mail (update_ctx=0x16f5130,
mail=0x17010f0)
at fts-build-mail.c:583
_data_stack_cur_id = 6
ret = 0
#4 0x00007f4834fca343 in fts_mail_index (_mail=0x17010f0) at
fts-storage.c:484
ft = 0x16fd570
flist = 0x16cc690
#5 0x00007f4834fca440 in fts_mail_precache (_mail=0x17010f0)
at fts-storage.c:501
_data_stack_cur_id = 5
mail = 0x17010f0
fmail = 0x1701580
ft = 0x16fd570
#6 0x00007f4835cf633a in mail_precache (mail=0x17010f0) at mail.c:401
_data_stack_cur_id = 4
p = 0x17010f0
#7 0x00000000004024b5 in index_mailbox_precache (conn=0x16a3040,
box=0x16d0580) at master-connection.c:93
storage = 0x16cba20
username = 0x16c79e8 "reuben"
box_vname = 0x16d0888 "Sent"
status = {messages = 453, recent = 0, unseen = 0,
uidvalidity = 1267935879, uidnext = 8558, first_unseen_seq = 0,
first_recent_uid = 8557, last_cached_seq = 448,
highest_modseq = 0,
highest_pvt_modseq = 0, keywords = 0x0, permanent_flags = 0,
permanent_keywords = 0, allow_new_keywords = 0,
nonpermanent_modseqs = 0, no_modseq_tracking = 0, have_guids
= 1,
have_save_guids = 1, have_only_guid128 = 0}
trans = 0x16e4b80
search_args = 0x0
ctx = 0x16d0370
mail = 0x17010f0
metadata = {guid = '\000' <repeats 15 times>,
virtual_size = 0,
cache_fields = 0x0,
precache_fields = (MAIL_FETCH_STREAM_HEADER |
MAIL_FETCH_STREAM_BODY | MAIL_FETCH_RECEIVED_DATE | MAIL_FETCH_SAVE_DATE
| MAIL_FETCH_PHYSICAL_SIZE | MAIL_FETCH_VIRTUAL_SIZE |
MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_GUID),
backend_ns_prefix = 0x0, backend_ns_type = (unknown: 0)}
seq = 449
percentage_str = "\000\000\000"
counter = 0
max = 5
percentage = 0
percentage_sent = 0
ret = 0
#8 0x000000000040295d in index_mailbox (conn=0x16a3040, user=0x16c7900,
mailbox=0x1694067 "Sent", max_recent_msgs=0, what=0x169406e
"i")
at master-connection.c:186
ns = 0x16cb8d0
box = 0x16d0580
status = {messages = 23709136, recent = 0, unseen = 23709136,
uidvalidity = 0, uidnext = 4294966784,
first_unseen_seq = 4294967295, first_recent_uid = 23683432,
last_cached_seq = 0, highest_modseq = 140726180650944,
highest_pvt_modseq = 4203059, keywords = 0x16a1bc0,
permanent_flags = 0, permanent_keywords = 0,
allow_new_keywords = 0,
nonpermanent_modseqs = 0, no_modseq_tracking = 0, have_guids
= 0,
have_save_guids = 0, have_only_guid128 = 0}
path = 0x1696180 "/home/reuben/Maildir/.Sent"
errstr = 0x7f48359cafc1 <master_service_settings_get+31>
"H\211E\370H\213E\370H\213"
error = MAIL_ERROR_NONE
sync_flags = MAILBOX_SYNC_FLAG_FULL_READ
ret = 0
#9 0x0000000000402b1e in master_connection_input_line (conn=0x16a3040,
line=0x16a3f42 "reuben\tSent\t0\ti") at master-connection.c:222
args = 0x1694070
input = {module = 0x403134 "mail",
service = 0x403139 "indexer-worker", username = 0x1694060
"reuben",
session_id = 0x0, local_ip = {family = 0, u = {ip6 = {__in6_u = {
__u6_addr8 = '\000' <repeats 15 times>,
__u6_addr16 =
{0, 0,
0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}},
ip4 = {
s_addr = 0}}}, remote_ip = {family = 0, u = {ip6 =
{__in6_u = {
__u6_addr8 = '\000' <repeats 15 times>,
__u6_addr16 =
{0, 0,
0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}},
ip4 = {
s_addr = 0}}}, local_port = 0, remote_port = 0,
userdb_fields = 0x0, flags_override_add = (unknown: 0),
flags_override_remove = (unknown: 0), no_userdb_lookup = 0}
service_user = 0x16a6000
user = 0x16c7900
str = 0x16a3f3e "1\t0"
error = 0x7f4835ff3bb0 "I\211\303f\017\032\\$0f\017\032T$
f\017\032L$\020f\017\032\004$L\213L$pL\213D$hH\213|$`H\213t$XH\213T$PH\213L$HH\213D$@H\201\304\210"
max_recent_msgs = 0
ret = 0
#10 0x0000000000402c94 in master_connection_input (conn=0x16a3040)
at master-connection.c:259
_data_stack_cur_id = 3
line = 0x16a3f42 "reuben\tSent\t0\ti"
ret = 32584
#11 0x00007f4835a53548 in io_loop_call_io (io=0x16a3080) at ioloop.c:501
ioloop = 0x169c740
t_id = 2
__FUNCTION__ = "io_loop_call_io"
#12 0x00007f4835a557f6 in io_loop_handler_run_internal (ioloop=0x169c740)
at ioloop-epoll.c:220
ctx = 0x16a15f0
events = 0x16a2430
event = 0x16a2430
list = 0x16a30e0
io = 0x16a3080
tv = {tv_sec = 2147483, tv_usec = 0}
events_count = 4
msecs = -1
ret = 1
i = 0
j = 0
call = true
__FUNCTION__ = "io_loop_handler_run_internal"
#13 0x00007f4835a536e5 in io_loop_handler_run (ioloop=0x169c740)
at ioloop.c:548
No locals.
#14 0x00007f4835a5363d in io_loop_run (ioloop=0x169c740) at ioloop.c:525
__FUNCTION__ = "io_loop_run"
#15 0x00007f48359c8215 in master_service_run (service=0x169c5d0,
callback=0x401fb6 <client_connected>) at master-service.c:569
No locals.
#16 0x0000000000402182 in main (argc=1, argv=0x169c390) at
indexer-worker.c:78
service_flags = MASTER_SERVICE_FLAG_KEEP_CONFIG_OPEN
storage_service_flags =
(MAIL_STORAGE_SERVICE_FLAG_DISALLOW_ROOT |
MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP |
MAIL_STORAGE_SERVICE_FLAG_TEMP_PRIV_DROP |
MAIL_STORAGE_SERVICE_FLAG_NO_IDLE_TIMEOUT)
c = -1
(gdb)
Thanks,
Reuben