Once in a while my web-server running a fairly recent amd64-STABLE
reboots, after a panic.
I have the vmcore.[789] of the last three panics, but the backtraces say
little to me; can some help interpret them? what's going on?
% uname -a
FreeBSD motoko.lapo.it 6.2-STABLE FreeBSD 6.2-STABLE #7: Fri Jun 15
15:41:02 CEST 2007 root@motoko.lapo.it:/usr/obj/usr/src/sys/MOTOKO amd64
% kgdb /usr/obj/usr/src/sys/MOTOKO/kernel.debug vmcore.9
kgdb: kvm_nlist(_stopped_cpus):
kgdb: kvm_nlist(_stoppcbs):
[GDB will not be able to debug user-mode threads:
/usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain
conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for
details.
This GDB was configured as "amd64-marcel-freebsd".
Unread portion of the kernel message buffer:
kernel trap 12 with interrupts disabled
Fatal trap 12: page fault while in kernel mode
fault virtual address = 0xd5a015
fault code = supervisor read data, page not present
instruction pointer = 0x8:0xffffffff80282fcf
stack pointer = 0x10:0xffffffffa4b31b80
frame pointer = 0x10:0xffffffff99d170a0
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags = resume, IOPL = 0
current process = 12 (swi4: clock sio)
trap number = 12
panic: page fault
Uptime: 18d20h28m34s
Physical memory: 1000 MB
Dumping 244 MB: 229 213 197 181 165 149 133 117 101 85 69 53 37 21 5
#0 doadump () at pcpu.h:172
172 pcpu.h: No such file or directory.
in pcpu.h
(kgdb) bt
#0 doadump () at pcpu.h:172
#1 0x0000000000000004 in ?? ()
#2 0xffffffff80273f63 in boot (howto=260)
at /usr/src/sys/kern/kern_shutdown.c:409
#3 0xffffffff80274566 in panic (fmt=0xffffff003cc7bbe0 "X??<")
at /usr/src/sys/kern/kern_shutdown.c:565
#4 0xffffffff803fdab1 in trap_fatal (frame=0xffffff003cc7bbe0,
eva=18446742975217640280) at /usr/src/sys/amd64/amd64/trap.c:668
#5 0xffffffff803fe026 in trap (frame {tf_rdi = -1714027920, tf_rsi =
1628883229, tf_rdx = 14000133,
tf_rcx = 1628883229, tf_r8 = -1531765488, tf_r9 = 1, tf_rax = 299472,
tf_rbx = 1, tf_rbp = -1714327392, tf_r10 = -2141205256, tf_r11 -1098491905056,
tf_r12 = 4, tf_r13 = -1099500839808, tf_r14 -1099511474880, tf_r15 = 2,
tf_trapno = 12, tf_addr = 14000149, tf_flags
= -2144823770, tf_err = 0, tf_rip = -2144849969, tf_cs = 8, tf_rflags 65538,
tf_rsp = -1531765872, tf_ss = 16})
at /usr/src/sys/amd64/amd64/trap.c:239
#6 0xffffffff803e859b in calltrap ()
at /usr/src/sys/amd64/amd64/exception.S:168
#7 0xffffffff80282fcf in softclock (dummy=0xffffffff99d60270)
at /usr/src/sys/kern/kern_timeout.c:220
#8 0xffffffff8025b3f5 in ithread_loop (arg=0xffffff0000025540)
at /usr/src/sys/kern/kern_intr.c:682
#9 0xffffffff80259e43 in fork_exit (
callout=0xffffffff8025b2b0 <ithread_loop>, arg=0xffffff0000025540,
frame=0xffffffffa4b31c50) at /usr/src/sys/kern/kern_fork.c:821
#10 0xffffffff803e88fe in fork_trampoline ()
at /usr/src/sys/amd64/amd64/exception.S:394
#11 0x0000000000000000 in ?? ()
[all zeroes from now on]
% kgdb /usr/obj/usr/src/sys/MOTOKO/kernel.debug vmcore.8
kgdb: kvm_nlist(_stopped_cpus):
kgdb: kvm_nlist(_stoppcbs):
[GDB will not be able to debug user-mode threads:
/usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain
conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for
details.
This GDB was configured as "amd64-marcel-freebsd".
Unread portion of the kernel message buffer:
Fatal trap 12: page fault while in kernel mode
fault virtual address = 0x6f
fault code = supervisor read data, page not present
instruction pointer = 0x8:0xffffffff8034c8bb
stack pointer = 0x10:0xffffffffa4b31b50
frame pointer = 0x10:0xffffff002e8f5480
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 12 (swi4: clock sio)
trap number = 12
panic: page fault
Uptime: 7d11h43m44s
Physical memory: 1000 MB
Dumping 238 MB: 223 207 191 175 159 143 127 111 95 79 63 47 31 15
#0 doadump () at pcpu.h:172
172 pcpu.h: No such file or directory.
in pcpu.h
(kgdb) bt
#0 doadump () at pcpu.h:172
#1 0x0000000000000004 in ?? ()
#2 0xffffffff80273f63 in boot (howto=260)
at /usr/src/sys/kern/kern_shutdown.c:409
#3 0xffffffff80274566 in panic (fmt=0xffffff003cc7bbe0 "X??<")
at /usr/src/sys/kern/kern_shutdown.c:565
#4 0xffffffff803fdab1 in trap_fatal (frame=0xffffff003cc7bbe0,
eva=18446742975217640280) at /usr/src/sys/amd64/amd64/trap.c:668
#5 0xffffffff803fde22 in trap_pfault (frame=0xffffffffa4b31aa0, usermode=0)
at /usr/src/sys/amd64/amd64/trap.c:580
#6 0xffffffff803fe0d3 in trap (frame {tf_rdi = -2142765429, tf_rsi =
-1098491905056, tf_rdx = 582,
tf_rcx = 0, tf_r8 = -1098491905056, tf_r9 = -1531766240, tf_rax = 0,
tf_rbx = -2141220736, tf_rbp = -1098730482560, tf_r10 = -1098730482504,
tf_r11 = -1714200192, tf_r12 = -1, tf_r13 = -1098730482560, tf_r14 -2144024512,
tf_r15 = 2, tf_trapno = 12, tf_addr = 111, tf_flags -1098699808768, tf_err = 0,
tf_rip = -2144024389, tf_cs = 8, tf_rflags 66182, tf_rsp = -1531765920, tf_ss =
16})
at /usr/src/sys/amd64/amd64/trap.c:353
#7 0xffffffff803e859b in calltrap ()
at /usr/src/sys/amd64/amd64/exception.S:168
#8 0xffffffff8034c8bb in nd6_llinfo_timer (arg=0xffffff002e8f5480)
at /usr/src/sys/netinet6/nd6.c:440
#9 0xffffffff802830da in softclock (dummy=0xffffffff8047fe8b)
at /usr/src/sys/kern/kern_timeout.c:290
#10 0xffffffff8025b3f5 in ithread_loop (arg=0xffffff0000025540)
at /usr/src/sys/kern/kern_intr.c:682
#11 0xffffffff80259e43 in fork_exit (
callout=0xffffffff8025b2b0 <ithread_loop>, arg=0xffffff0000025540,
frame=0xffffffffa4b31c50) at /usr/src/sys/kern/kern_fork.c:821
#12 0xffffffff803e88fe in fork_trampoline ()
at /usr/src/sys/amd64/amd64/exception.S:394
#13 0x0000000000000000 in ?? ()
[all zeroes from now on]
% kgdb /usr/obj/usr/src/sys/MOTOKO/kernel.debug vmcore.7
kgdb: kvm_nlist(_stopped_cpus):
kgdb: kvm_nlist(_stoppcbs):
[GDB will not be able to debug user-mode threads:
/usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain
conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for
details.
This GDB was configured as "amd64-marcel-freebsd".
Unread portion of the kernel message buffer:
kernel trap 9 with interrupts disabled
Fatal trap 9: general protection fault while in kernel mode
instruction pointer = 0x8:0xffffffff80282fcf
stack pointer = 0x10:0xffffffffa4b31b80
frame pointer = 0x10:0xffffffff99d170a0
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags = resume, IOPL = 0
current process = 12 (swi4: clock sio)
trap number = 9
panic: general protection fault
Uptime: 14d23h12m49s
Physical memory: 1000 MB
Dumping 247 MB: 232 216 200 184 168 152 136 120 104 88 72 56 40 24 8
#0 doadump () at pcpu.h:172
172 pcpu.h: No such file or directory.
in pcpu.h
(kgdb) bt
#0 doadump () at pcpu.h:172
#1 0x0000000000000004 in ?? ()
#2 0xffffffff80273f63 in boot (howto=260)
at /usr/src/sys/kern/kern_shutdown.c:409
#3 0xffffffff80274566 in panic (fmt=0xffffff003cc7bbe0 "X??<")
at /usr/src/sys/kern/kern_shutdown.c:565
#4 0xffffffff803fdab1 in trap_fatal (frame=0xffffff003cc7bbe0,
eva=18446742975217640280) at /usr/src/sys/amd64/amd64/trap.c:668
#5 0xffffffff803fdfa2 in trap (frame {tf_rdi = -1713989424, tf_rsi =
1293144707, tf_rdx 7146756794183803759, tf_rcx = 1293144707, tf_r8 =
-1531765488, tf_r9 1, tf_rax = 337968, tf_rbx = 1, tf_rbp = -1714327392, tf_r10
-2141205256, tf_r11 = -1098491905056, tf_r12 = 4, tf_r13 -1099500839808, tf_r14
= -1099511474880, tf_r15 = 2, tf_trapno = 9,
tf_addr = 0, tf_flags = -2144823770, tf_err = 0, tf_rip = -2144849969,
tf_cs = 8, tf_rflags = 65538, tf_rsp = -1531765872, tf_ss = 16})
at /usr/src/sys/amd64/amd64/trap.c:470
#6 0xffffffff803e859b in calltrap ()
at /usr/src/sys/amd64/amd64/exception.S:168
#7 0xffffffff80282fcf in softclock (dummy=0xffffffff99d698d0)
at /usr/src/sys/kern/kern_timeout.c:220
#8 0xffffffff8025b3f5 in ithread_loop (arg=0xffffff0000025540)
at /usr/src/sys/kern/kern_intr.c:682
#9 0xffffffff80259e43 in fork_exit (
callout=0xffffffff8025b2b0 <ithread_loop>, arg=0xffffff0000025540,
frame=0xffffffffa4b31c50) at /usr/src/sys/kern/kern_fork.c:821
#10 0xffffffff803e88fe in fork_trampoline ()
at /usr/src/sys/amd64/amd64/exception.S:394
#11 0x0000000000000000 in ?? ()
[all zeroes from now on]