We commisioned a new box running FreeBSD 6.2 Stable (as of Sunday
afternoon UK time), to replace a broken 4.x box.
It has been crashing around once per day since then. I have gathered a
couple of cores so far and it looks like it's the same thing causing the
crash. I'm no expert in reading these things, but it looks like it's
happening the IPv6 stack? This host is IPv6 connected whereas our other
6.x boxes aren't..
The stack traces are attached below. Any help would be much appreciated.
Thanks
Jake
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for
details.
This GDB was configured as "i386-marcel-freebsd".
Unread portion of the kernel message buffer:
kernel trap 12 with interrupts disabled
Fatal trap 12: page fault while in kernel mode
cpuid = 0; apic id = 00
fault virtual address = 0x104
fault code = supervisor read, page not present
instruction pointer = 0x20:0xc0512029
stack pointer = 0x28:0xf388b88c
frame pointer = 0x28:0xf388b8a0
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = resume, IOPL = 0
current process = 682 (named)
trap number = 12
panic: page fault
cpuid = 0
Uptime: 1d4h14m2s
Dumping 3839 MB (2 chunks)
chunk 0: 1MB (159 pages) ... ok
chunk 1: 3839MB (982743 pages) 3823 3807 3791 3775 3759 3743 3727 3711 3695
3679 3663 3647 3631 3615 3599 3583 3567 3551 3535 3519 3503 3487 3471 3455 3439
3423 3407 3391 3375 3359 3343 3327 3311 3295 3279 3263 3247 3231 3215 3199 3183
3167 3151 3135 3119 3103 3087 3071 3055 3039 3023 3007 2991 2975 2959 2943 2927
2911 2895 2879 2863 2847 2831 2815 2799 2783 2767 2751 2735 2719 2703 2687 2671
2655 2639 2623 2607 2591 2575 2559 2543 2527 2511 2495 2479 2463 2447 2431 2415
2399 2383 2367 2351 2335 2319 2303 2287 2271 2255 2239 2223 2207 2191 2175 2159
2143 2127 2111 2095 2079 2063 2047 2031 2015 1999 1983 1967 1951 1935 1919 1903
1887 1871 1855 1839 1823 1807 1791 1775 1759 1743 1727 1711 1695 1679 1663 1647
1631 1615 1599 1583 1567 1551 1535 1519 1503 1487 1471 1455 1439 1423 1407 1391
1375 1359 1343 1327 1311 1295 1279 1263 1247 1231 1215 1199 1183 1167 1151 1135
1119 1103 1087 1071 1055 1039 1023 1007 991 975 959 943 927 911 895 879 863 847
831 815 799 783 767 751 735 719
703 687 671 655 639 623 607 591 575 559 543 527 511 495 479 463 447 431 415
399 383 367 351 335 319 303 287 271 255 239 223 207 191 175 159 143 127 111 95
79 63 47 31 15
#0 doadump () at pcpu.h:165
in pcpu.h
(kgdb) bt
#0 doadump () at pcpu.h:165
#1 0xc051c88e in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:409
#2 0xc051cc3f in panic (fmt=0xc06f2b48 "%s")
at /usr/src/sys/kern/kern_shutdown.c:565
#3 0xc06cd925 in trap_fatal (frame=0xf388b84c, eva=0)
at /usr/src/sys/i386/i386/trap.c:837
#4 0xc06cd00d in trap (frame {tf_fs = -1067581432, tf_es = -209190872,
tf_ds = -922091480, tf_edi = -919646848, tf_esi = 4, tf_ebp = -209143648, tf_isp
= -209143688, tf_ebx = -917625596, tf_edx = 6, tf_ecx = 3, tf_eax = 1, tf_trapno
= 12, tf_err = 0, tf_eip = -1068425175, tf_cs = 32, tf_eflags = 65538, tf_esp =
-917625596, tf_ss = -917667672}) at /usr/src/sys/i386/i386/trap.c:270
#5 0xc06b694a in calltrap () at /usr/src/sys/i386/i386/exception.s:139
#6 0xc0512029 in _mtx_lock_sleep (m=0xc94e2504, tid=3375320448, opts=0,
file=0x0, line=0) at /usr/src/sys/kern/kern_mutex.c:546
#7 0xc05ece2e in nd6_output (ifp=0xc90a6800, origifp=0x1, m0=0xc9824b00,
dst=0xc92d001c, rt0=0xc94e2528) at /usr/src/sys/netinet6/nd6.c:2010
#8 0xc05e6036 in ip6_output (m0=0xf388ba60, opt=0xc95f2500, ro=0xf388ba60,
flags=0, im6o=0x0, ifpp=0x0, inp=0xc94df438)
at /usr/src/sys/netinet6/ip6_output.c:994
#9 0xc05f626c in udp6_output (in6p=0xc94df438, m=0xc9824b00,
addr6=0xc9824ba8, control=0x0, td=0xc92f4d80)
at /usr/src/sys/netinet6/udp6_output.c:322
#10 0xc05f7afc in udp6_send (so=0xc94e7c84, flags=1, m=0xc9824b00,
addr=0xc926ad20, control=0x0, td=0xc92f4d80)
at /usr/src/sys/netinet6/udp6_usrreq.c:794
#11 0xc0561094 in sosend (so=0xc94e7c84, addr=0xc926ad20, uio=0xf388bc30,
top=0xc9824b00, control=0x0, flags=0, td=0xc92f4d80)
at /usr/src/sys/kern/uipc_socket.c:836
#12 0xc0567be4 in kern_sendit (td=0xc92f4d80, s=31, mp=0xf388bca8, flags=0,
control=0x1, segflg=UIO_SYSSPACE) at /usr/src/sys/kern/uipc_syscalls.c:772
#13 0xc0567a6d in sendit (td=0x1, s=1, mp=0xf388bca8, flags=1)
at /usr/src/sys/kern/uipc_syscalls.c:712
#14 0xc0567f64 in sendmsg (td=0x1, uap=0xf388bd04)
at /usr/src/sys/kern/uipc_syscalls.c:920
#15 0xc06cdd0c in syscall (frame {tf_fs = 135987259, tf_es = 163446843,
tf_ds = -1078001605, tf_edi = 0, tf_esi = 0, tf_ebp = -1077942424, tf_isp =
-209142428, tf_ebx = 0, tf_edx = 144072704, tf_ecx = 144072704, tf_eax = 28,
tf_trapno = 0, tf_err = 2, tf_eip = 674625719, tf_cs = 51, tf_eflags = 642,
tf_esp = -1077942788, tf_ss = 59})
at /usr/src/sys/i386/i386/trap.c:983
#16 0xc06b699f in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception.s:200
#17 0x00000033 in ?? ()
(kgdb)
-------------------
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for
details.
This GDB was configured as "i386-marcel-freebsd".
Unread portion of the kernel message buffer:
kernel trap 12 with interrupts disabled
Fatal trap 12: page fault while in kernel mode
cpuid = 1; apic id = 01
fault virtual address = 0x104
fault code = supervisor read, page not present
instruction pointer = 0x20:0xc0512029
stack pointer = 0x28:0xf389a88c
frame pointer = 0x28:0xf389a8a0
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = resume, IOPL = 0
current process = 689 (named)
trap number = 12
panic: page fault
cpuid = 1
Uptime: 1d1h12m54s
Dumping 3839 MB (2 chunks)
chunk 0: 1MB (159 pages) ... ok
chunk 1: 3839MB (982743 pages) 3823 3807 3791 3775 3759 3743 3727 3711 3695
3679 3663 3647 3631 3615 3599 3583 3567 3551 3535 3519 3503 3487 3471 3455 3439
3423 3407 3391 3375 3359 3343 3327 3311 3295 3279 3263 3247 3231 3215 3199 3183
3167 3151 3135 3119 3103 3087 3071 3055 3039 3023 3007 2991 2975 2959 2943 2927
2911 2895 2879 2863 2847 2831 2815 2799 2783 2767 2751 2735 2719 2703 2687 2671
2655 2639 2623 2607 2591 2575 2559 2543 2527 2511 2495 2479 2463 2447 2431 2415
2399 2383 2367 2351 2335 2319 2303 2287 2271 2255 2239 2223 2207 2191 2175 2159
2143 2127 2111 2095 2079 2063 2047 2031 2015 1999 1983 1967 1951 1935 1919 1903
1887 1871 1855 1839 1823 1807 1791 1775 1759 1743 1727 1711 1695 1679 1663 1647
1631 1615 1599 1583 1567 1551 1535 1519 1503 1487 1471 1455 1439 1423 1407 1391
1375 1359 1343 1327 1311 1295 1279 1263 1247 1231 1215 1199 1183 1167 1151 1135
1119 1103 1087 1071 1055 1039 1023 1007 991 975 959 943 927 911 895 879 863 847
831 815 799 783 767 751 735 719
703 687 671 655 639 623 607 591 575 559 543 527 511 495 479 463 447 431 415
399 383 367 351 335 319 303 287 271 255 239 223 207 191 175 159 143 127 111 95
79 63 47 31 15
#0 doadump () at pcpu.h:165
in pcpu.h
(kgdb) bt
#0 doadump () at pcpu.h:165
#1 0xc051c88e in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:409
#2 0xc051cc3f in panic (fmt=0xc06f2b48 "%s")
at /usr/src/sys/kern/kern_shutdown.c:565
#3 0xc06cd925 in trap_fatal (frame=0xf389a84c, eva=0)
at /usr/src/sys/i386/i386/trap.c:837
#4 0xc06cd00d in trap (frame {tf_fs = -1067581432, tf_es = -209125336,
tf_ds = -922091480, tf_edi = -919476736, tf_esi = 4, tf_ebp = -209082208, tf_isp
= -209082248, tf_ebx = -917731828, tf_edx = 6, tf_ecx = 3, tf_eax = 1, tf_trapno
= 12, tf_err = 0, tf_eip = -1068425175, tf_cs = 32, tf_eflags = 65538, tf_esp =
-917731828, tf_ss = -920005464}) at /usr/src/sys/i386/i386/trap.c:270
#5 0xc06b694a in calltrap () at /usr/src/sys/i386/i386/exception.s:139
#6 0xc0512029 in _mtx_lock_sleep (m=0xc94c860c, tid=3375490560, opts=0,
file=0x0, line=0) at /usr/src/sys/kern/kern_mutex.c:546
#7 0xc05ece2e in nd6_output (ifp=0xc90a6800, origifp=0x1, m0=0xd065be00,
dst=0xc92d2c9c, rt0=0xc94c1630) at /usr/src/sys/netinet6/nd6.c:2010
#8 0xc05e6036 in ip6_output (m0=0xf389aa60, opt=0xc943fd80, ro=0xf389aa60,
flags=0, im6o=0x0, ifpp=0x0, inp=0xc94c70b4)
at /usr/src/sys/netinet6/ip6_output.c:994
#9 0xc05f626c in udp6_output (in6p=0xc94c70b4, m=0xd065be00,
addr6=0xd065be98, control=0x0, td=0xc931e600)
at /usr/src/sys/netinet6/udp6_output.c:322
#10 0xc05f7afc in udp6_send (so=0xc93109bc, flags=1, m=0xd065be00,
addr=0xc92913a0, control=0x0, td=0xc931e600)
at /usr/src/sys/netinet6/udp6_usrreq.c:794
#11 0xc0561094 in sosend (so=0xc93109bc, addr=0xc92913a0, uio=0xf389ac30,
top=0xd065be00, control=0x0, flags=0, td=0xc931e600)
at /usr/src/sys/kern/uipc_socket.c:836
#12 0xc0567be4 in kern_sendit (td=0xc931e600, s=31, mp=0xf389aca8, flags=0,
control=0x1, segflg=UIO_SYSSPACE) at /usr/src/sys/kern/uipc_syscalls.c:772
#13 0xc0567a6d in sendit (td=0x1, s=1, mp=0xf389aca8, flags=1)
at /usr/src/sys/kern/uipc_syscalls.c:712
#14 0xc0567f64 in sendmsg (td=0x1, uap=0xf389ad04)
at /usr/src/sys/kern/uipc_syscalls.c:920
#15 0xc06cdd0c in syscall (frame {tf_fs = 175702075, tf_es = 59, tf_ds =
-1078001605, tf_edi = 0, tf_esi = 0, tf_ebp = -1077942440, tf_isp = -209080988,
tf_ebx = 0, tf_edx = 141930240, tf_ecx = 141930240, tf_eax = 28, tf_trapno = 0,
tf_err = 2, tf_eip = 674625719, tf_cs = 51, tf_eflags = 642, tf_esp =
-1077942804, tf_ss = 59})
at /usr/src/sys/i386/i386/trap.c:983
#16 0xc06b699f in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception.s:200
#17 0x00000033 in ?? ()
(kgdb)
Jake