My router running FBSD 6.1-RELEASE, 6.1-STABLE and now 6.2-PRERELEASE
periodically would reboot. It would always have the same basic
information in the kernel panic. I have finally gotten it setup for
a debug kernel and a place to store the crash.
Here is the information, what are my next steps.
Jim
Script started on Thu Oct 26 14:28:46 2006
router# kgdb kernel.debug /var/crash/vmcore.0
kgdb: kvm_nlist(_stopped_cpus):
kgdb: kvm_nlist(_stoppcbs):
[GDB will not be able to debug user-mode threads: /usr/lib/
libthread_db.so: Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and
you are
welcome to change it and/or distribute copies of it under certain
conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for
details.
This GDB was configured as "i386-marcel-freebsd".
Unread portion of the kernel message buffer:
kernel trap 12 with interrupts disabled
Fatal trap 12: page fault while in kernel mode
fault virtual address = 0x1104768
fault code = supervisor read, page not present
instruction pointer = 0x20:0xc04e50a1
stack pointer = 0x28:0xcc680b08
frame pointer = 0x28:0xcc680b0c
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = resume, IOPL = 0
current process = 3690 (ifconfig)
trap number = 12
panic: page fault
Uptime: 5h36m14s
Dumping 126 MB (2 chunks)
chunk 0: 1MB (160 pages) ... ok
chunk 1: 126MB (32240 pages) 110 94 78 62 46 30 14
#0 doadump () at pcpu.h:165
165 __asm __volatile("movl %%fs:0,%0" : "=r"
(td));
(kgdb) list *0xc04e50a1
0xc04e50a1 is in turnstile_setowner (/usr/src/sys/kern/
subr_turnstile.c:433).
428
429 mtx_assert(&td_contested_lock, MA_OWNED);
430 MPASS(owner->td_proc->p_magic == P_MAGIC);
431 MPASS(ts->ts_owner == NULL);
432 ts->ts_owner = owner;
433 LIST_INSERT_HEAD(&owner->td_contested, ts, ts_link);
434 }
435
436 /*
437 * Malloc a turnstile for a new thread, initialize it and
return it.
(kgdb) bt
#0 doadump () at pcpu.h:165
#1 0xc04c4b02 in boot (howto=260) at /usr/src/sys/kern/
kern_shutdown.c:409
#2 0xc04c4d98 in panic (fmt=0xc06683c3 "%s")
at /usr/src/sys/kern/kern_shutdown.c:565
#3 0xc0647000 in trap_fatal (frame=0xcc680ac8, eva=17844072)
at /usr/src/sys/i386/i386/trap.c:837
#4 0xc06467e2 in trap (frame {tf_fs = -1060962296, tf_es = -865599448,
tf_ds = -1067515864,
tf_edi = -1067971588, tf_esi = -1050101120, tf_ebp = -865596660,
tf_isp = -865596684, tf_ebx = -1050353088, tf_edx = -1050353088,
tf_ecx = 17843956, tf_eax = -1050101088, tf_trapno = 12, tf_err = 0,
tf_eip = -1068609375, tf_cs = 32, tf_eflags = 65543, tf_esp =
-1050101120, tf_ss = -865596624})
at /usr/src/sys/i386/i386/trap.c:270
#5 0xc0635a9a in calltrap () at /usr/src/sys/i386/i386/exception.s:139
#6 0xc04e50a1 in turnstile_setowner (ts=0xc164e240, owner=0x11046f4)
at /usr/src/sys/kern/subr_turnstile.c:432
#7 0xc04e5398 in turnstile_wait (lock=0xc0580e5c, owner=0x11046f4)
at /usr/src/sys/kern/subr_turnstile.c:591
#8 0xc04bb284 in _mtx_lock_sleep (m=0xc0580e5c, tid=3244866176, opts=0,
file=0x0, line=0) at /usr/src/sys/kern/kern_mutex.c:579
#9 0xc05327df in if_delmulti (ifp=0xc0580bfc, sa=0x262beca)
at /usr/src/sys/net/if.c:2083
#10 0xc0581783 in in6_delmulti (in6m=0xc1980980)
at /usr/src/sys/netinet6/mld6.c:649
#11 0xc05731c0 in in6_ifdetach (ifp=0xc15ba400)
at /usr/src/sys/netinet6/in6_ifattach.c:806
#12 0xc05301f4 in if_detach (ifp=0xc15ba400) at /usr/src/sys/net/if.c:
665
#13 0xc0535508 in gif_destroy (sc=0xc19a3900) at /usr/src/sys/net/
if_gif.c:209
#14 0xc05355ba in gif_clone_destroy (ifp=0xc168baa0)
at /usr/src/sys/net/if_gif.c:226
#15 0xc0533aca in ifc_simple_destroy (ifc=0xc06a3520, ifp=0x11046f4)
at /usr/src/sys/net/if_clone.c:478
#16 0xc0533109 in if_clone_destroy (name=0xc18e8140 "gif0")
at /usr/src/sys/net/if_clone.c:172
---Type <return> to continue, or q <return> to quit---
#17 0xc0531cb8 in ifioctl (so=0xc19c52c8, cmd=2149607801,
data=0xc18e8140 "gif0", td=0xc168ba80) at /usr/src/sys/net/if.c:
1533
#18 0xc04ec9bf in soo_ioctl (fp=0xc168baa0, cmd=2149607801,
data=0xc18e8140,
active_cred=0xc14fad00, td=0xc168ba80)
at /usr/src/sys/kern/sys_socket.c:214
#19 0xc04e7089 in ioctl (td=0xc168ba80, uap=0xcc680d04) at file.h:264
#20 0xc0647317 in syscall (frame {tf_fs = 59, tf_es = 59, tf_ds = 59,
tf_edi = 134571680,
tf_esi = 1, tf_ebp = -1077942312, tf_isp = -865596060, tf_ebx =
-1077941000, tf_edx = 134583517, tf_ecx = 0, tf_eax = 54, tf_trapno =
12, tf_err = 2, tf_eip = 672448383, tf_cs = 51, tf_eflags = 646,
tf_esp = -1077942340, tf_ss = 59})
at /usr/src/sys/i386/i386/trap.c:983
#21 0xc0635aef in Xint0x80_syscall () at /usr/src/sys/i386/i386/
exception.s:200
#22 0x00000033 in ?? ()
Previous frame inner to this frame (corrupt stack?)
(kgdb) up
#1 0xc04c4b02 in boot (howto=260) at /usr/src/sys/kern/
kern_shutdown.c:409
409 doadump();
(kgdb) up
#2 0xc04c4d98 in panic (fmt=0xc06683c3 "%s")
at /usr/src/sys/kern/kern_shutdown.c:565
565 boot(bootopt);
(kgdb) up
#3 0xc0647000 in trap_fatal (frame=0xcc680ac8, eva=17844072)
at /usr/src/sys/i386/i386/trap.c:837
837 panic("%s", trap_msg[type]);
(kgdb) up
#4 0xc06467e2 in trap (frame {tf_fs = -1060962296, tf_es = -865599448,
tf_ds = -1067515864,
tf_edi = -1067971588, tf_esi = -1050101120, tf_ebp = -865596660,
tf_isp = -865596684, tf_ebx = -1050353088, tf_edx = -1050353088,
tf_ecx = 17843956, tf_eax = -1050101088, tf_trapno = 12, tf_err = 0,
tf_eip = -1068609375, tf_cs = 32, tf_eflags = 65543, tf_esp =
-1050101120, tf_ss = -865596624})
at /usr/src/sys/i386/i386/trap.c:270
270 trap_fatal(&frame, eva);
(kgdb) up
#5 0xc0635a9a in calltrap () at /usr/src/sys/i386/i386/exception.s:139
139 call trap
Current language: auto; currently asm
(kgdb) up
#6 0xc04e50a1 in turnstile_setowner (ts=0xc164e240, owner=0x11046f4)
at /usr/src/sys/kern/subr_turnstile.c:432
432 ts->ts_owner = owner;
Current language: auto; currently c
(kgdb) list
427 {
428
429 mtx_assert(&td_contested_lock, MA_OWNED);
430 MPASS(owner->td_proc->p_magic == P_MAGIC);
431 MPASS(ts->ts_owner == NULL);
432 ts->ts_owner = owner;
433 LIST_INSERT_HEAD(&owner->td_contested, ts, ts_link);
434 }
435
436 /*
(kgdb) p owner
$1 = (struct thread *) 0x11046f4
(kgdb) p ts
$2 = (struct turnstile *) 0xc164e240
(kgdb) p tx s_link
No symbol "ts_link" in current context.
(kgdb) router# ^Dexit
Script done on Thu Oct 26 14:31:20 2006
--
/"\ ASCII Ribbon Campaign .
\ / - NO HTML/RTF in e-mail .
X - NO Word docs in e-mail .
/ \ -----------------------------------------------------------------
jeh@FreeBSD.org http://www.FreeBSD.org The Power to Serve
jim@TheHousleys.Net http://www.TheHousleys.net
---------------------------------------------------------------------
"Eagles may soar, but weasels don't get sucked into jet engines"
-- Anon