Hi, I experienced this new fxp LOR (probably when ntpd started during boot): hand-copied: lock order reversal 1st 0xcc666360 inp (udpinp) @ sys/netinet/udp_usrreq.c:772 2nd 0xc14c8264 fxp0 (network driver) @ /usr/src/sys/modules/fxp/../../dev/fxp/if_fxp.c:1232 KDB: stack backtrace: kdb_backtrace+0x2e(c06fd115, c14c8264, c14a57e0, c06fe820, c06fe7bd) witness_checkorder+0x6a6(c14c8264, 9, c06fe7bd, 4d0, c0602589) _mtx_lock_flags+0x8a(c14c8264, 0, c06fe7bd, 4d0, c14c8000) fxp_start+0x37(c14c8000, 12b, 0, c14c8000) and continuing from dmesg: if_start(c14c8000,0,c0602589,184,2) at 0xc050b1c9 = if_start+0x99 ether_output_frame(c14c8000,c15d3900,6,ca422b10,ca422aac) at 0xc050c908 = ether_output_frame+0x218 ether_output(c14c8000,c15d3900,ca422b10,c16687bc,c04bbc77) at 0xc050c6de = ether_output+0x44e ip_output(c15d3900,0,ca422b0c,0,0) at 0xc05210c7 = ip_output+0x7c7 udp_output(c16662d0,c15d3900,0,0,c148fa80) at 0xc053659a = udp_output+0x53a udp_send(c1665dec,0,c15d3900,0,0) at 0xc0536da0 = udp_send+0x30 sosend(c1665dec,0,ca422c48,c15d3900,0) at 0xc04d33b1 = sosend+0x701 kern_sendit(c148fa80,d,ca422cc4,0,0) at 0xc04d9bef = kern_sendit+0x13f sendit(c148fa80,d,ca422cc4,0,810001d) at 0xc04d9a81 = sendit+0x1a1 sendto(c148fa80,ca422d14,18,431,6) at 0xc04d9d6b = sendto+0x5b syscall(2f,2f,2f,2,0) at 0xc05d9d40 = syscall+0x2a0 Xint0x80_syscall() at 0xc05c885f = Xint0x80_syscall+0x1f --- syscall (133, FreeBSD ELF32, sendto), eip = 0x28233cff, esp = 0xbfbfd4ec, ebp = 0xbfbfd518 --- followed by another new LOR: KDB: enter: witness_checkorder lock order reversal 1st 0xc066f20c udp (udp) @ /usr/src/sys/netinet/udp_usrreq.c:246 2nd 0xc14c8264 fxp0 (network driver) @ /usr/src/sys/modules/fxp/../../dev/fxp/if_fxp.c:1232 KDB: stack backtrace: kdb_backtrace(c05fd115,c14c8264,c14a57e0,c06fe820,c06fe7bd) at 0xc04b0a7e = kdb_backtrace+0x2e witness_checkorder(c14c8264,9,c06fe7bd,4d0,c0602589) at 0xc04bbb96 = witness_checkorder+0x6a6 _mtx_lock_flags(c14c8264,0,c06fe7bd,4d0,c14c8000) at 0xc048a95a = _mtx_lock_flags+0x8a fxp_start(c14c8000,12b,0,c14c8000) at 0xc06fbdc7 = fxp_start+0x37 if_start(c14c8000,0,c0602589,184,2) at 0xc050b1c9 = if_start+0x99 ether_output_frame(c14c8000,c15d6000,6,c9be5b20,c9be5abc) at 0xc050c908 = ether_output_frame+0x218 ether_output(c14c8000,c15d6000,c9be5b20,c16687bc,18a) at 0xc050c6de = ether_output+0x44e ip_output(c15d6000,0,c9be5b1c,0,0) at 0xc05210c7 = ip_output+0x7c7 icmp_send(c15d6000,0,c05a61a8,0,0) at 0xc051e567 = icmp_send+0x87 icmp_reflect(c15d6000,c15d60c8,14,c15d6100,1c) at 0xc051e490 = icmp_reflect+0x330 icmp_error(c14d1600,3,3,0,0) at 0xc051db85 = icmp_error+0x275 udp_input(c14d1600,14,c9be5c98,c048aa6a,c066e714) at 0xc053547a = udp_input+0x5ea ip_input(c14d1600,0,c0602a22,e6,c066dcd8) at 0xc051ef22 = ip_input+0x5a2 netisr_processqueue(c066dcd8,c0643a80,1,c05f8d46,c1404040) at 0xc050e64e = netisr_processqueue+0x8e swi_net(0,0,c05f72d8,269,c0643a80) at 0xc050e8a9 = swi_net+0xe9 ithread_loop(c13dc500,c9be5d48,c05f70cf,30e,0) at 0xc047dbf2 = ithread_loop+0x172 fork_exit(c047da80,c13dc500,c9be5d48) at 0xc047cc16 = fork_exit+0xc6 fork_trampoline() at 0xc05c886c = fork_trampoline+0x8 --- trap 0x1, eip = 0, esp = 0xc9be5d7c, ebp = 0 --- The box panics with automatic dump and reboot: KDB: enter: witness_checkorder panic: blockable sleep lock (sleep mutex) tty @ /usr/src/sys/kern/kern_event.c:1495 KDB: stack backtrace: Uptime: 15m33s Dumping 191 MB 16 32 48 64 80 96 112 128 144 160 176 Dump complete This is on: FreeBSD 5.4-STABLE #0: Tue Apr 26 07:30:25 CEST 2005 root@82-168-79-254-bbxl.xdsl.tiscali.nl:/usr/obj/usr/src/sys/RENE Regards, Rene -- "It won't fit on the line." -- me, 2001 -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 187 bytes Desc: not available Url : http://lists.freebsd.org/pipermail/freebsd-stable/attachments/20050427/064e4276/attachment.bin
On Wed, Apr 27, 2005 at 08:01:28PM +0200, Rene Ladan wrote:> Hi, > > I experienced this new fxp LOR (probably when ntpd started during boot):[snip 2 LORs]> The box panics with automatic dump and reboot: > > KDB: enter: witness_checkorder > panic: blockable sleep lock (sleep mutex) tty @ /usr/src/sys/kern/kern_event.c:1495 > KDB: stack backtrace: > Uptime: 15m33s > Dumping 191 MB > 16 32 48 64 80 96 112 128 144 160 176 > Dump completeSome dump analysis (especially frames 34 and 35 look interesting): [GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"] GNU gdb 6.1.1 [FreeBSD] Copyright 2004 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "i386-marcel-freebsd". #0 doadump () at pcpu.h:159 159 pcpu.h: No such file or directory. in pcpu.h (kgdb) bt f #0 doadump () at pcpu.h:159 No locals. #1 0xc04948aa in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:410 first_buf_printf = 1 #2 0xc0494c18 in panic ( fmt=0xc05fcf6f "blockable sleep lock (%s) %s @ %s:%d") at /usr/src/sys/kern/kern_shutdown.c:566 td = (struct thread *) 0xc13e7180 bootopt = 256 newpanic = 1 ap = 0xc9be56a8 "{?`???_?~l_??\005" buf = "blockable sleep lock (sleep mutex) tty @ /usr/src/sys/kern/kern_event.c:1495", '\0' <repeats 179 times> #3 0xc04bb5ae in witness_checkorder (lock=0xc158c510, flags=9, file=0xc05f6c7e "/usr/src/sys/kern/kern_event.c", line=1495) at /usr/src/sys/kern/subr_witness.c:709 lock_list = (struct lock_list_entry **) 0x15 lle = (struct lock_list_entry *) 0xc9be56fc lock1 = (struct lock_instance *) 0xc13e7180 lock2 = (struct lock_instance *) 0x0 class = (struct lock_class *) 0xc0620ffc w = (struct witness *) 0xc0649468 w1 = (struct witness *) 0x20 td = (struct thread *) 0xc13e7180 ---Type <return> to continue, or q <return> to quit--- i = 1923 j = -1066895508 __func__ = "witness_checkorder" #4 0xc048a95a in _mtx_lock_flags (m=0xc158c510, opts=0, file=0xc05f6c7e "/usr/src/sys/kern/kern_event.c", line=1495) at /usr/src/sys/kern/kern_mutex.c:271 No locals. #5 0xc0476ff5 in knote (list=0xc158c498, hint=0, islocked=0) at /usr/src/sys/kern/kern_event.c:1495 kq = (struct kqueue *) 0xc158c438 kn = (struct knote *) 0xc158c400 #6 0xc04c98ae in ttwwakeup (tp=0xc158c400) at /usr/src/sys/kern/tty.c:2398 No locals. #7 0xc05baf21 in scstart (tp=0xc158c400) at /usr/src/sys/dev/syscons/syscons.c:1369 rbp = (struct clist *) 0xc158c438 len = 0 buf = "lwh?\204\a\000\000\000p\000\000`wh??W??Bk[?lwh?\204\a\000\000 \000\000\000\000p\000\000BK\000 \224W??\032\205[? ?h?lwh?`wh?\000\017\000\000?W??ul[?`wh?\204\a\000\000 \000\000\000\000\017\000\000\000\000\000\000`wh?`wh? ?h??W????[?`wh?\204\a\000\000\000\000\000" scp = (scr_stat *) 0xc0687760 #8 0xc05be305 in scgetc (sc=0xc068aa20, flags=3) at /usr/src/sys/dev/syscons/syscons.c:3211 ---Type <return> to continue, or q <return> to quit--- scp = (scr_stat *) 0xc0687760 tp = (struct tty *) 0x0 c = 6 this_scr = -910272488 f = 0 i = 0 #9 0xc05bb379 in sccngetch (flags=2) at /usr/src/sys/dev/syscons/syscons.c:1555 fkey = {str = '\0' <repeats 15 times>, len = 0 '\0'} fkeycp = 0 scp = (scr_stat *) 0xc0687760 p = (u_char *) 0x0 cur_mode = 1 c = -1067201760 #10 0xc05bb1c2 in sccncheckc (cd=0xc06350e0) at /usr/src/sys/dev/syscons/syscons.c:1478 No locals. #11 0xc04cc1f8 in cncheckc () at /usr/src/sys/kern/tty_cons.c:567 cnd = (struct cn_device *) 0xc066d0e0 cn = (struct consdev *) 0x0 c = 0 #12 0xc04cc1a5 in cngetc () at /usr/src/sys/kern/tty_cons.c:548 c = 0 #13 0xc042a765 in db_readline (lstart=0xc063cb20 "c\n", lsize=120) ---Type <return> to continue, or q <return> to quit--- at /usr/src/sys/ddb/db_input.c:324 No locals. #14 0xc042a8aa in db_read_line () at /usr/src/sys/ddb/db_lex.c:55 i = 0 #15 0xc0428fc1 in db_command_loop () at /usr/src/sys/ddb/db_command.c:453 No locals. #16 0xc042b125 in db_trap (type=3, code=0) at /usr/src/sys/ddb/db_main.c:221 jb = {{_jb = {-910272284, -910272312, -910272232, -1052872320, 0, -1069371194, 0, 0, 0, 0, -910272232, -1068823136}}} prev_jb = (void *) 0x0 bkpt = 0 #17 0xc04b0e27 in kdb_trap (type=0, code=0, tf=0xc9be5978) at /usr/src/sys/kern/subr_kdb.c:418 handled = -910272136 #18 0xc05d9428 in trap (frame {tf_fs = -1051983848, tf_es = 16, tf_ds = -910295024, tf_edi = 9, tf_esi = -1051950492, tf_ebp = -910272064, tf_isp = -910272092, tf_ebx = -1067004164, tf_edx = 1, tf_ecx = -1056878592, tf_eax = 31, tf_trapno = 3, tf_err = 0, tf_eip = -1068823776, tf_cs = 8, tf_eflags = 646, tf_esp = -1067468212, tf_ss = -1067537639}) at /usr/src/sys/i386/i386/trap.c:576 td = (struct thread *) 0xc13e7180 p = (struct proc *) 0xc13e654c sticks = 0 i = 0 ---Type <return> to continue, or q <return> to quit--- ucode = 0 type = 3 code = 0 eva = 0 #19 0xc05c880a in calltrap () at /usr/src/sys/i386/i386/exception.s:140 No locals. #20 0xc14c0018 in ?? () No symbol table info available. #21 0x00000010 in ?? () No symbol table info available. #22 0xc9be0010 in ?? () No symbol table info available. #23 0x00000009 in ?? () No symbol table info available. #24 0xc14c8264 in ?? () No symbol table info available. #25 0xc9be59c0 in ?? () No symbol table info available. #26 0xc9be59a4 in ?? () No symbol table info available. #27 0xc066cefc in w_locklistdata () No symbol table info available. #28 0x00000001 in ?? () No symbol table info available. ---Type <return> to continue, or q <return> to quit--- #29 0xc1015000 in ?? () No symbol table info available. #30 0x0000001f in ?? () No symbol table info available. #31 0x00000003 in ?? () No symbol table info available. #32 0x00000000 in ?? () No symbol table info available. #33 0xc04b0b20 in kdb_enter (msg=0x0) at cpufunc.h:56 No locals. #34 0xc04bbbab in witness_checkorder (lock=0xc14c8264, flags=9, file=0xc06fe7bd "/usr/src/sys/modules/fxp/../../dev/fxp/if_fxp.c", line=1232) at /usr/src/sys/kern/subr_witness.c:946 lock_list = (struct lock_list_entry **) 0xc13e71f0 lle = (struct lock_list_entry *) 0xc066cef8 lock1 = (struct lock_instance *) 0xc066cefc lock2 = (struct lock_instance *) 0x0 class = (struct lock_class *) 0xc0620ffc w = (struct witness *) 0xc0649530 w1 = (struct witness *) 0xc064a778 td = (struct thread *) 0xc066cefc i = -1 j = 0 __func__ = "witness_checkorder" ---Type <return> to continue, or q <return> to quit--- #35 0xc048a95a in _mtx_lock_flags (m=0xc14c8264, opts=0, file=0xc06fe7bd "/usr/src/sys/modules/fxp/../../dev/fxp/if_fxp.c", line=1232) at /usr/src/sys/kern/kern_mutex.c:271 No locals. #36 0xc06fbdc7 in ?? () No symbol table info available. #37 0xc14c8264 in ?? () No symbol table info available. #38 0x00000000 in ?? () No symbol table info available. #39 0xc06fe7bd in ?? () No symbol table info available. #40 0x000004d0 in ?? () No symbol table info available. #41 0xc14c8000 in ?? () No symbol table info available. #42 0xc14c8000 in ?? () No symbol table info available. #43 0xc9be5a70 in ?? () No symbol table info available. #44 0xc050b1c9 in if_start (ifp=0xc14c8264) at /usr/src/sys/net/if.c:1959 No locals. Previous frame identical to this frame (corrupt stack?) (kgdb) frame 33 #33 0xc04b0b20 in kdb_enter (msg=0x0) at cpufunc.h:56 56 cpufunc.h: No such file or directory. in cpufunc.h (kgdb) frame 34 #34 0xc04bbbab in witness_checkorder (lock=0xc14c8264, flags=9, file=0xc06fe7bd "/usr/src/sys/modules/fxp/../../dev/fxp/if_fxp.c", line=1232) at /usr/src/sys/kern/subr_witness.c:946 946 kdb_enter(__func__); (kgdb) frame 35 #35 0xc048a95a in _mtx_lock_flags (m=0xc14c8264, opts=0, file=0xc06fe7bd "/usr/src/sys/modules/fxp/../../dev/fxp/if_fxp.c", line=1232) at /usr/src/sys/kern/kern_mutex.c:271 271 WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, (kgdb) print *m $1 = {mtx_object = {lo_class = 0xc0620ffc, lo_name = 0xc14a57e0 "fxp0", lo_type = 0xc06fe820 "network driver", lo_flags = 196608, lo_list = { tqe_next = 0xc14c8208, tqe_prev = 0xc14bf010}, lo_witness = 0xc0649530}, mtx_lock = 4, mtx_recurse = 0} (kgdb) q I still have this dump (~65MB uncompressed) available.> This is on: > > FreeBSD 5.4-STABLE #0: Tue Apr 26 07:30:25 CEST 2005 root@82-168-79-254-bbxl.xdsl.tiscali.nl:/usr/obj/usr/src/sys/RENE >Regards, Rene -- "It won't fit on the line." -- me, 2001 -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 187 bytes Desc: not available Url : http://lists.freebsd.org/pipermail/freebsd-stable/attachments/20050428/7784e445/attachment.bin