Volker
2007-Aug-26 03:01 UTC
some LORs + non-sleepable locks on recent stable i386 causing freezes
Hi! While fighting machine freezes with a recent kernel + world, I catched the following witness messages: (this one looks similar to LOR #199): lock order reversal: (sleepable after non-sleepable) 1st 0xc653aec0 ath0 (ath0) @ /usr/src/sys/modules/ath/../../dev/ath/if_ath.c:5210 2nd 0xc63b4cfc user map (user map) @ /usr/src/sys/vm/vm_map.c:3074 KDB: stack backtrace: kdb_backtrace(c07c6a1f,c63b4cfc,c07d9630,c07d9630,c07d9675,...) at kdb_backtrace+0x2e witness_checkorder(c63b4cfc,9,c07d9675,c02,8061000,...) at witness_checkorder+0x5ef _sx_xlock(c63b4cfc,c07d9675,c02,c089fa00,eed329ac,...) at _sx_xlock+0x4d _vm_map_lock_read(c63b4cb8,c07d9675,c02,29a02b7,8061000,...) at _vm_map_lock_read+0x4a vm_map_lookup(eed32a44,8061000,2,eed32a48,eed32a38,...) at vm_map_lookup+0x38 vm_fault(c63b4cb8,8061000,2,8,c6567300,...) at vm_fault+0x7e trap_pfault(eed32b2c,0,8061000,c07c66cb,8061000,...) at trap_pfault+0x11a trap(8,28,28,8061000,c66ea800,...) at trap+0x37d calltrap() at calltrap+0x5 --- trap 0xc, eip = 0xc0768664, esp = 0xeed32b6c, ebp = 0xeed32ba0 --- slow_copyout(c64c1800,c66ff080,c653ab30,c0286938,0,...) at slow_copyout+0x4 ieee80211_ioctl(c653a230,c0286938,c66ff080,145a,c66ff080,...) at ieee80211_ioctl+0xcb ath_ioctl(c64c1800,c0286938,c66ff080,8,c07cb304,...) at ath_ioctl+0x1c6 ifhwioctl(c0286938,c64c1800,c66ff080,c6567300,8c,...) at ifhwioctl+0xacd ifioctl(c685d590,c0286938,c66ff080,c6567300,2,...) at ifioctl+0xe1 soo_ioctl(c675ddc8,c0286938,c66ff080,c63aad80,c6567300,...) at soo_ioctl+0x3ef ioctl(c6567300,eed32d04,c,c07abc38,3,...) at ioctl+0x44d syscall(3b,3b,3b,8060028,1e,...) at syscall+0x2c0 Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (54, FreeBSD ELF32, ioctl), eip = 0x281681c3, esp 0xbfbfe79c, ebp = 0xbfbfe7b8 --- This one seems to be a new (undocumented) one: lock order reversal: (Giant after non-sleepable) 1st 0xc08a3220 in_multi_mtx (in_multi_mtx) @ /usr/src/sys/netinet/in.c:972 2nd 0xc0854680 Giant (Giant) @ /usr/src/sys/kern/kern_synch.c:218 KDB: stack backtrace: kdb_backtrace(c07c6a1f,c0854680,c07c3547,c07c3547,c07c427f,...) at kdb_backtrace+0x2e witness_checkorder(c0854680,9,c07c427f,da,c0854640,...) at witness_checkorder+0x5ef _mtx_lock_flags(c0854680,0,c07c427f,da,0,...) at _mtx_lock_flags+0x32 msleep(c6727100,0,4c,c07b9a84,0,...) at msleep+0x324 usbd_transfer(c6727100,eed2ca8c,c054ec6d,c6727100,c657f900,...) at usbd_transfer+0x16c usbd_sync_transfer(c6727100,c657f900,0,1388,eed2cae8,...) at usbd_sync_transfer+0x1c usbd_do_request_flags_pipe(c657f900,c657f880,eed2cae8,eed2cae7,0,...) at usbd_do_request_flags_pipe+0x7d usbd_do_request_flags(c657f900,eed2cae8,eed2cae7,0,0,...) at usbd_do_request_flags+0x3c usbd_do_request(c657f900,eed2cae8,eed2cae7,c85d1fb0,f0c0,...) at usbd_do_request+0x37 aue_csr_read_1(c6580200,0,c089fa00,c0854680,0,...) at aue_csr_read_1+0x58 aue_setmulti(c6580200,c07cb304,81f,81f,c682f060,...) at aue_setmulti+0x65 aue_ioctl(c657b000,80206931,0,81f,c,...) at aue_ioctl+0x138 if_addmulti(c657b000,eed2cb88,eed2cb84,3cc,c682f060,...) at if_addmulti+0x239 in_addmulti(eed2cbc4,c657b000,c07cefc7,2c6,1,...) at in_addmulti+0x97 in_ifinit(c657b000,c6830000,c671c310,0,eed2cc20,...) at in_ifinit+0x51b in_control(c685d6f4,8040691a,c671c300,c657b000,c6567000,...) at in_control+0x95e ifioctl(c685d6f4,8040691a,c671c300,c6567000,2,...) at ifioctl+0x1bc soo_ioctl(c675dbd0,8040691a,c671c300,c63aad80,c6567000,...) at soo_ioctl+0x3ef ioctl(c6567000,eed2cd04,c,c07e3c99,3,...) at ioctl+0x44d syscall(3b,3b,3b,80595c0,0,...) at syscall+0x2c0 Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (54, FreeBSD ELF32, ioctl), eip = 0x281681c3, esp 0xbfbfe9ac, ebp = 0xbfbfe9d8 --- and finally a non-sleepable lock: Sleeping on "usbsyn" with the following non-sleepable locks held: exclusive sleep mutex in_multi_mtx r = 0 (0xc08a3220) locked @ /usr/src/sys/netinet/in.c:972 KDB: stack backtrace: kdb_backtrace(c089fa14,eed2c9e8,1,c654be40,1,...) at kdb_backtrace+0x2e witness_warn(5,0,c07c426e,c07b9a84,c6727100,...) at witness_warn+0x1d3 msleep(c6727100,0,4c,c07b9a84,0,...) at msleep+0x62 usbd_transfer(c6727100,eed2ca8c,c054ec6d,c6727100,c657f900,...) at usbd_transfer+0x16c usbd_sync_transfer(c6727100,c657f900,0,1388,eed2cae8,...) at usbd_sync_transfer+0x1c usbd_do_request_flags_pipe(c657f900,c657f880,eed2cae8,eed2cae7,0,...) at usbd_do_request_flags_pipe+0x7d usbd_do_request_flags(c657f900,eed2cae8,eed2cae7,0,0,...) at usbd_do_request_flags+0x3c usbd_do_request(c657f900,eed2cae8,eed2cae7,5d1fb0,f0c0,...) at usbd_do_request+0x37 aue_csr_read_1(c6580200,0,c089fa00,c0854680,0,...) at aue_csr_read_1+0x58 aue_setmulti(c6580200,c07cb304,81f,81f,c682f060,...) at aue_setmulti+0x65 aue_ioctl(c657b000,80206931,0,81f,c,...) at aue_ioctl+0x138 if_addmulti(c657b000,eed2cb88,eed2cb84,3cc,c682f060,...) at if_addmulti+0x239 in_addmulti(eed2cbc4,c657b000,c07cefc7,2c6,1,...) at in_addmulti+0x97 in_ifinit(c657b000,c6830000,c671c310,0,eed2cc20,...) at in_ifinit+0x51b in_control(c685d6f4,8040691a,c671c300,c657b000,c6567000,...) at in_control+0x95e ifioctl(c685d6f4,8040691a,c671c300,c6567000,2,...) at ifioctl+0x1bc soo_ioctl(c675dbd0,8040691a,c671c300,c63aad80,c6567000,...) at soo_ioctl+0x3ef ioctl(c6567000,eed2cd04,c,c07e3c99,3,...) at ioctl+0x44d syscall(3b,3b,3b,80595c0,0,...) at syscall+0x2c0 Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (54, FreeBSD ELF32, ioctl), eip = 0x281681c3, esp 0xbfbfe9ac, ebp = 0xbfbfe9d8 --- %uname -v FreeBSD 6.2-STABLE #1: Sun Aug 26 01:36:20 CEST 2007 Kernel sources sync'ed and compiled yesterday. One of these three (or all together) is causing machine freezes when running w/o witness. The problem has been introduced sometime within the last 6-8 weeks (never had freezes before on that machine running 6-STABLE). Volker