Jason Thomson
2007-Feb-22 19:16 UTC
Help debugging non-sleepable lock panic on 6.2-RELEASE
This happens when we put our DELL PE2950 running MySQL 4.0.27 under load.
We have a kernel core dump, but I'm unsure what to do next.
It looks like the process triggering the problem is mysqld.
mysqld is compiled from ports with no switches, and the threading
library is set to libthr using libmap.conf
We can usually reproduce this, so if there's anything to try, please
let me know.
Thanks in advance,
Jason.
panic: _sx_xlock (user map): xlock already held @
/usr/src/sys/vm/vm_map.c:3074
cpuid = 0
Uptime: 5h31m9s
Dumping 8191 MB (3 chunks)
chunk 0: 1MB (156 pages) ... ok
chunk 1: 3327MB (851624 pages) 3311 <snip> 15 ... ok
chunk 2: 4864MB (1245184 pages) 4849 <snip> 1
#0 doadump () at pcpu.h:172
172 pcpu.h: No such file or directory.
in pcpu.h
(kgdb) thread
[Current thread is 129 (Thread 100339)]
(kgdb) bt
#0 doadump () at pcpu.h:172
#1 0xffffffff8027a479 in boot (howto=260) at
/usr/src/sys/kern/kern_shutdown.c:409
#2 0xffffffff80279f0b in panic (fmt=0xffffffff8042bf78 "%s (%s): xlock
already held @ %s:%d") at /usr/src/sys/kern/kern_shutdown.c:565
#3 0xffffffff80280796 in _sx_xlock (sx=0xffffff01b188b8f0,
file=0xffffffff804425c3 "/usr/src/sys/vm/vm_map.c", line=3074) at
/usr/src/sys/kern/kern_sx.c:168
#4 0xffffffff803921c4 in vm_map_lookup (var_map=0xffffffffb9785740,
vaddr=0, fault_typea=1 '\001', out_entry=0xffffffffb9785748,
object=0xffffffffb9785730,
pindex=0xffffffffb9785738, out_prot=0x0, wired=0xffffffffb978570c)
at /usr/src/sys/vm/vm_map.c:3074
#5 0xffffffff8038a47a in vm_fault (map=0xffffff01b188b880, vaddr=0,
fault_type=1 '\001', fault_flags=0) at /usr/src/sys/vm/vm_fault.c:235
#6 0xffffffff803bb5ac in trap_pfault (frame=0xffffffffb97858d0,
usermode=0) at /usr/src/sys/amd64/amd64/trap.c:550
#7 0xffffffff803bb899 in trap (frame {tf_rdi = -2141426760, tf_rsi =
-1092610754784, tf_rdx =
-1092610754784, tf_rcx = 4, tf_r8 = -2143114792, tf_r9 = -1092610754784,
tf_rax = 1, tf_rbx = -1183295072, tf_rbp = -1183295008, tf_r10 = 0,
tf_r11 = 4, tf_r12 = -1092471900528, tf_r13 = 2, tf_r14 =
-1092238133120, tf_r15 = 8782208, tf_trapno = 12, tf_addr = 338,
tf_flags = -2144922467, tf_err = 0, tf_rip = -2143739452, tf_cs = 8,
tf_rflags = 66050, tf_rsp = -1183295088, tf_ss = 16}) at
/usr/src/sys/amd64/amd64/trap.c:352
#8 0xffffffff803a6edb in calltrap () at
/usr/src/sys/amd64/amd64/exception.S:168
#9 0xffffffff803921c4 in vm_map_lookup (var_map=0xffffffffb9785a08,
vaddr=8782208, fault_typea=2 '\002', out_entry=0xffffffffb9785a10,
object=0xffffff01a399b698,
pindex=0xffffffffb9785a18, out_prot=0x1 <Address 0x1 out of
bounds>, wired=0xffffffffb9785a24) at /usr/src/sys/vm/vm_map.c:3074
#10 0xffffffff80288db9 in umtx_key_get (td=0xffffff019b52f720,
umtx=0x860180, key=0xffffff01a399b690) at /usr/src/sys/kern/kern_umtx.c:312
#11 0xffffffff80288eb5 in _do_lock (td=0xffffff019b52f720,
umtx=0x860180, id=100339, timo=0) at /usr/src/sys/kern/kern_umtx.c:362
#12 0xffffffff8028a2df in _umtx_op (td=0xffffff019b52f720, uap=0x187f3)
at /usr/src/sys/kern/kern_umtx.c:545
#13 0xffffffff803bc022 in syscall (frame {tf_rdi = 8782208, tf_rsi = 0,
tf_rdx = 100339, tf_rcx = 0, tf_r8
= 0, tf_r9 = 1, tf_rax = 454, tf_rbx = 100339, tf_rbp = 8782208, tf_r10
= 0, tf_r11 = 1494439248, tf_r12 = 8765008, tf_r13 = 1024, tf_r14 = 0,
tf_r15 = 0, tf_trapno = 22, tf_addr = 0, tf_flags = 8, tf_err = 2,
tf_rip = 34377391676, tf_cs = 43, tf_rflags = 582, tf_rsp =
140737479959448, tf_ss = 35}) at /usr/src/sys/amd64/amd64/trap.c:792
#14 0xffffffff803a7078 in Xfast_syscall () at
/usr/src/sys/amd64/amd64/exception.S:270
#15 0x00000008010d5e3c in ?? ()
(kgdb) p *td
$1 = {td_proc = 0xffffff01b14f8358, td_ksegrp = 0xffffff011f741090,
td_plist = {tqe_next = 0xffffff0072234000, tqe_prev =
0xffffff017bd61010}, td_kglist = {tqe_next = 0x0,
tqe_prev = 0xffffff011f7410a8}, td_slpq = {tqe_next = 0x0, tqe_prev
= 0xffffff0000dd9cc0}, td_lockq = {tqe_next = 0xffffff002ff9a260,
tqe_prev = 0xffffffffb99eca10},
td_runq = {tqe_next = 0x0, tqe_prev = 0xffffff011f7410b8}, td_selq =
{tqh_first = 0x0, tqh_last = 0x0}, td_sleepqueue = 0xffffff0000dd9cc0,
td_turnstile = 0xffffff0072b3d800,
td_umtxq = 0xffffff01a399b680, td_tid = 100339, td_flags = 83951618,
td_inhibitors = 0, td_pflags = 0, td_dupfd = 0, td_wchan = 0x0, td_wmesg
= 0x0, td_lastcpu = 0 '\0',
td_oncpu = 0 '\0', td_owepreempt = 0 '\0', td_locks = 2,
td_blocked =
0x0, td_ithd = 0x0, td_lockname = 0x0, td_contested = {lh_first = 0x0},
td_sleeplocks = 0x0,
td_intr_nesting_level = 0, td_pinned = 0, td_mailbox = 0x0, td_ucred
= 0xffffff01fda65500, td_standin = 0x0, td_upcall = 0x0, td_sticks =
3502, td_uuticks = 0, td_usticks = 0,
td_intrval = 0, td_oldsigmask = {__bits = {0, 0, 0, 0}}, td_sigmask =
{__bits = {159751, 0, 0, 0}}, td_siglist = {__bits = {0, 0, 0, 0}},
td_generation = 2243687, td_sigstk = {
ss_sp = 0x0, ss_size = 0, ss_flags = 0}, td_kflags = 0, td_xsig =
0, td_profil_addr = 0, td_profil_ticks = 0, td_base_pri = 199 '?',
td_priority = 199 '?',
td_pcb = 0xffffffffb9785d10, td_state = TDS_RUNNING, td_retval = {0,
100339}, td_slpcallout = {c_links = {sle = {sle_next = 0x0}, tqe =
{tqe_next = 0x0, tqe_prev = 0x0}},
c_time = 0, c_arg = 0x0, c_func = 0, c_mtx = 0x0, c_flags = 16},
td_frame = 0xffffffffb9785c40, td_kstack_obj = 0xffffff01b59061c0,
td_kstack = 18446744072526241792,
td_kstack_pages = 4, td_altkstack_obj = 0x0, td_altkstack = 0,
td_altkstack_pages = 0, td_critnest = 1, td_md = {md_spinlock_count = 1,
md_saved_flags = 582},
td_sched = 0xffffff019b52f950, td_ar = 0x0}
(kgdb) p *td->td_proc
$2 = {p_list = {le_next = 0xffffff0072cc8358, le_prev =
0xffffff01b2dd2a08}, p_ksegrps = {tqh_first = 0xffffff021ec355a0,
tqh_last = 0xffffff01b13755a8}, p_threads = {
tqh_first = 0xffffff017aef2be0, tqh_last = 0xffffff01b14324d0},
p_suspended = {tqh_first = 0x0, tqh_last = 0xffffff01b14f8388}, p_ucred
= 0xffffff01fda65500,
p_fd = 0xffffff018a0c2400, p_fdtol = 0x0, p_stats =
0xffffff01b444dc00, p_limit = 0xffffff000a8e9700, p_sigacts =
0xffffff002d527000, p_flag = 16514, p_sflag = 1,
p_state = PRS_NORMAL, p_pid = 49461, p_hash = {le_next = 0x0, le_prev
= 0xffffffff8b9279a8}, p_pglist = {le_next = 0x0, le_prev =
0xffffff0072cc83e8},
p_pptr = 0xffffff0072cc8358, p_sibling = {le_next = 0x0, le_prev =
0xffffff0072cc8410}, p_children = {lh_first = 0x0}, p_mtx = {mtx_object
= {lo_class = 0xffffffff8058f260,
lo_name = 0xffffffff8042a96e "process lock", lo_type =
0xffffffff8042a96e "process lock", lo_flags = 4390912, lo_list =
{tqe_next = 0x0, tqe_prev = 0x0}, lo_witness = 0x0},
mtx_lock = 4, mtx_recurse = 0}, p_oppid = 0, p_vmspace =
0xffffff01b188b880, p_swtime = 2321, p_realtimer = {it_interval =
{tv_sec = 0, tv_usec = 0}, it_value = {
tv_sec = 48463, tv_usec = 184910}}, p_rux = {rux_runtime = {sec =
7678, frac = 12488269182194442856}, rux_uticks = 11112, rux_sticks =
86876, rux_iticks = 0,
rux_uu = 812236325, rux_su = 5875186825, rux_iu = 1}, p_crux =
{rux_runtime = {sec = 0, frac = 0}, rux_uticks = 0, rux_sticks = 0,
rux_iticks = 0, rux_uu = 0, rux_su = 0,
rux_iu = 0}, p_profthreads = 0, p_maxthrwaits = 0, p_traceflag = 0,
p_tracevp = 0x0, p_tracecred = 0x0, p_textvp = 0xffffff01ba0bf7c0,
p_siglist = {__bits = {0, 0, 0, 0}},
p_lock = 1 '\001', p_sigiolst = {slh_first = 0x0}, p_sigparent = 20,
p_sig = 0, p_code = 0, p_stops = 0, p_stype = 0, p_step = 0 '\0',
p_pfsflags = 0 '\0', p_nlminfo = 0x0,
p_aioinfo = 0x0, p_singlethread = 0x0, p_suspcount = 0, p_xthread =
0x0, p_boundary_count = 0, p_procscopegrp = 0xffffff01b13755a0, p_magic
= 3203398350,
p_comm = "mysqld", '\0' <repeats 13 times>, p_pgrp =
0xffffff012c296c80, p_sysent = 0xffffffff805ab4e0, p_args =
0xffffff0133157100, p_cpulimit = 9223372036854775807,
p_nice = 0 '\0', p_xstat = 0, p_klist = {kl_list = {slh_first = 0x0},
kl_lock = 0xffffffff8025b500 <knlist_mtx_lock>, kl_unlock =
0xffffffff8025b520 <knlist_mtx_unlock>,
kl_locked = 0xffffffff8025b540 <knlist_mtx_locked>, kl_lockarg =
0xffffff01b14f8418}, p_numthreads = 41, p_numksegrps = 41, p_md =
<incomplete type>, p_itcallout = {
c_links = {sle = {sle_next = 0x0}, tqe = {tqe_next = 0x0, tqe_prev
= 0xffffffffa4a85450}}, c_time = 4783675, c_arg = 0xffffff01b14f8358,
c_func = 0xffffffff80287800 <realitexpire>, c_mtx = 0x0, c_flags =
22}, p_acflag = 0, p_ru = 0x0, p_peers = 0x0, p_leader =
0xffffff01b14f8358, p_emuldata = 0x0,
p_label = 0x0, p_sched = 0xffffff01b14f86b0, p_au = 0x0, p_ktr =
{stqh_first = 0x0, stqh_last = 0xffffff01b14f86a0}}
Dmesg.
Copyright (c) 1992-2007 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 6.2-RELEASE #3: Thu Feb 22 11:13:19 GMT 2007
root@roley.mintel.co.uk:/usr/obj/usr/src/sys/PE2950
ACPI APIC Table: <DELL PE_SC3 >
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) Xeon(TM) CPU 3.73GHz (3725.14-MHz K8-class CPU)
Origin = "GenuineIntel" Id = 0xf64 Stepping = 4
Features=0xbfebfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM,PBE>
Features2=0xe4bd<SSE3,RSVD2,MON,DS_CPL,VMX,EST,CNTX-ID,CX16,<b14>,<b15>>
AMD Features=0x20100800<SYSCALL,NX,LM>
AMD Features2=0x1<LAHF>
Cores per package: 2
Logical CPUs per core: 2
real memory = 9395240960 (8960 MB)
avail memory = 8295198720 (7910 MB)
FreeBSD/SMP: Multiprocessor System Detected: 8 CPUs
cpu0 (BSP): APIC ID: 0
cpu1 (AP): APIC ID: 1
cpu2 (AP): APIC ID: 2
cpu3 (AP): APIC ID: 3
cpu4 (AP): APIC ID: 4
cpu5 (AP): APIC ID: 5
cpu6 (AP): APIC ID: 6
cpu7 (AP): APIC ID: 7
ioapic0: Changing APIC ID to 8
ioapic1: Changing APIC ID to 9
ioapic1: WARNING: intbase 64 != expected base 24
ioapic0 <Version 2.0> irqs 0-23 on motherboard
ioapic1 <Version 2.0> irqs 64-87 on motherboard
kbd0 at kbdmux0
acpi0: <DELL PE_SC3> on motherboard
acpi0: Power Button (fixed)
Timecounter "ACPI-fast" frequency 3579545 Hz quality 1000
acpi_timer0: <24-bit timer at 3.579545MHz> port 0x808-0x80b on acpi0
cpu0: <ACPI CPU> on acpi0
cpu1: <ACPI CPU> on acpi0
cpu2: <ACPI CPU> on acpi0
cpu3: <ACPI CPU> on acpi0
cpu4: <ACPI CPU> on acpi0
cpu5: <ACPI CPU> on acpi0
cpu6: <ACPI CPU> on acpi0
cpu7: <ACPI CPU> on acpi0
pcib0: <ACPI Host-PCI bridge> port 0xcf8-0xcff on acpi0
pci0: <ACPI PCI bus> on pcib0
pcib1: <ACPI PCI-PCI bridge> at device 2.0 on pci0
pci6: <ACPI PCI bus> on pcib1
pcib2: <ACPI PCI-PCI bridge> at device 0.0 on pci6
pci7: <ACPI PCI bus> on pcib2
pcib3: <ACPI PCI-PCI bridge> at device 0.0 on pci7
pci8: <ACPI PCI bus> on pcib3
pcib4: <PCI-PCI bridge> at device 0.0 on pci8
pci9: <PCI bus> on pcib4
bce0: <Broadcom NetXtreme II BCM5708 1000Base-T (B1), v0.9.6> mem
0xf4000000-0xf5ffffff irq 16 at device 0.0 on pci9
bce0: ASIC ID 0x57081010; Revision (B1); PCI-X 64-bit 133MHz
miibus0: <MII bus> on bce0
brgphy0: <BCM5708C 10/100/1000baseTX PHY> on miibus0
brgphy0: 10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, 1000baseTX,
1000baseTX-FDX, auto
bce0: Ethernet address: 00:13:72:64:06:de
pcib5: <ACPI PCI-PCI bridge> at device 1.0 on pci7
pci10: <ACPI PCI bus> on pcib5
pcib6: <PCI-PCI bridge> at device 0.3 on pci6
pci11: <PCI bus> on pcib6
pcib7: <ACPI PCI-PCI bridge> at device 3.0 on pci0
pci1: <ACPI PCI bus> on pcib7
pcib8: <ACPI PCI-PCI bridge> at device 0.0 on pci1
pci2: <ACPI PCI bus> on pcib8
mfi0: <Dell PERC 5/i> mem 0xd80f0000-0xd80fffff,0xfc4e0000-0xfc4fffff
irq 78 at device 14.0 on pci2
mfi0: 2438 (225460047s/0x0020/0) - Shutdown command received from host
mfi0: 2439 (4278190080s/0x0020/0) - PCI 0x041028 0x0415 0x041028
0x041f03: Firmware initialization started (PCI ID 0015/1028/1f03/1028)
mfi0: 2440 (4278190080s/0x0020/0) - Type 18: Firmware version 1.00.01-0088
mfi0: 2441 (4278190117s/0x0008/0) - Battery Present
mfi0: 2442 (4278190145s/0x0004/0) - PD 08(e1/s255) event: Enclosure
(SES) discovered on PD 08(e1/s255)
mfi0: 2443 (4278190145s/0x0002/0) - PD 08(e1/s255) event: Inserted: PD
08(e1/s255)
mfi0: 2444 (4278190145s/0x0002/0) - Type 29: Inserted: PD 08(e1/s255)
Info: enclPd=08, scsiType=d, portMap=00,
sasAddr=500130205da27900,0000000000000000
mfi0: 2445 (4278190145s/0x0002/0) - PD 00(e1/s0) event: Inserted: PD
00(e1/s0)
mfi0: 2446 (4278190145s/0x0002/0) - Type 29: Inserted: PD 00(e1/s0)
Info: enclPd=08, scsiType=0, portMap=01,
sasAddr=50010b90001ec97e,0000000000000000
mfi0: 2447 (4278190145s/0x0002/0) - PD 01(e1/s1) event: Inserted: PD
01(e1/s1)
mfi0: 2448 (4278190145s/0x0002/0) - Type 29: Inserted: PD 01(e1/s1)
Info: enclPd=08, scsiType=0, portMap=02,
sasAddr=50010b90001ec41a,0000000000000000
mfi0: 2449 (4278190145s/0x0002/0) - PD 02(e1/s2) event: Inserted: PD
02(e1/s2)
mfi0: 2450 (4278190145s/0x0002/0) - Type 29: Inserted: PD 02(e1/s2)
Info: enclPd=08, scsiType=0, portMap=04,
sasAddr=50010b90001ec9c2,0000000000000000
mfi0: 2451 (4278190145s/0x0002/0) - PD 03(e1/s3) event: Inserted: PD
03(e1/s3)
mfi0: 2452 (4278190145s/0x0002/0) - Type 29: Inserted: PD 03(e1/s3)
Info: enclPd=08, scsiType=0, portMap=08,
sasAddr=50010b90001ec9ae,0000000000000000
mfi0: 2453 (225460132s/0x0020/0) - Adapter ticks 225460132 elapsed 66s:
Time established as 02/22/07 11:48:52; (66 seconds since power on)
mfi0: 2454 (225460188s/0x0008/0) - Battery temperature is normal
mfi0: 2455 (225460188s/0x0008/0) - Current capacity of the battery is
above threshold
mfi0: 2456 (4278190080s/0x0020/0) - PCI 0x041028 0x0415 0x041028
0x041f03: Firmware initialization started (PCI ID 0015/1028/1f03/1028)
mfi0: 2457 (4278190080s/0x0020/0) - Type 18: Firmware version 1.00.01-0088
mfi0: 2458 (4278190130s/0x0008/0) - Battery Present
mfi0: 2459 (4278190158s/0x0004/0) - PD 08(e1/s255) event: Enclosure
(SES) discovered on PD 08(e1/s255)
mfi0: 2460 (4278190158s/0x0002/0) - PD 08(e1/s255) event: Inserted: PD
08(e1/s255)
mfi0: 2461 (4278190158s/0x0002/0) - Type 29: Inserted: PD 08(e1/s255)
Info: enclPd=08, scsiType=d, portMap=00,
sasAddr=500130205da27900,0000000000000000
mfi0: 2462 (4278190158s/0x0002/0) - PD 00(e1/s0) event: Inserted: PD
00(e1/s0)
mfi0: 2463 (4278190158s/0x0002/0) - Type 29: Inserted: PD 00(e1/s0)
Info: enclPd=08, scsiType=0, portMap=01,
sasAddr=50010b90001ec97e,0000000000000000
mfi0: 2464 (4278190158s/0x0002/0) - PD 01(e1/s1) event: Inserted: PD
01(e1/s1)
mfi0: 2465 (4278190158s/0x0002/0) - Type 29: Inserted: PD 01(e1/s1)
Info: enclPd=08, scsiType=0, portMap=02,
sasAddr=50010b90001ec41a,0000000000000000
mfi0: 2466 (4278190158s/0x0002/0) - PD 02(e1/s2) event: Inserted: PD
02(e1/s2)
mfi0: 2467 (4278190158s/0x0002/0) - Type 29: Inserted: PD 02(e1/s2)
Info: enclPd=08, scsiType=0, portMap=04,
sasAddr=50010b90001ec9c2,0000000000000000
mfi0: 2468 (4278190158s/0x0002/0) - PD 03(e1/s3) event: Inserted: PD
03(e1/s3)
mfi0: 2469 (4278190158s/0x0002/0) - Type 29: Inserted: PD 03(e1/s3)
Info: enclPd=08, scsiType=0, portMap=08,
sasAddr=50010b90001ec9ae,0000000000000000
mfi0: 2470 (225481183s/0x0020/0) - Adapter ticks 225481183 elapsed 79s:
Time established as 02/22/07 17:39:43; (79 seconds since power on)
mfi0: 2471 (225481239s/0x0008/0) - Battery temperature is normal
mfi0: 2472 (225481239s/0x0008/0) - Current capacity of the battery is
above threshold
pcib9: <PCI-PCI bridge> at device 0.2 on pci1
pci3: <PCI bus> on pcib9
pcib10: <ACPI PCI-PCI bridge> at device 4.0 on pci0
pci12: <ACPI PCI bus> on pcib10
pcib11: <PCI-PCI bridge> at device 5.0 on pci0
pci13: <PCI bus> on pcib11
pcib12: <ACPI PCI-PCI bridge> at device 6.0 on pci0
pci14: <ACPI PCI bus> on pcib12
pcib13: <PCI-PCI bridge> at device 7.0 on pci0
pci15: <PCI bus> on pcib13
pcib14: <ACPI PCI-PCI bridge> at device 28.0 on pci0
pci4: <ACPI PCI bus> on pcib14
pcib15: <PCI-PCI bridge> at device 0.0 on pci4
pci5: <PCI bus> on pcib15
bce1: <Broadcom NetXtreme II BCM5708 1000Base-T (B1), v0.9.6> mem
0xf8000000-0xf9ffffff irq 16 at device 0.0 on pci5
bce1: ASIC ID 0x57081010; Revision (B1); PCI-X 64-bit 133MHz
miibus1: <MII bus> on bce1
brgphy1: <BCM5708C 10/100/1000baseTX PHY> on miibus1
brgphy1: 10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, 1000baseTX,
1000baseTX-FDX, auto
bce1: Ethernet address: 00:13:72:64:06:dc
uhci0: <UHCI (generic) USB controller> port 0xdce0-0xdcff irq 21 at
device 29.0 on pci0
uhci0: [GIANT-LOCKED]
usb0: <UHCI (generic) USB controller> on uhci0
usb0: USB revision 1.0
uhub0: Intel UHCI root hub, class 9/0, rev 1.00/1.00, addr 1
uhub0: 2 ports with 2 removable, self powered
uhci1: <UHCI (generic) USB controller> port 0xdcc0-0xdcdf irq 20 at
device 29.1 on pci0
uhci1: [GIANT-LOCKED]
usb1: <UHCI (generic) USB controller> on uhci1
usb1: USB revision 1.0
uhub1: Intel UHCI root hub, class 9/0, rev 1.00/1.00, addr 1
uhub1: 2 ports with 2 removable, self powered
uhci2: <UHCI (generic) USB controller> port 0xdca0-0xdcbf irq 21 at
device 29.2 on pci0
uhci2: [GIANT-LOCKED]
usb2: <UHCI (generic) USB controller> on uhci2
usb2: USB revision 1.0
uhub2: Intel UHCI root hub, class 9/0, rev 1.00/1.00, addr 1
uhub2: 2 ports with 2 removable, self powered
ehci0: <EHCI (generic) USB 2.0 controller> mem 0xfc600000-0xfc6003ff irq
21 at device 29.7 on pci0
ehci0: [GIANT-LOCKED]
usb3: EHCI version 1.0
usb3: companion controllers, 2 ports each: usb0 usb1 usb2
usb3: <EHCI (generic) USB 2.0 controller> on ehci0
usb3: USB revision 2.0
uhub3: Intel EHCI root hub, class 9/0, rev 2.00/1.00, addr 1
uhub3: 6 ports with 6 removable, self powered
uhub4: vendor 0x413c product 0xa001, class 9/0, rev 2.00/0.00, addr 2
uhub4: multiple transaction translators
uhub4: 2 ports with 2 removable, self powered
ukbd0: Dell DRAC5, rev 1.10/0.00, addr 3, iclass 3/1
kbd1 at ukbd0
ums0: Dell DRAC5, rev 1.10/0.00, addr 3, iclass 3/1
ums0: X report 0x0002 not supported
device_attach: ums0 attach returned 6
ugen0: DELL INC. DRAC5 VIRTUAL MEDIA, rev 2.00/0.00, addr 4
uhub5: vendor 0x04b4 product 0x6560, class 9/0, rev 2.00/0.0b, addr 5
uhub5: multiple transaction translators
uhub5: 4 ports with 4 removable, self powered
pcib16: <ACPI PCI-PCI bridge> at device 30.0 on pci0
pci16: <ACPI PCI bus> on pcib16
pci16: <display, VGA> at device 13.0 (no driver attached)
isab0: <PCI-ISA bridge> at device 31.0 on pci0
isa0: <ISA bus> on isab0
atapci0: <Intel 63XXESB2 UDMA100 controller> port
0x1f0-0x1f7,0x3f6,0x170-0x177,0x376,0xfc00-0xfc0f at device 31.1 on pci0
ata0: <ATA channel 0> on atapci0
ata1: <ATA channel 1> on atapci0
fdc0: <floppy drive controller> port 0x3f0-0x3f5,0x3f7 irq 6 drq 2 on
acpi0
fdc0: does not respond
device_attach: fdc0 attach returned 6
sio0: configured irq 4 not in bitmap of probed irqs 0
sio0: port may not be enabled
sio0: <16550A-compatible COM port> port 0x3f8-0x3ff irq 4 on acpi0
sio0: type 16550A
sio1: configured irq 3 not in bitmap of probed irqs 0
sio1: port may not be enabled
sio1: <16550A-compatible COM port> port 0x2f8-0x2ff irq 3 flags 0x10 on
acpi0
sio1: type 16550A, console
fdc0: <floppy drive controller> port 0x3f0-0x3f5,0x3f7 irq 6 drq 2 on
acpi0
fdc0: does not respond
device_attach: fdc0 attach returned 6
orm0: <ISA Option ROMs> at iomem 0xc0000-0xc8fff,0xec000-0xeffff on isa0
sc0: <System console> at flags 0x100 on isa0
sc0: VGA <16 virtual consoles, flags=0x300>
vga0: <Generic ISA VGA> at port 0x3c0-0x3df iomem 0xa0000-0xbffff on isa0
ukbd1: vendor 0x0d3d USBPS2, rev 1.10/0.01, addr 2, iclass 3/1
kbd2 at ukbd1
ums0: vendor 0x0d3d USBPS2, rev 1.10/0.01, addr 2, iclass 3/1
ums0: 5 buttons and Z dir.
Timecounters tick every 10.000 msec
acd0: DVDROM <PHILIPS DVD-ROM SDR089/TD36> at ata0-master UDMA33
mfid0: <MFI Logical Disk> on mfi0
mfid0: 418176MB (856424448 sectors) RAID volume 'Virtual Disk 0' is
optimal
SMP: AP CPU #6 Launched!
SMP: AP CPU #4 Launched!
SMP: AP CPU #1 Launched!
SMP: AP CPU #7 Launched!
SMP: AP CPU #5 Launched!
SMP: AP CPU #3 Launched!
SMP: AP CPU #2 Launched!
Trying to mount root from ufs:/dev/mfid0s1a
WARNING: / was not properly dismounted
WARNING: /data was not properly dismounted
WARNING: /tmp was not properly dismounted
WARNING: /usr was not properly dismounted
WARNING: /var was not properly dismounted
Conf File.
machine amd64
cpu HAMMER
ident PE2950
makeoptions DEBUG=-g # Build kernel with gdb(1) debug
symbols
options SCHED_4BSD # 4BSD scheduler
options PREEMPTION # Enable kernel thread preemption
options INET # InterNETworking
options FFS # Berkeley Fast Filesystem
options SOFTUPDATES # Enable FFS soft updates support
options UFS_ACL # Support for access control lists
options UFS_DIRHASH # Improve performance on big
directories
options MD_ROOT # MD is a potential root device
options NFSCLIENT # Network Filesystem Client
options NFSSERVER # Network Filesystem Server
options NFS_ROOT # NFS usable as /, requires
NFSCLIENT
options NTFS # NT File System
options MSDOSFS # MSDOS Filesystem
options CD9660 # ISO 9660 Filesystem
options PROCFS # Process filesystem (requires
PSEUDOFS)
options PSEUDOFS # Pseudo-filesystem framework
options GEOM_GPT # GUID Partition Tables.
options COMPAT_43 # Needed by COMPAT_LINUX32
options COMPAT_IA32 # Compatible with i386 binaries
options COMPAT_FREEBSD4 # Compatible with FreeBSD4
options COMPAT_FREEBSD5 # Compatible with FreeBSD5
options COMPAT_LINUX32 # Compatible with i386 linux
binaries
options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI
options KTRACE # ktrace(1) support
options SYSVSHM # SYSV-style shared memory
options SYSVMSG # SYSV-style message queues
options SYSVSEM # SYSV-style semaphores
options _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time
extensions
options KBD_INSTALL_CDEV # install a CDEV entry in /dev
options AHC_REG_PRETTY_PRINT # Print register bitfields in debug
# output. Adds ~128k to driver.
options AHD_REG_PRETTY_PRINT # Print register bitfields in debug
# output. Adds ~215k to driver.
options ADAPTIVE_GIANT # Giant mutex is adaptive.
options SMP
option INVARIANTS
option INVARIANT_SUPPORT
options KDB
options DDB
options KDB_UNATTENDED
options BREAK_TO_DEBUGGER
device atpic # 8259A compatability
options LINPROCFS # Cannot be a module yet.
device acpi
device pci
device fdc
device ata
device atadisk # ATA disk drives
device ataraid # ATA RAID drives
device atapicd # ATAPI CDROM drives
device atapifd # ATAPI floppy drives
device atapist # ATAPI tape drives
options ATA_STATIC_ID # Static device numbering
device mpt # LSI-Logic MPT-Fusion
device scbus # SCSI bus (required for SCSI)
device ch # SCSI media changers
device da # Direct Access (disks)
device sa # Sequential Access (tape etc)
device cd # CD
device pass # Passthrough device (direct SCSI access)
device ses # SCSI Environmental Services (and SAF-TE)
device mfi
device kbdmux
device vga # VGA video card driver
device splash # Splash screen and screen saver support
device sc
device agp # support several AGP chipsets
device sio # 8250, 16[45]50 based serial ports
device miibus # MII bus support
device bce
device loop # Network loopback
device random # Entropy device
device ether # Ethernet support
device pty # Pseudo-ttys (telnet etc)
device bpf # Berkeley packet filter
device uhci # UHCI PCI->USB interface
device ohci # OHCI PCI->USB interface
device ehci # EHCI PCI->USB interface (USB 2.0)
device usb # USB Bus (required)
device ugen # Generic
device uhid # "Human Interface Devices"
device ukbd # Keyboard
device ums # Mouse
Jason Thomson
2007-Feb-28 13:54 UTC
Help debugging non-sleepable lock panic on 6.2-RELEASE
Regarding: http://lists.freebsd.org/pipermail/freebsd-stable/2007-February/033241.html Konstantin Belousov wrote:> On Thu, Feb 22, 2007 at 06:50:37PM +0000, Jason Thomson wrote: > > > Try this (already committed to CURRENT as rev. 1.579). I would be very > interesting in getting feedback.Konstantin, thanks for your help. The patch below *appears* to have helped. The server survived for more than 24 hours under heavy (mysqld / apache) load. Previously, it would not typically survive this test for more than 5 hours. It's difficult to be 100% sure as the server load is not reliably reproducable. I will keep you informed of any developments. Thanks for your help. Kind regards, Jason.> > Index: sys/amd64/amd64/pmap.c > ==================================================================> RCS file: /usr/local/arch/ncvs/src/sys/amd64/amd64/pmap.c,v > retrieving revision 1.578 > retrieving revision 1.579 > diff -u -r1.578 -r1.579 > --- sys/amd64/amd64/pmap.c 18 Feb 2007 06:33:01 -0000 1.578 > +++ sys/amd64/amd64/pmap.c 19 Feb 2007 10:55:16 -0000 1.579 > @@ -77,7 +77,7 @@ > */ > > #include <sys/cdefs.h> > -__FBSDID("$FreeBSD: src/sys/amd64/amd64/pmap.c,v 1.578 2007/02/18 06:33:01 alc Exp $"); > +__FBSDID("$FreeBSD: src/sys/amd64/amd64/pmap.c,v 1.579 2007/02/19 10:55:16 kib Exp $"); > > /* > * Manages physical address maps. > @@ -1542,9 +1542,15 @@ > while ((*pmap_pde(kernel_pmap, kernel_vm_end) & PG_V) != 0) { > kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); > nkpt++; > + if (kernel_vm_end - 1 >= kernel_map->max_offset) { > + kernel_vm_end = kernel_map->max_offset; > + break; > + } > } > } > addr = roundup2(addr, PAGE_SIZE * NPTEPG); > + if (addr - 1 >= kernel_map->max_offset) > + addr = kernel_map->max_offset; > while (kernel_vm_end < addr) { > pde = pmap_pde(kernel_pmap, kernel_vm_end); > if (pde == NULL) { > @@ -1562,6 +1568,10 @@ > } > if ((*pde & PG_V) != 0) { > kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); > + if (kernel_vm_end - 1 >= kernel_map->max_offset) { > + kernel_vm_end = kernel_map->max_offset; > + break; > + } > continue; > } > > @@ -1581,6 +1591,10 @@ > *pmap_pde(kernel_pmap, kernel_vm_end) = newpdir; > > kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); > + if (kernel_vm_end - 1 >= kernel_map->max_offset) { > + kernel_vm_end = kernel_map->max_offset; > + break; > + } > } > } >