freebsd stable - Oct 2007 - em lockups during heavy network I/O on RELENG_7

Hello,

I have managed to lock my (amd64, RELENG_7) machine up twice today. In
both cases, I was transferring a file to my laptop (in one case over
SMB, the other over FTP). Both resulted in a hard lock (no panic). One
of the lockups had an "em1: watchdog timeout" message on the console,
the other had no abnormal messages on the console. Both transfers were
over 100baseTx-FD ethernet over my em card (em1).

This is a new occurrence on RELENG_7. The box was stable on
6.2-RELEASE. The hardware is as follows:

Asus P5B motherboard (BIOS revison 1405)
Intel Core 2 Quad Q6600
Two Intel em cards (em0: internet facing, em1: LAN facing)

Further below is my full kernel config, but here are the options I
have that GENERIC does not (not necessarily in order, as they were
sorted when I was diff'ing with GENERIC):

device atapicam
device coretemp
device if_bridge
device pf
device pflog
device tap
ident PFLOG64
machine amd64
makeoptions COPTFLAGS="-O2 -pipe"
options ALTQ
options ALTQ_CBQ
options ALTQ_HFSC
options ALTQ_NOPCC
options ALTQ_PRIQ
options ALTQ_RED
options ALTQ_RIO
options COMPAT_LINUX32
options HZ=1000
options LIBICONV
options LIBMCHAIN
options NETSMB
options SC_PIXEL_MODE
options SMBFS
options UDF
options VGA_WIDTH90

I plan on removing the HZ, LIBICONV, LIBMCHAIN, SC_PIXEL_MODE, and
VGA_WIDTH90 options and seeing if that helps. Do any of the other
options look like likely culprits?

If the above doesn't help, is there a way to debug hard lockups? I
have 15mbit FiOS and have maxed em0 out without causing this.  So I'm
guessing either I'm not pushing enough packets through em0 to induce
this, or the problem is related to em1 sharing an interrupt with a USB
controller:

interrupt                          total       rate
irq1: atkbd0                           6          0
irq6: fdc0                            10          0
irq17: uhci1+                         91          0
irq19: uhci3+                     924492         32
irq22: em0                         85919          3
irq23: em1 uhci2+                   6627          0
cpu0: timer                     56950868       1999
cpu1: timer                     56950834       1999
cpu2: timer                     56950835       1999
cpu3: timer                     56950834       1999
Total                          228820516       8035

Anyway, here is the full kernel configuration:

makeoptions    COPTFLAGS="-O2 -pipe"

machine		amd64
cpu		HAMMER
ident		PFLOG64
options		COMPAT_IA32
options		COMPAT_LINUX32

##########################################################################

options SMP            # Symmetric MultiProcessor Kernel
options	STOP_NMI
options SCHED_4BSD
options PREEMPTION
options HZ=1000

options 	INET			# InterNETworking
options 	FFS			# Berkeley Fast Filesystem
options 	SOFTUPDATES		# Enable FFS soft updates support
options 	UFS_ACL			# Support for access control lists
options 	UFS_DIRHASH		# Improve performance on big directories
options		UFS_GJOURNAL	# Enable gjournal-based UFS
options 	NFSCLIENT		# Network Filesystem Client
options 	NFSSERVER		# Network Filesystem Server
options		NETSMB
options		SMBFS
options		LIBICONV
options		LIBMCHAIN
options 	CD9660			# ISO 9660 Filesystem
options		UDF
options 	PROCFS			# Process filesystem (requires PSEUDOFS)
options 	PSEUDOFS		# Pseudo-filesystem framework
options		GEOM_LABEL              # Provides labelization
options		GEOM_PART_GPT           # GUID Partition Tables.
options 	COMPAT_43TTY		# Compatible with BSD 4.3 [KEEP THIS!]
options 	COMPAT_FREEBSD4		# Compatible with FreeBSD4
options		COMPAT_FREEBSD5		# Compatible with FreeBSD5
options		COMPAT_FREEBSD6		# Compatible with FreeBSD6
options 	SCSI_DELAY=5000		# Delay (in ms) before probing SCSI
options 	KTRACE			# ktrace(1) support
options 	SYSVSHM			# SYSV-style shared memory
options 	SYSVMSG			# SYSV-style message queues
options 	SYSVSEM			# SYSV-style semaphores
options 	_KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
options 	KBD_INSTALL_CDEV	# install a CDEV entry in /dev
options 	AHC_REG_PRETTY_PRINT	# Print register bitfields in debug
					# output.  Adds ~128k to driver.
options 	AHD_REG_PRETTY_PRINT	# Print register bitfields in debug
					# output.  Adds ~215k to driver.
options 	ADAPTIVE_GIANT		# Giant mutex is adaptive.
options		AUDIT                   # Security event auditing

# Bus support.  Do not remove isa, even if you have no isa slots
device		pci
device		acpi

# Floppy drives
device		fdc

# coretemp: on-die sensor on Intel Core and newer CPUs
device          coretemp

# ATA and ATAPI devices
device		ata
device		atadisk		# ATA disk drives
device		atapicd		# ATAPI CDROM drives
options 	ATA_STATIC_ID	# Static device numbering

# SCSI peripherals
device		scbus		# SCSI bus (required for SCSI)
device		da		# Direct Access (disks)
device		cd		# CD
device		pass		# Passthrough device (direct SCSI access)
device		atapicam

# atkbdc0 controls both the keyboard and the PS/2 mouse
device		atkbdc		# AT keyboard controller
device		atkbd		# AT keyboard
device		psm		# PS/2 mouse

device		vga		# VGA video card driver
options		VGA_WIDTH90
options		SC_PIXEL_MODE

device		splash		# Splash screen and screen saver support

# syscons is the default console driver, resembling an SCO console
device		sc

device		if_bridge

# PCI Ethernet NICs.
device		em		# Intel PRO/1000 adapter Gigabit Ethernet Card

# Pseudo devices.
device		loop		# Network loopback
device		random		# Entropy device
device		ether		# Ethernet support
device		tap
device		pty		# Pseudo-ttys (telnet etc)
device		md		# Memory "disks"

# The `bpf' device enables the Berkeley Packet Filter.
# Be aware of the administrative consequences of enabling this!
# Note that 'bpf' is required for DHCP.
device		bpf		# Berkeley packet filter
device		pf
device		pflog

# USB support
device		uhci		# UHCI PCI->USB interface
device		ohci		# OHCI PCI->USB interface
device		ehci		# EHCI PCI->USB interface (USB 2.0)
device		usb		# USB Bus (required)
device		ugen		# Generic
device		umass		# Disks/Mass storage - Requires scbus and da

# FireWire support
device		firewire	# FireWire bus code
device		sbp		# SCSI over FireWire (Requires scbus and da)

# altq support
options         ALTQ
options         ALTQ_CBQ        # Class Bases Queuing (CBQ)
options         ALTQ_RED        # Random Early Detection (RED)
options         ALTQ_RIO        # RED In/Out
options         ALTQ_HFSC       # Hierarchical Packet Scheduler (HFSC)
options         ALTQ_PRIQ       # Priority Queuing (PRIQ)
options         ALTQ_NOPCC      # Required for SMP build


Thanks!
Josh

Sorry, I should have also included dmesg output. The "not properly
dismounted" errors are obviously from the last crash :)

Here is /var/run/dmesg.boot:

Copyright (c) 1992-2007 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
	The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 7.0-PRERELEASE #4: Tue Oct 16 16:00:16 EDT 2007
    root@pflog.net:/usr/obj/usr/src/sys/PFLOG
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) Core(TM)2 Quad CPU    Q6600  @ 2.40GHz (3400.28-MHz K8-class CPU)
  Origin = "GenuineIntel"  Id = 0x6fb  Stepping = 11
 
Features=0xbfebfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM,PBE>
  Features2=0xe3bd<SSE3,RSVD2,MON,DS_CPL,VMX,EST,TM2,SSSE3,CX16,xTPR,PDCM>
  AMD Features=0x20100800<SYSCALL,NX,LM>
  AMD Features2=0x1<LAHF>
  Cores per package: 4
usable memory = 2139217920 (2040 MB)
avail memory  = 2064588800 (1968 MB)
ACPI APIC Table: <MSTEST OEMAPIC >
FreeBSD/SMP: Multiprocessor System Detected: 4 CPUs
 cpu0 (BSP): APIC ID:  0
 cpu1 (AP): APIC ID:  1
 cpu2 (AP): APIC ID:  2
 cpu3 (AP): APIC ID:  3
ioapic0 <Version 2.0> irqs 0-23 on motherboard
netsmb_dev: loaded
acpi0: <MSTEST TESTONLY> on motherboard
acpi0: [ITHREAD]
acpi0: Power Button (fixed)
acpi0: reservation of 0, a0000 (3) failed
acpi0: reservation of 100000, 7ff00000 (3) failed
Timecounter "ACPI-fast" frequency 3579545 Hz quality 1000
acpi_timer0: <24-bit timer at 3.579545MHz> port 0x808-0x80b on acpi0
acpi_hpet0: <High Precision Event Timer> iomem 0xfed00000-0xfed003ff on
acpi0
Timecounter "HPET" frequency 14318180 Hz quality 900
cpu0: <ACPI CPU> on acpi0
coretemp0: <CPU On-Die Thermal Sensors> on cpu0
cpu1: <ACPI CPU> on acpi0
coretemp1: <CPU On-Die Thermal Sensors> on cpu1
cpu2: <ACPI CPU> on acpi0
coretemp2: <CPU On-Die Thermal Sensors> on cpu2
cpu3: <ACPI CPU> on acpi0
coretemp3: <CPU On-Die Thermal Sensors> on cpu3
pcib0: <ACPI Host-PCI bridge> port 0xcf8-0xcff on acpi0
pci0: <ACPI PCI bus> on pcib0
pcib1: <ACPI PCI-PCI bridge> irq 16 at device 1.0 on pci0
pci1: <ACPI PCI bus> on pcib1
vgapci0: <VGA-compatible display> mem
0xfd000000-0xfdffffff,0xc0000000-0xcfffffff,0xfc000000-0xfcffffff irq
16 at device 0.0 on pci1
uhci0: <UHCI (generic) USB controller> port 0xe000-0xe01f irq 16 at
device 26.0 on pci0
uhci0: [GIANT-LOCKED]
uhci0: [ITHREAD]
usb0: <UHCI (generic) USB controller> on uhci0
usb0: USB revision 1.0
uhub0: <Intel UHCI root hub, class 9/0, rev 1.00/1.00, addr 1> on usb0
uhub0: 2 ports with 2 removable, self powered
uhci1: <UHCI (generic) USB controller> port 0xe080-0xe09f irq 17 at
device 26.1 on pci0
uhci1: [GIANT-LOCKED]
uhci1: [ITHREAD]
usb1: <UHCI (generic) USB controller> on uhci1
usb1: USB revision 1.0
uhub1: <Intel UHCI root hub, class 9/0, rev 1.00/1.00, addr 1> on usb1
uhub1: 2 ports with 2 removable, self powered
ehci0: <EHCI (generic) USB 2.0 controller> mem 0xfebff400-0xfebff7ff
irq 18 at device 26.7 on pci0
ehci0: [GIANT-LOCKED]
ehci0: [ITHREAD]
usb2: EHCI version 1.0
usb2: companion controllers, 2 ports each: usb0 usb1
usb2: <EHCI (generic) USB 2.0 controller> on ehci0
usb2: USB revision 2.0
uhub2: <Intel EHCI root hub, class 9/0, rev 2.00/1.00, addr 1> on usb2
uhub2: 4 ports with 4 removable, self powered
pcib2: <ACPI PCI-PCI bridge> irq 16 at device 28.0 on pci0
pci3: <ACPI PCI bus> on pcib2
pcib3: <ACPI PCI-PCI bridge> irq 16 at device 28.4 on pci0
pci2: <ACPI PCI bus> on pcib3
atapci0: <JMicron JMB363 SATA300 controller> mem 0xfe8fe000-0xfe8fffff
irq 16 at device 0.0 on pci2
atapci0: AHCI Version 01.00 controller with 2 ports detected
ata0: <ATA channel 0> on atapci0
ata0: [ITHREAD]
ata1: <ATA channel 1> on atapci0
ata1: [ITHREAD]
atapci1: <JMicron JMB363 UDMA133 controller> port
0xbc00-0xbc07,0xb880-0xb883,0xb800-0xb807,0xb480-0xb483,0xb400-0xb40f
at device 0.1 on pci2
atapci1: [ITHREAD]
ata2: <ATA channel 0> on atapci1
ata2: [ITHREAD]
uhci2: <UHCI (generic) USB controller> port 0xd800-0xd81f irq 23 at
device 29.0 on pci0
uhci2: [GIANT-LOCKED]
uhci2: [ITHREAD]
usb3: <UHCI (generic) USB controller> on uhci2
usb3: USB revision 1.0
uhub3: <Intel UHCI root hub, class 9/0, rev 1.00/1.00, addr 1> on usb3
uhub3: 2 ports with 2 removable, self powered
uhci3: <UHCI (generic) USB controller> port 0xd880-0xd89f irq 19 at
device 29.1 on pci0
uhci3: [GIANT-LOCKED]
uhci3: [ITHREAD]
usb4: <UHCI (generic) USB controller> on uhci3
usb4: USB revision 1.0
uhub4: <Intel UHCI root hub, class 9/0, rev 1.00/1.00, addr 1> on usb4
uhub4: 2 ports with 2 removable, self powered
uhci4: <UHCI (generic) USB controller> port 0xdc00-0xdc1f irq 18 at
device 29.2 on pci0
uhci4: [GIANT-LOCKED]
uhci4: [ITHREAD]
usb5: <UHCI (generic) USB controller> on uhci4
usb5: USB revision 1.0
uhub5: <Intel UHCI root hub, class 9/0, rev 1.00/1.00, addr 1> on usb5
uhub5: 2 ports with 2 removable, self powered
ehci1: <EHCI (generic) USB 2.0 controller> mem 0xfebff000-0xfebff3ff
irq 23 at device 29.7 on pci0
ehci1: [GIANT-LOCKED]
ehci1: [ITHREAD]
usb6: EHCI version 1.0
usb6: companion controllers, 2 ports each: usb3 usb4 usb5
usb6: <EHCI (generic) USB 2.0 controller> on ehci1
usb6: USB revision 2.0
uhub6: <Intel EHCI root hub, class 9/0, rev 2.00/1.00, addr 1> on usb6
uhub6: 6 ports with 6 removable, self powered
pcib4: <ACPI PCI-PCI bridge> at device 30.0 on pci0
pci4: <ACPI PCI bus> on pcib4
em0: <Intel(R) PRO/1000 Network Connection Version - 6.5.3> port
0xcc00-0xcc3f mem 0xfeae0000-0xfeafffff,0xfeac0000-0xfeadffff irq 22
at device 1.0 on pci4
em0: Ethernet address: 00:0e:0c:6c:b9:16
em0: [FILTER]
em1: <Intel(R) PRO/1000 Network Connection Version - 6.5.3> port
0xc880-0xc8bf mem 0xfea80000-0xfea9ffff,0xfea60000-0xfea7ffff irq 23
at device 2.0 on pci4
em1: Ethernet address: 00:0e:0c:6c:b9:0a
em1: [FILTER]
isab0: <PCI-ISA bridge> at device 31.0 on pci0
isa0: <ISA bus> on isab0
atapci2: <Intel ICH8 SATA300 controller> port
0xec00-0xec07,0xe880-0xe883,0xe800-0xe807,0xe480-0xe483,0xe400-0xe41f
mem 0xfebff800-0xfebfffff irq 19 at device 31.2 on pci0
atapci2: [ITHREAD]
atapci2: AHCI Version 01.10 controller with 4 ports detected
ata3: <ATA channel 0> on atapci2
ata3: [ITHREAD]
ata4: <ATA channel 1> on atapci2
ata4: [ITHREAD]
ata5: <ATA channel 2> on atapci2
ata5: port not implemented
ata5: [ITHREAD]
ata6: <ATA channel 3> on atapci2
ata6: port not implemented
ata6: [ITHREAD]
ata7: <ATA channel 4> on atapci2
ata7: [ITHREAD]
ata8: <ATA channel 5> on atapci2
ata8: [ITHREAD]
pci0: <serial bus, SMBus> at device 31.3 (no driver attached)
acpi_button0: <Power Button> on acpi0
fdc0: <floppy drive controller (FDE)> port 0x3f0-0x3f5,0x3f7 irq 6 drq
2 on acpi0
fdc0: [FILTER]
fd0: <1440-KB 3.5" drive> on fdc0 drive 0
atkbdc0: <Keyboard controller (i8042)> port 0x60,0x64 irq 1 on acpi0
atkbd0: <AT Keyboard> irq 1 on atkbdc0
kbd0 at atkbd0
atkbd0: [GIANT-LOCKED]
atkbd0: [ITHREAD]
psm0: <PS/2 Mouse> irq 12 on atkbdc0
psm0: [GIANT-LOCKED]
psm0: [ITHREAD]
psm0: model IntelliMouse Explorer, device ID 4
orm0: <ISA Option ROMs> at iomem 0xd2000-0xd2fff,0xd3000-0xd3fff on isa0
sc0: <System console> at flags 0x100 on isa0
sc0: VGA <16 virtual consoles, flags=0x300>
vga0: <Generic ISA VGA> at port 0x3c0-0x3df iomem 0xa0000-0xbffff on isa0
Timecounters tick every 1.000 msec
acd0: DVDR <SONY DVD RW DRU-800A/KY06> at ata2-master UDMA66
ad6: 476940MB <WDC WD5000AAKS-07TMA0 12.01C01> at ata3-master SATA300
ad8: 381553MB <Seagate ST3400633AS 3.AAH> at ata4-master SATA150
ad14: 476940MB <WDC WD5000AAKS-00YGA0 12.01C02> at ata7-master SATA300
SMP: AP CPU #1 Launched!
SMP: AP CPU #2 Launched!
SMP: AP CPU #3 Launched!
Trying to mount root from ufs:/dev/ad6s1a
WARNING: / was not properly dismounted
acd0: FAILURE - INQUIRY ILLEGAL REQUEST asc=0x24 ascq=0x00
cd0 at ata2 bus 0 target 0 lun 0
cd0: <SONY DVD RW DRU-800A KY06> Removable CD-ROM SCSI-0 device
cd0: 66.000MB/s transfers
cd0: Attempt to query device size failed: NOT READY, Medium not present
WARNING: /tmp was not properly dismounted
WARNING: /var was not properly dismounted
WARNING: /data/video was not properly dismounted
WARNING: /backup was not properly dismounted

Thanks,
Josh