kernel crash ...

Vincent Blondel
Thu, 16 Nov 2006 16:00:11 -0000

Hello all,

Sorry to spam this list but this morning at 03:00 AM I get back a kernel
crash. Seems mailwrapper crashed now.

Do I make a new build/install world/kernel ?

Please, can somebody help me solve this problem.

Just for info I put some details of my config below + last kernel debug.

Regards.
Vincent

---

kernel S2468GN

machine         i386
cpu             I686_CPU
ident           S2468GN

# To make an SMP kernel, the next line is needed
options         SMP                     # Symmetric MultiProcessor
Kernel

# To statically compile in device wiring instead of /boot/device.hints
#hints          "GENERIC.hints"         # Default places to look for
devices.

makeoptions     DEBUG=-g                # Build kernel with gdb(1) debug
symbols

options         SCHED_4BSD              # 4BSD scheduler
options         PREEMPTION              # Enable kernel thread
preemption
options         INET                    # InterNETworking
options         INET6                   # IPv6 communications protocols
options         FFS                     # Berkeley Fast Filesystem
options         SOFTUPDATES             # Enable FFS soft updates
support
options         UFS_ACL                 # Support for access control
lists
options         UFS_DIRHASH             # Improve performance on big
directories
options         MD_ROOT                 # MD is a potential root device
#options        NFSCLIENT               # Network Filesystem Client
#options        NFSSERVER               # Network Filesystem Server
#options        NFS_ROOT                # NFS usable as /, requires
NFSCLIENT
#options        MSDOSFS                 # MSDOS Filesystem
#options        CD9660                  # ISO 9660 Filesystem
options         PROCFS                  # Process filesystem (requires
PSEUDOFS)
options         PSEUDOFS                # Pseudo-filesystem framework
options         GEOM_GPT                # GUID Partition Tables.
options         GEOM_MIRROR             # Soft Mirror
options         COMPAT_43               # Compatible with BSD 4.3 [KEEP
THIS!]
options         COMPAT_FREEBSD4         # Compatible with FreeBSD4
options         COMPAT_FREEBSD5         # Compatible with FreeBSD5
options         SCSI_DELAY=5000         # Delay (in ms) before probing
SCSI
options         KTRACE                  # ktrace(1) support
options         SYSVSHM                 # SYSV-style shared memory
options         SYSVMSG                 # SYSV-style message queues
options         SYSVSEM                 # SYSV-style semaphores
options         _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time
extensions
options         KBD_INSTALL_CDEV        # install a CDEV entry in /dev
options         ADAPTIVE_GIANT          # Giant mutex is adaptive.

device          apic                    # I/O APIC

# Bus support.
device          eisa
device          pci

# Floppy drives
device          fdc

# ATA and ATAPI devices
device          ata
device          atadisk         # ATA disk drives
device          ataraid         # ATA RAID drives
device          atapicd         # ATAPI CDROM drives
device          atapifd         # ATAPI floppy drives
device          atapist         # ATAPI tape drives
options         ATA_STATIC_ID   # Static device numbering

# SCSI peripherals
device          scbus           # SCSI bus (required for SCSI)
device          ch              # SCSI media changers
device          da              # Direct Access (disks)
device          sa              # Sequential Access (tape etc)
device          cd              # CD
device          pass            # Passthrough device (direct SCSI
access)
device          ses             # SCSI Environmental Services (and
SAF-TE)

# RAID controllers
device          aac             # Adaptec FSA RAID
device          aacp            # SCSI passthrough for aac (requires
CAM)

# atkbdc0 controls both the keyboard and the PS/2 mouse
device          atkbdc          # AT keyboard controller
device          atkbd           # AT keyboard
device          psm             # PS/2 mouse

device          kbdmux          # keyboard multiplexer

device          vga             # VGA video card driver

# syscons is the default console driver, resembling an SCO console
device          sc

# Enable this for the pcvt (VT220 compatible) console driver
#device         vt
#options        XSERVER         # support for X server on a vt console
#options        FAT_CURSOR      # start with block cursor

# Power management support (see NOTES for more options)
#device         apm
# Add suspend/resume support for the i8254.
device          pmtimer

# Serial (COM) ports
device          sio             # 8250, 16[45]50 based serial ports

# Parallel port
device          ppc
device          ppbus           # Parallel port bus (required)
device          lpt             # Printer
device          plip            # TCP/IP over parallel
device          ppi             # Parallel port interface device
#device         vpo             # Requires scbus and da

# If you've got a "dumb" serial or parallel PCI card that is
# supported by the puc(4) glue driver, uncomment the following
# line to enable it (connects to the sio and/or ppc drivers):
#device         puc

# PCI Ethernet NICs that use the common MII bus controller code.
# NOTE: Be sure to keep the 'device miibus' line in order to use these
NICs!
device          miibus          # MII bus support
device          xl              # 3Com 3c90x (``Boomerang'',
``Cyclone'')

# Pseudo devices.
device          loop            # Network loopback
device          random          # Entropy device
device          ether           # Ethernet support
device          sl              # Kernel SLIP
device          ppp             # Kernel PPP
device          tun             # Packet tunnel.
device          pty             # Pseudo-ttys (telnet etc)
device          md              # Memory "disks"
device          gif             # IPv6 and IPv4 tunneling
device          faith           # IPv6-to-IPv4 relaying (translation)

# The `bpf' device enables the Berkeley Packet Filter.
# Be aware of the administrative consequences of enabling this!
# Note that 'bpf' is required for DHCP.
device          bpf             # Berkeley packet filter

options UKBD_DFLT_KEYMAP
makeoptions UKBD_DFLT_KEYMAP=fr.iso.acc

options         SC_DISABLE_REBOOT       # Disable Ctrl+Alt+Delete

---

root@sbepfkaa [/usr/src/sys/i386/conf/kernels] # dmesg
Copyright (c) 1992-2006 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights
reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 6.2-PRERELEASE #0: Sat Nov 11 17:18:06 CET 2006
    root@sbedfkdv.srv.xtra-net.be:/usr/obj/usr/src/sys/S2468GN
MPTable: <TYAN     PAULANER    >
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: AMD Athlon(tm) MP 1800+ (1533.40-MHz 686-class CPU)
  Origin = "AuthenticAMD"  Id = 0x662  Stepping = 2

Features=0x383fbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,MMX,FXSR,SSE>
  AMD Features=0xc0480800<SYSCALL,MP,MMX+,3DNow+,3DNow>
real memory  = 1073217536 (1023 MB)
avail memory = 1045417984 (996 MB)
FreeBSD/SMP: Multiprocessor System Detected: 2 CPUs
 cpu0 (BSP): APIC ID:  1
 cpu1 (AP): APIC ID:  0
ioapic0: Assuming intbase of 0
ioapic0 <Version 1.1> irqs 0-23 on motherboard
kbd1 at kbdmux0
cpu0 on motherboard
cpu1 on motherboard
pcib0: <MPTable Host-PCI bridge> pcibus 0 on motherboard
pci0: <PCI bus> on pcib0
pcib1: <PCI-PCI bridge> at device 1.0 on pci0
pci1: <PCI bus> on pcib1
isab0: <PCI-ISA bridge> at device 7.0 on pci0
isa0: <ISA bus> on isab0
atapci0: <AMD 768 UDMA100 controller> port
0x1f0-0x1f7,0x3f6,0x170-0x177,0x376,0xf000-0xf00f at device 7.1 on pci0
ata0: <ATA channel 0> on atapci0
ata1: <ATA channel 1> on atapci0
pci0: <bridge> at device 7.3 (no driver attached)
aac0: <Adaptec SCSI RAID 5400S> port 0x1000-0x10ff mem
0xf4000000-0xf4001fff irq 16 at device 8.0 on pci0
aac0: [FAST]
aac0: Adaptec Raid Controller 2.0.0-1
pcib2: <MPTable PCI-PCI bridge> at device 16.0 on pci0
pci2: <PCI bus> on pcib2
pci2: <serial bus, USB> at device 0.0 (no driver attached)
xl0: <3Com 3c905C-TX Fast Etherlink XL> port 0x2400-0x247f mem
0xf4102000-0xf410207f irq 16 at device 4.0 on pci2
miibus0: <MII bus> on xl0
xlphy0: <3c905C 10/100 internal PHY> on miibus0
xlphy0:  10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, auto
xl0: Ethernet address: 00:04:76:e9:32:89
pci2: <display, VGA> at device 7.0 (no driver attached)
xl1: <3Com 3c980C Fast Etherlink XL> port 0x2480-0x24ff mem
0xf4102400-0xf410247f irq 18 at device 8.0 on pci2
miibus1: <MII bus> on xl1
ukphy0: <Generic IEEE 802.3u media interface> on miibus1
ukphy0:  10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, auto
xl1: Ethernet address: 00:e0:81:23:30:b4
xl2: <3Com 3c980C Fast Etherlink XL> port 0x2800-0x287f mem
0xf4102800-0xf410287f irq 19 at device 9.0 on pci2
miibus2: <MII bus> on xl2
ukphy1: <Generic IEEE 802.3u media interface> on miibus2
ukphy1:  10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, auto
xl2: Ethernet address: 00:e0:81:23:30:b5
pmtimer0 on isa0
orm0: <ISA Option ROMs> at iomem
0xc0000-0xc7fff,0xc8000-0xc87ff,0xc8800-0xc8fff,0xc9000-0xccfff,0xcd000-0xcd7ff,0xe0000-0xe3fff
on isa0
atkbdc0: <Keyboard controller (i8042)> at port 0x60,0x64 on isa0
atkbd0: <AT Keyboard> irq 1 on atkbdc0
kbd0 at atkbd0
atkbd0: [GIANT-LOCKED]
fdc0: <Enhanced floppy controller> at port 0x3f0-0x3f5,0x3f7 irq 6 drq 2
on isa0
fdc0: [FAST]
fd0: <1440-KB 3.5" drive> on fdc0 drive 0
ppc0: <Parallel port> at port 0x378-0x37f irq 7 on isa0
ppc0: SMC-like chipset (ECP/EPP/PS2/NIBBLE) in COMPATIBLE mode
ppc0: FIFO with 16/16/9 bytes threshold
ppbus0: <Parallel port bus> on ppc0
plip0: <PLIP network interface> on ppbus0
lpt0: <Printer> on ppbus0
lpt0: Interrupt-driven port
ppi0: <Parallel I/O> on ppbus0
sc0: <System console> at flags 0x100 on isa0
sc0: VGA <16 virtual consoles, flags=0x100>
sio0 at port 0x3f8-0x3ff irq 4 flags 0x10 on isa0
sio0: type 16550A, console
sio1 at port 0x2f8-0x2ff irq 3 on isa0
sio1: type 16550A
vga0: <Generic ISA VGA> at port 0x3c0-0x3df iomem 0xa0000-0xbffff on
isa0
unknown: <PNP0303> can't assign resources (port)
unknown: <PNP0c02> can't assign resources (memory)
unknown: <PNP0501> can't assign resources (port)
unknown: <PNP0501> can't assign resources (port)
unknown: <PNP0401> can't assign resources (port)
unknown: <PNP0700> can't assign resources (port)
Timecounters tick every 1.000 msec
acd0: DVDROM <Pioneer DVD-ROM ATAPIModel DVD-106S 0122/E1.22> at
ata0-master UDMA66
aacd0: <RAID 5> on aac0
aacd0: 52353MB (107219712 sectors)
aacd1: <Passthrough> on aac0
aacd1: 35242MB (72176567 sectors)
aacd2: <Passthrough> on aac0
aacd2: 35074MB (71833096 sectors)
SMP: AP CPU #1 Launched!
Trying to mount root from ufs:/dev/aacd0s1a
WARNING: / was not properly dismounted
WARNING: /home was not properly dismounted
/home: mount pending error: blocks 24 files 4
WARNING: /tmp was not properly dismounted
WARNING: /usr was not properly dismounted
/usr: mount pending error: blocks 304 files 61
WARNING: /var was not properly dismounted
/var: mount pending error: blocks 4612 files 1
Accounting enabled
xl1: transmission error: 90
xl1: tx underrun, increasing tx start threshold to 120 bytes
xl1: watchdog timeout
xl1: link state changed to DOWN
xl1: link state changed to UP
root@sbepfkaa [/usr/src/sys/i386/conf/kernels] #

---

root@sbepfkaa [/root] # egrep -v '^$|^#' /etc/sysctl.conf

kern.maxfiles=16384
kern.corefile="/var/coredumps/%N.%P.%U.core"

---

root@sbepfkaa [/root] # egrep -v '^$|^#' /boot/loader.conf
console="comconsole"
kern.ipc.msgmnb=8192
kern.ipc.msgssz=64
kern.ipc.msgtql=2048

----

root@sbepfkaa [/usr/obj/usr/src/sys/S2468GN] # kgdb
kernel.debug /var/crash/vmcore.1
[GDB will not be able to debug user-mode
threads: /usr/lib/libthread_db.so: Undefined symbol
"ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you
are
welcome to change it and/or distribute copies of it under certain
conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for
details.
This GDB was configured as "i386-marcel-freebsd".

Unread portion of the kernel message buffer:
x0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, def32 1, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 14294 (mailwrapper)
trap number             = 12
panic: page fault
cpuid = 0
Uptime: 6h23m20s
Dumping 1023 MB (2 chunks)
  chunk 0: 1MB (159 pages) ... ok
  chunk 1: 1023MB (261760 pages) 1007 991 975 959 943 927 911 895 879
863 847 831 815 799 783 767 751 735 719 703 687 671 655 639 623 607 591
575 559 543 527 511 495 479 463 447 431 415 399 383 367 351 335 319 303
287 271 255 239 223 207 191 175 159 143 127 111 95 79 63 47 31 15

#0  doadump () at pcpu.h:165
165     pcpu.h: No such file or directory.
        in pcpu.h
(kgdb) list
160     in pcpu.h
(kgdb) quit
root@sbepfkaa [/usr/obj/usr/src/sys/S2468GN] #



On Wed, 2006-11-15 at 21:09 +0100, Vincent Blondel
wrote:
> Hello all,
>
> -- System: FreeBSD-6.2-BETA3 | Tyan S2468GN --
>
> I got a kernel crash on my web server this evening. I am now trying to
> debug the crash image generated on /var/crash/vmcore.0 but I am not
> comfortable with this procedure.
>
> As far as I can see it seems process httpd crashed but I do not know
> what I have to do to obtain precise info on this crash.
>
> I give you below a snapshot of I what I tried until now.
>
> So, could somebody help me debugging this crash ?
>
> Many thanks for your help.
>
> Regards
> Vincent.
>
> --
>
> root@sbepfkaa [/usr/obj/usr/src/sys/S2468GN] # kgdb
> kernel.debug /var/crash/vmcore.0
> [GDB will not be able to debug user-mode
> threads: /usr/lib/libthread_db.so: Undefined symbol
"ps_pglobal_lookup"]
> GNU gdb 6.1.1 [FreeBSD]
> Copyright 2004 Free Software Foundation, Inc.
> GDB is free software, covered by the GNU General Public License, and you
> are
> welcome to change it and/or distribute copies of it under certain
> conditions.
> Type "show copying" to see the conditions.
> There is absolutely no warranty for GDB.  Type "show warranty"
for
> details.
> This GDB was configured as "i386-marcel-freebsd".
>
> Unread portion of the kernel message buffer:
> kernel trap 12 with interrupts disabled
>
>
> Fatal trap 12: page fault while in kernel mode
> cpuid = 0; apic id = 01
> fault virtual address   = 0x27675eda
> fault code              = supervisor write, page not present
> instruction pointer     = 0x20:0xc062a143
> stack pointer           = 0x28:0xe7063988
> frame pointer           = 0x28:0xe70639a4
> code segment            = base 0x0, limit 0xfffff, type 0x1b
>                         = DPL 0, pres 1, def32 1, gran 1
> processor eflags        = resume, IOPL = 0
> current process         = 1213 (httpd)
> trap number             = 12
> panic: page fault
> cpuid = 0
> Uptime: 4d0h17m19s
> Dumping 1023 MB (2 chunks)
>   chunk 0: 1MB (159 pages) ... ok
>   chunk 1: 1023MB (261760 pages) 1007 991 975 959 943 927 911 895 879
> 863 847 831 815 799 783 767 751 735 719 703 687 671 655 639 623 607 591
> 575 559 543 527 511 495 479 463 447 431 415 399 383 367 351 335 319 303
> 287 271 255 239 223 207 191 175 159 143 127 111 95 79 63 47 31 15
>
> #0  doadump () at pcpu.h:165
> 165     pcpu.h: No such file or directory.
>         in pcpu.h
> (kgdb) list*0xc062a143
> 0xc062a143 is at /usr/src/sys/i386/i386/swtch.s:108.
> 103     /usr/src/sys/i386/i386/swtch.s: No such file or directory.
>         in /usr/src/sys/i386/i386/swtch.s
> (kgdb) backtrace
> #0  doadump () at pcpu.h:165
> #1  0xc04e3436 in boot (howto=260)
> at /usr/src/sys/kern/kern_shutdown.c:409
> #2  0xc04e375d in panic (fmt=0xc064447a "%s")
> at /usr/src/sys/kern/kern_shutdown.c:565
> #3  0xc062bd30 in trap_fatal (frame=0xe7063948, eva=661085914)
> at /usr/src/sys/i386/i386/trap.c:837
> #4  0xc062b4e6 in trap (frame>       {tf_fs = 8, tf_es = 40, tf_ds = 40,
tf_edi = -990658024, tf_esi > -992064896, tf_ebp = -419022428, tf_isp =
-419022476, tf_ebx > -999450368, tf_edx = -419021423, tf_ecx = -992064896,
tf_eax > -1068542761, tf_trapno = 12, tf_err = 2, tf_eip = -1067278013, tf_cs
> 32, tf_eflags = 65666, tf_esp = -1068542761, tf_ss = -1068542761})
> at /usr/src/sys/i386/i386/trap.c:270
> #5  0xc061810a in calltrap () at /usr/src/sys/i386/i386/exception.s:139
> #6  0xc062a143 in cpu_switch () at /usr/src/sys/i386/i386/swtch.s:108
> Previous frame inner to this frame (corrupt stack?)
> (kgdb) quit
> root@sbepfkaa [/usr/obj/usr/src/sys/S2468GN] #
>
> _______________________________________________
> freebsd-stable@freebsd.org mailing list
> http://lists.freebsd.org/mailman/listinfo/freebsd-stable
> To unsubscribe, send any mail to
"freebsd-stable-unsubscribe@freebsd.org"