Not sure what all to do, but doing a 'gdb -k kernel.debug /dev/mem', a
backtrack on one of the processes shows ... server has been up 12 days
now, and running a June 8th kernel ...
#0 0x20f4f0 in ?? ()
(kgdb) proc 67258
(kgdb) bt
#0 mi_switch () at machine/globals.h:119
#1 0x8014a1f9 in tsleep (ident=0x8a3d2200, priority=8, wmesg=0x80263d4a
"inode", timo=0) at /usr/src/sys/kern/kern_synch.c:479
#2 0x80141507 in acquire (lkp=0x8a3d2200, extflags=16777280, wanted=1536) at
/usr/src/sys/kern/kern_lock.c:147
#3 0x8014179c in lockmgr (lkp=0x8a3d2200, flags=16973826, interlkp=0xbdc27b6c,
p=0xc26fa9c0) at /usr/src/sys/kern/kern_lock.c:355
#4 0x80172b73 in vop_stdlock (ap=0xc2a95d1c) at
/usr/src/sys/kern/vfs_default.c:256
#5 0x801f0955 in ufs_vnoperate (ap=0xc2a95d1c) at
/usr/src/sys/ufs/ufs/ufs_vnops.c:2376
#6 0x8017d450 in vn_lock (vp=0xbdc27b00, flags=131074, p=0xc26fa9c0) at
vnode_if.h:861
#7 0x80173b1c in lookup (ndp=0xc2a95ec4) at /usr/src/sys/kern/vfs_lookup.c:301
#8 0x80173824 in namei (ndp=0xc2a95ec4) at /usr/src/sys/kern/vfs_lookup.c:153
#9 0x8017c867 in vn_open (ndp=0xc2a95ec4, fmode=1, cmode=420) at
/usr/src/sys/kern/vfs_vnops.c:138
#10 0x8017899c in open (p=0xc26fa9c0, uap=0xc2a95f80) at
/usr/src/sys/kern/vfs_syscalls.c:1029
#11 0x8023c485 in syscall2 (frame={tf_fs = 47, tf_es = 47, tf_ds = 2143223855,
tf_edi = 4, tf_esi = 672253248, tf_ebp = 2143284976, tf_isp = -1029087276,
tf_ebx = 672178060, tf_edx = 672253248,
tf_ecx = 16, tf_eax = 5, tf_trapno = 12, tf_err = 2, tf_eip = 672086720,
tf_cs = 31, tf_eflags = 535, tf_esp = 2143284932, tf_ss = 47}) at
/usr/src/sys/i386/i386/trap.c:1175
#12 0x802299ab in Xint0x80_syscall ()
#13 0x2807ac6c in ?? ()
#14 0x2807abf0 in ?? ()
#15 0x804acbd in ?? ()
#16 0x8049e1e in ?? ()
and another one:
(kgdb) proc 66479
(kgdb) bt
#0 mi_switch () at machine/globals.h:119
#1 0x8014a1f9 in tsleep (ident=0x8a3d2200, priority=8, wmesg=0x80263d4a
"inode", timo=0) at /usr/src/sys/kern/kern_synch.c:479
#2 0x80141507 in acquire (lkp=0x8a3d2200, extflags=16777280, wanted=1536) at
/usr/src/sys/kern/kern_lock.c:147
#3 0x8014179c in lockmgr (lkp=0x8a3d2200, flags=16973826, interlkp=0xbdc27b6c,
p=0xbcc74400) at /usr/src/sys/kern/kern_lock.c:355
#4 0x80172b73 in vop_stdlock (ap=0xbcebdcc8) at
/usr/src/sys/kern/vfs_default.c:256
#5 0x801f0955 in ufs_vnoperate (ap=0xbcebdcc8) at
/usr/src/sys/ufs/ufs/ufs_vnops.c:2376
#6 0x8017d450 in vn_lock (vp=0xbdc27b00, flags=131074, p=0xbcc74400) at
vnode_if.h:861
#7 0x80173b1c in lookup (ndp=0xbcebdedc) at /usr/src/sys/kern/vfs_lookup.c:301
#8 0x80173824 in namei (ndp=0xbcebdedc) at /usr/src/sys/kern/vfs_lookup.c:153
#9 0x8013e1f6 in execve (p=0xbcc74400, uap=0xbcebdf80) at
/usr/src/sys/kern/kern_exec.c:165
#10 0x8023c485 in syscall2 (frame={tf_fs = 2143223855, tf_es = 47, tf_ds =
2143223855, tf_edi = 134582432, tf_esi = 4, tf_ebp = 2143286828, tf_isp =
-1125392428, tf_ebx = 673558412,
tf_edx = 2143286892, tf_ecx = 134594560, tf_eax = 59, tf_trapno = 12,
tf_err = 2, tf_eip = 673241568, tf_cs = 31, tf_eflags = 531, tf_esp =
2143286784, tf_ss = 47})
at /usr/src/sys/i386/i386/trap.c:1175
#11 0x802299ab in Xint0x80_syscall ()
cannot open /proc/66479/mem
(kgdb)
vnodes, kvm, mallocs all appear to be normal:
Jun 23 12:33:00 neptune root: debug.numvnodes: 355550 - debug.freevnodes: 118314
- debug.vnlru_nowhere: 61 - vlruwt
Jun 23 12:33:00 neptune root:
Jun 23 12:33:00 neptune root: unpath 40495 712K 1261K204800K 15310411
0 0 16,32,64
Jun 23 12:33:00 neptune root: temp 61298 7542K 11330K204800K 81655865
0 0 16,32,64,128,256,512,1K,2K,4K,8K,16K,64K,128K
Jun 23 12:33:00 neptune root:
Jun 23 12:33:00 neptune root: vm.kvm_size: 2139095040
Jun 23 12:33:00 neptune root: vm.kvm_free: 822083584
Jun 23 12:33:00 neptune root:
Jun 23 12:33:00 neptune root: 567/1748/16384 mbuf clusters in use
(current/peak/max)