Hi,
A machine dedicated to network services - firewall, NAT, VPN, IPsec -
crashes from time to time. Most of the crashes had a wildly corrupted
stack, but this morning I was able to capture one red-handed. The OS is
-STABLE (RELENG_4) soon after the RELENG_4_9_RELEASE was tagged on the
other branch:
FreeBSD portal.nanolink.com 4.9-STABLE FreeBSD 4.9-STABLE #4: Tue Oct 28
08:59:40 EET 2003
root@portal.nanolink.com:/usr/obj/usr/src-bsd/4.0S/src/sys/PORTAL i386
The panic backtrace and the kernel config are attached (after the sig);
there is a suspicious-looking null vp in frame #9, vlrureclaim().
G'luck,
Peter
--
Peter Pentchev roam@ringlet.net roam@sbnd.net roam@FreeBSD.org
PGP key: http://people.FreeBSD.org/~roam/roam.key.asc
Key fingerprint FDBA FD79 C26F 3C51 C95E DF9E ED18 B68D 1619 4553
This sentence claims to be an Epimenides paradox, but it is lying.
Script started on Wed Nov 5 08:58:11 2003
Setting up interactive shell params..
[root@portal:p2 /var/crash]# gdb -k
/usr/obj/usr/src-bsd/4.0S/src/sys/PORTAL/kernel.debug vmcore.22
GNU gdb 4.18 (FreeBSD)
Copyright 1998 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for
details.
This GDB was configured as "i386-unknown-freebsd"...Deprecated
bfd_read called at
/usr/src-bsd/4.0S/src/gnu/usr.bin/binutils/gdb/../../../../contrib/gdb/gdb/dbxread.c
line 2627 in elfstab_build_psymtabs
Deprecated bfd_read called at
/usr/src-bsd/4.0S/src/gnu/usr.bin/binutils/gdb/../../../../contrib/gdb/gdb/dbxread.c
line 933 in fill_symbuf
IdlePTD at phsyical address 0x003ad000
initial pcb at physical address 0x00306d40
panicstr: vm_page_remove(): page not found in hash
panic messages:
---
panic: vm_page_remove(): page not found in hash
syncing disks... 128 3 2 2 2 2 2 2 2 19 1 1 1 1 1 1 1 7 1 1 1 1 1 1 1 1 1 1 1 1
1 1 1 1 1 1 1 1
giving up on 1 buffers
Uptime: 4d17h28m34s
dumping to dev #da/0x20001, offset 128
dump 511 510 509 508 507 506 505 504 503 502 501 500 499 498 497 496 495 494 493
492 491 490 489 488 487 486 485 484 483 482 481 480 479 478 477 476 475 474 473
472 471 470 469 468 467 466 465 464 463 462 461 460 459 458 457 456 455 454 453
452 451 450 449 448 447 446 445 444 443 442 441 440 439 438 437 436 435 434 433
432 431 430 429 428 427 426 425 424 423 422 421 420 419 418 417 416 415 414 413
412 411 410 409 408 407 406 405 404 403 402 401 400 399 398 397 396 395 394 393
392 391 390 389 388 387 386 385 384 383 382 381 380 379 378 377 376 375 374 373
372 371 370 369 368 367 366 365 364 363 362 361 360 359 358 357 356 355 354 353
352 351 350 349 348 347 346 345 344 343 342 341 340 339 338 337 336 335 334 333
332 331 330 329 328 327 326 325 324 323 322 321 320 319 318 317 316 315 314 313
312 311 310 309 308 307 306 305 304 303 302 301 300 299 298 297 296 295 294 293
292 291 290 289 288 287 286 285 284 283 282 281 280 279 278 277 276 275 274 273
272 271 270 269 268 267 266 265 264 263 262 261 260 259 258 257 256 255 254 253
252 251 250 249 248 247 246 245 244 243 242 241 240 239 238 237 236 235 234 233
232 231 230 229 228 227 226 225 224 223 222 221 220 219 218 217 216 215 214 213
212 211 210 209 208 207 206 205 204 203 202 201 200 199 198 197 196 195 194 193
192 191 190 189 188 187 186 185 184 183 182 181 180 179 178 177 176 175 174 173
172 171 170 169 168 167 166 165 164 163 162 161 160 159 158 157 156 155 154 153
152 151 150 149 148 147 146 145 144 143 142 141 140 139 138 137 136 135 134 133
132 131 130 129 128 127 126 125 124 123 122 121 120 119 118 117 116 115 114 113
112 111 110 109 108 107 106 105 104 103 102 101 100 99 98 97 96 95 94 93 92 91
90 89 88 87 86 85 84 83 82 81 80 79 78 77 76 75 74 73 72 71 70 69 68 67 66 65 64
63 62 61 60 59 58 57 56 55 54 53 52 51 50 49 48 47 46 45 44 43 42 41 40 39 38 37
36 35 34 33 32 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11 10
9 8 7 6 5 4 3 2 1 0
---
#0 dumpsys () at /usr/src-bsd/4.0S/src/sys/kern/kern_shutdown.c:487
487 if (dumping++) {
(kgdb) bt
#0 dumpsys () at /usr/src-bsd/4.0S/src/sys/kern/kern_shutdown.c:487
#1 0xc0180154 in boot (howto=256)
at /usr/src-bsd/4.0S/src/sys/kern/kern_shutdown.c:316
#2 0xc01805a1 in panic (
fmt=0xc02ca380 "vm_page_remove(): page not found in hash")
at /usr/src-bsd/4.0S/src/sys/kern/kern_shutdown.c:595
#3 0xc025379b in vm_page_remove (m=0xc080b13c)
at /usr/src-bsd/4.0S/src/sys/vm/vm_page.c:461
#4 0xc0253e34 in vm_page_free_toq (m=0xc080b13c)
at /usr/src-bsd/4.0S/src/sys/vm/vm_page.c:1104
#5 0xc0252c0b in vm_object_page_remove (object=0xe4def844, start=0, end=0,
clean_only=1) at /usr/src-bsd/4.0S/src/sys/vm/vm_page.h:514
#6 0xc01aef8a in vinvalbuf (vp=0xe5275900, flags=1, cred=0x0, p=0xde78e440,
slpflag=0, slptimeo=0) at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:903
#7 0xc01b0017 in vclean (vp=0xe5275900, flags=8, p=0xde78e440)
at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:1898
#8 0xc01b023c in vgonel (vp=0xe5275900, p=0xde78e440)
at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:2062
#9 0xc01ae79c in vlrureclaim (mp=0xcd2c5000)
at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:513
#10 0xc01ae882 in vnlru_proc ()
at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:561
(kgdb) fr 10
#10 0xc01ae882 in vnlru_proc ()
at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:561
561 done += vlrureclaim(mp);
(kgdb) info loc
mp = (struct mount *) 0xcd2c5000
nmp = (struct mount *) 0xcd2c5000
done = 31
p = (struct proc *) 0xde78e440
(kgdb) up
Initial frame selected; you cannot go up.
(kgdb) down
#9 0xc01ae79c in vlrureclaim (mp=0xcd2c5000)
at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:513
513 vgonel(vp, curproc);
(kgdb) info loc
mp = (struct mount *) 0xcd2c5000
vp = (struct vnode *) 0x0
done = 630
trigger = 7
usevnodes = 0
count = 1652
(kgdb) down
#8 0xc01b023c in vgonel (vp=0xe5275900, p=0xde78e440)
at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:2062
2062 vclean(vp, DOCLOSE, p);
(kgdb) info loc
vp = (struct vnode *) 0xe5275900
p = (struct proc *) 0x0
s = 0
(kgdb) down
#7 0xc01b0017 in vclean (vp=0xe5275900, flags=8, p=0xde78e440)
at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:1898
1898 vinvalbuf(vp, V_SAVE, NOCRED, p, 0, 0);
(kgdb) info loc
vp = (struct vnode *) 0xe5275900
flags = 0
p = (struct proc *) 0xde78e440
active = 0
(kgdb) down
#6 0xc01aef8a in vinvalbuf (vp=0xe5275900, flags=1, cred=0x0, p=0xde78e440,
slpflag=0, slptimeo=0) at /usr/src-bsd/4.0S/src/sys/kern/vfs_subr.c:903
903 vm_object_page_remove(object, 0, 0,
(kgdb) info loc
cred = (struct ucred *) 0x0
p = (struct proc *) 0x0
bp = (struct buf *) 0xe5275900
nbp = (struct buf *) 0xde78e440
blist = (struct buf *) 0x0
s = 6815808
error = -450406144
object = 0xe4def844
(kgdb) down
#5 0xc0252c0b in vm_object_page_remove (object=0xe4def844, start=0, end=0,
clean_only=1) at /usr/src-bsd/4.0S/src/sys/vm/vm_page.h:514
514 vm_page_free_toq(m);
(kgdb) info loc
m = 0xc080b13c
p = 0xc080b13c
next = 0x0
size = 0
all = 1
(kgdb) down
#4 0xc0253e34 in vm_page_free_toq (m=0xc080b13c)
at /usr/src-bsd/4.0S/src/sys/vm/vm_page.c:1104
1104 vm_page_remove(m);
(kgdb) info loc
m = 0xc080b13c
s = 6815808
pq = (struct vpgqueues *) 0x0
object = 0xe4def844
(kgdb) down
#3 0xc025379b in vm_page_remove (m=0xc080b13c)
at /usr/src-bsd/4.0S/src/sys/vm/vm_page.c:461
461 panic("vm_page_remove(): page not found in hash");
(kgdb) info loc
bucket = (struct vm_page **) 0x0
m = 0xc080b13c
object = 0x0
(kgdb) down
#2 0xc01805a1 in panic (
fmt=0xc02ca380 "vm_page_remove(): page not found in hash")
at /usr/src-bsd/4.0S/src/sys/kern/kern_shutdown.c:595
595 boot(bootopt);
(kgdb) info loc
fmt = 0xc02ca380 "vm_page_remove(): page not found in hash"
bootopt = 256
buf = "vm_page_remove(): page not found in hash", '\000'
<repeats 215 times>
(kgdb) down
#1 0xc0180154 in boot (howto=256)
at /usr/src-bsd/4.0S/src/sys/kern/kern_shutdown.c:316
316 dumpsys();
(kgdb) info loc
howto = 256
(kgdb) down
#0 dumpsys () at /usr/src-bsd/4.0S/src/sys/kern/kern_shutdown.c:487
487 if (dumping++) {
(kgdb) info loc
error = 0
(kgdb) down
Bottom (i.e., innermost) frame selected; you cannot go down.
(kgdb) quit
[root@portal:p2 /var/crash]# exit
exit
Script done on Wed Nov 5 08:59:51 2003
# portal kernel config
#
# For more information on this file, please read the handbook section on
# Kernel Configuration Files:
#
# http://www.freebsd.org/handbook/kernelconfig-config.html
#
# The handbook is also available locally in /usr/share/doc/handbook
# if you've installed the doc distribution, otherwise always see the
# FreeBSD World Wide Web server (http://www.FreeBSD.ORG/) for the
# latest information.
#
# An exhaustive list of options and more detailed explanations of the
# device lines is also present in the ./LINT configuration file. If you are
# in doubt as to the purpose or necessity of a line, check first in LINT.
#
# $FreeBSD: src/sys/i386/conf/GENERIC,v 1.246.2.34 2001/08/12 13:13:46 joerg Exp
$
machine i386
cpu I686_CPU
ident PORTAL
maxusers 32
options MAXDSIZ="(512*1024*1024)"
options DFLDSIZ="(256*1024*1024)"
makeoptions DEBUG="-ggdb -g3" #Build kernel with gdb(1) debug symbols
options DDB
options DDB_UNATTENDED
options FFS #Berkeley Fast Filesystem
options SOFTUPDATES
options FFS_ROOT #FFS usable as root device [keep this!]
options CD9660 #ISO 9660 Filesystem
options PROCFS #Process filesystem
options COMPAT_43 #Compatible with BSD 4.3 [KEEP THIS!]
options SCSI_DELAY=15000 #Delay (in ms) before probing SCSI
options KTRACE #ktrace(1) support
options SYSVSHM #SYSV-style shared memory
options SHMMNI=2048
options SHMSEG=128
options SYSVMSG #SYSV-style message queues
options MSGMNI=2048
options MSGTQL=2048
options SYSVSEM #SYSV-style semaphores
options SEMMNI=128
options SEMMNS=2048
options SEMMNU=1024
options SEMUME=128
options P1003_1B #Posix P1003_1B real-time extentions
options _KPOSIX_PRIORITY_SCHEDULING
options ICMP_BANDLIM #Rate limit bad replies
options INET #InterNETworking
options INET6
options DUMMYNET
options RANDOM_IP_ID
options IPFIREWALL
options IPFIREWALL_VERBOSE
options IPFIREWALL_FORWARD
options IPFIREWALL_VERBOSE_LIMIT=100
options IPFIREWALL_DEFAULT_TO_ACCEPT
options IPV6FIREWALL
options IPV6FIREWALL_VERBOSE
options IPV6FIREWALL_DEFAULT_TO_ACCEPT
options IPFILTER #ipfilter support
options IPFILTER_LOG #ipfilter logging
options NETGRAPH #netgraph(4) system
options NETGRAPH_BPF
options NETGRAPH_IFACE
options NETGRAPH_KSOCKET
options NETGRAPH_MPPC_ENCRYPTION
options NETGRAPH_PPP
options NETGRAPH_PPTPGRE
options NETGRAPH_SOCKET
options NETGRAPH_VJC
pseudo-device gif
pseudo-device faith 1
pseudo-device stf
options IPSEC
options IPSEC_ESP
device isa
device pci
# Floppy drives
device fdc0 at isa? port IO_FD1 irq 6 drq 2
device fd0 at fdc0 drive 0
# SCSI Controllers
device ahc # AHA2940 and onboard AIC7xxx devices
options AHC_ALLOW_MEMIO
# SCSI peripherals
device scbus # SCSI bus (required)
device da # Direct Access (disks)
device cd # CD
# atkbdc0 controls both the keyboard and the PS/2 mouse
device atkbdc0 at isa? port IO_KBD
device atkbd0 at atkbdc? irq 1
device psm0 at atkbdc? irq 12
options ATKBD_DFLT_KEYMAP
makeoptions ATKBD_DFLT_KEYMAP="o1.phonetic.ctrlcaps"
device vga0 at isa?
# splash screen/screen saver
pseudo-device splash
# syscons is the default console driver, resembling an SCO console
device sc0 at isa?
options MAXCONS=16
options SC_MOUSE_CHAR=3
options SC_TWOBUTTON_MOUSE
options SC_DFLT_FONT
makeoptions SC_DFLT_FONT="cp1251"
# Floating point support - do not disable.
device npx0 at nexus? port IO_NPX irq 13
# Serial (COM) ports
device sio0 at isa? port IO_COM1 flags 0x10 irq 4
device sio1 at isa? port IO_COM2 irq 3
# Parallel port
device ppc0 at isa? irq 7
device ppbus # Parallel port bus (required)
device lpt # Printer
# PCI Ethernet NICs that use the common MII bus controller code.
# NOTE: Be sure to keep the 'device miibus' line in order to use these
NICs!
device miibus # MII bus support
device xl # 3Com 3c90x (``Boomerang'', ``Cyclone'')
# Pseudo devices - the number indicates how many units to allocate.
pseudo-device loop # Network loopback
pseudo-device ether # Ethernet support
pseudo-device sl 1 # Kernel SLIP
pseudo-device ppp 1 # Kernel PPP
pseudo-device tun # Packet tunnel.
pseudo-device pty # Pseudo-ttys (telnet etc)
pseudo-device md # Memory "disks"
pseudo-device speaker
pseudo-device snp 4
# The `bpf' pseudo-device enables the Berkeley Packet Filter.
# Be aware of the administrative consequences of enabling this!
pseudo-device bpf #Berkeley packet filter
options SC_HISTORY_SIZE=2000
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 187 bytes
Desc: not available
Url :
http://lists.freebsd.org/pipermail/freebsd-stable/attachments/20031105/81564612/attachment.bin