Happened on yesterday's 5.3-STABLE.
It is a mount from SPARC/Solaris serving a ~600GB raw disk to FreeBSD.
On the FreeBSD side were 10 iozone processes looping in automated mode
(-a). Iozone runs fine up to 512K block until the server dies terrible
death. After reboot the filesystem (UFS2 with softupdates) cannot be
mounted or checked:
ravell# fsck -t ufs /dev/ggate0
** /dev/ggate0
Cannot find file system superblock
ioctl (GCINFO): Inappropriate ioctl for device
fsck_ufs: /dev/ggate0: can't read disk label
ravell# dumpfs /dev/ggate0
dumpfs: /dev/ggate0: could not read superblock to fill out disk
I attach some routine DDB checks. Kernel crash file available upon
request (4GB download).
As a side note, would it be hard to eliminate userland pages from the
crashdump to both increase secrecy and shorten crashfile/dump duration?
Can be done in both Solaris and Tru64, would be good to have in FreeBSD too.
Regards, /S
-------------- next part --------------
Fatal trap 12: page fault while in kernel mode
cpuid = 2; apic id = 06
fault virtual address = 0x8a8a9304
fault code = supervisor read, page not present
instruction pointer = 0x8:0xc05a1913
stack pointer = 0x10:0xf3d1793c
frame pointer = 0x10:0xf3d17978
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 62173 (iozone)
[thread pid 62173 tid 100279 ]
Stopped at ffs_alloccg+0x1b: cmpl $0,0x4(%eax,%edx,1)
db>
db> doadump
No such command
db> call doadump
Dumping 3839 MB
16 32 48 64 80 96 112 128 144 160 176 192 208 224 240 256 272 288 304 320 336
352 368 384 400 416 432 448 464 480 496 512 5284
Dump complete
0xf
db> trace
Tracing pid 62173 tid 100279 td 0xc6206e10
ffs_alloccg(c6c46dac,cc304030,a5a5a5ad,a5a5a5a5,4000) at ffs_alloccg+0x1b
ffs_hashalloc(c6c46dac,cc304030,a5a5a5ad,a5a5a5a5,4000) at ffs_hashalloc+0x28
ffs_alloc(c6c46dac,20d,0,a5a5a5ad,a5a5a5a5) at ffs_alloc+0x175
ffs_balloc_ufs2(c6d73108,834000,0,4000,c64eb800) at ffs_balloc_ufs2+0x14b7
ffs_write(f3d17c10) at ffs_write+0x303
vn_write(c77dce58,f3d17c88,c64eb800,0,c6206e10) at vn_write+0x210
dofilewrite(c6206e10,c77dce58,3,8300000,10000) at dofilewrite+0xa8
write(c6206e10,f3d17d14,3,6eb5,292) at write+0x39
syscall(2f,2f,bfbf002f,0,10000) at syscall+0x283
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x280e85f3, esp = 0xbfbfe5dc, ebp =
0xbfbfe6f8 ---
db> ps
pid proc uarea uid ppid pgrp flag stat wmesg wchan cmd
62346 c62608d4 f3dbf000 0 62146 62346 0025002 stop[SUSP] vi
62174 c62601c4 f3d61000 0 62166 62166 0004002 [LOCK Giant c5382540] iozone
62173 c6204710 f3cdb000 0 62163 62163 0004002 [CPU 2] iozone
62172 c61b9a98 f3cbe000 0 62160 62160 0004002 [LOCK Giant c5382540] iozone
62171 c67d0388 f3e60000 0 62159 62159 0004002 [LOCK Giant c5382540] iozone
62170 c5655710 f3b5d000 0 62164 62164 0004002 [LOCK Giant c5382540] iozone
62169 c67d01c4 f3e5f000 0 62167 62167 0004002 [LOCK Giant c5382540] iozone
62168 c547154c f3ac6000 0 62165 62165 0004002 [LOCK Giant c5382540] iozone
62167 c54658d4 f3a9f000 0 62146 62167 0000002 [SLPQ pause 0xc546590c][SLP]
zsh
62166 c5655e20 f3b61000 0 62146 62166 0000002 [SLPQ pause 0xc5655e58][SLP]
zsh
62165 c627ba98 f3e17000 0 62146 62165 0000002 [SLPQ pause 0xc627bad0][SLP]
zsh
62164 c67cde20 f3e54000 0 62146 62164 0000002 [SLPQ pause 0xc67cde58][SLP]
zsh
62163 c627b000 f3d4c000 0 62146 62163 0000002 [SLPQ pause 0xc627b038][SLP]
zsh
62162 c565ac5c f3bcc000 0 62157 62157 0004002 [LOCK Giant c5382540] iozone
62161 c6204c5c f3d1a000 0 62156 62156 0004002 [LOCK Giant c5382540] iozone
62160 c6204388 f3cd9000 0 62146 62160 0000002 [SLPQ pause 0xc62043c0][SLP]
zsh
62159 c67d0000 f3e5e000 0 62146 62159 0000002 [SLPQ pause 0xc67d0038][SLP]
zsh
62158 c67cd54c f3e4f000 0 62155 62155 0004002 [LOCK Giant c5382540] iozone
62157 c54691c4 f3aa4000 0 62146 62157 0000002 [SLPQ pause 0xc54691fc][SLP]
zsh
62156 c62041c4 f3cd8000 0 62146 62156 0000002 [SLPQ pause 0xc62041fc][SLP]
zsh
62155 c56551c4 f3b58000 0 62146 62155 0000002 [SLPQ pause 0xc56551fc][SLP]
zsh
62146 c6207a98 f3d22000 0 62145 62146 0004002 [SLPQ ttyin 0xc5bcdc10][SLP]
zsh
62145 c59fba98 f3c09000 0 1 62145 0000000 [SLPQ select 0xc06a7844][SLP]
screen
62121 c625e8d4 f3d5c000 0 1 62121 0000000 [LOCK Giant c5382540] ggatec
55230 c61ba710 f3cc5000 0 1 55227 0000002 [SLPQ bpf 0xc65b4400][SLP]
arpwatch
46756 c59fbe20 f3c0b000 25 1 46756 0000100 [SLPQ pause 0xc59fbe58][SLP]
sendmail
45868 c67cd000 f3d37000 25 1 45868 0000100 [SLPQ pause 0xc67cd038][SLP]
sendmail
45860 c54711c4 f3ac4000 0 1 45860 0000100 [SLPQ select 0xc06a7844][SLP]
sendmail
39732 c627a388 f3d45000 0 39730 39732 0004002 [SLPQ ttyin 0xc560e610][SLP]
zsh
39730 c59fa1c4 f3bd2000 0 401 39730 0000100 [SLPQ select 0xc06a7844][SLP]
sshd
13920 c5465a98 f3aa0000 0 13856 13920 0004002 [SLPQ ttyin 0xc733f010][SLP]
zsh
13856 c627b54c f3d4f000 0 401 13856 0000100 [SLPQ select 0xc06a7844][SLP]
sshd
43780 c6204a98 f3d19000 0 1 43780 0000000 [SLPQ select 0xc06a7844][SLP]
ntpd
9097 c546554c f3a9d000 0 9095 9097 0004002 [SLPQ ttyin 0xc53d5a10][SLP]
zsh
9095 c61b9c5c f3cbf000 0 401 9095 0000100 [SLPQ select 0xc06a7844][SLP]
sshd
527 c5469710 f3aa7000 70 526 519 0000002 [SLPQ select 0xc06a7844][SLP]
postgres
526 c54bfc5c f3b38000 70 523 519 0000002 [SLPQ select 0xc06a7844][SLP]
postgres
525 c54bfe20 f3b39000 70 523 519 0000002 [SLPQ select 0xc06a7844][SLP]
postgres
523 c59fa710 f3bd5000 70 1 519 0004002 [SLPQ select 0xc06a7844][SLP]
postgres
467 c546954c f3aa6000 0 1 467 0004002 [SLPQ ttyin 0xc53d5410][SLP]
getty
466 c5656e20 f3bc4000 0 1 466 0004002 [SLPQ ttyin 0xc51a2a10][SLP]
getty
465 c54c31c4 f3b3b000 0 1 465 0004002 [SLPQ ttyin 0xc5055210][SLP]
getty
464 c5655a98 f3b5f000 0 1 464 0004002 [SLPQ ttyin 0xc5055410][SLP]
getty
463 c50d7710 e908f000 0 1 463 0004002 [SLPQ ttyin 0xc5055610][SLP]
getty
462 c5469e20 f3aab000 0 1 462 0004002 [SLPQ ttyin 0xc5055810][SLP]
getty
461 c54c3388 f3b3c000 0 1 461 0004002 [SLPQ ttyin 0xc5055a10][SLP]
getty
460 c54651c4 f3a9b000 0 1 460 0004002 [SLPQ ttyin 0xc5055c10][SLP]
getty
459 c544ae20 f3a51000 0 1 459 0004002 [SLPQ ttyin 0xc5153010][SLP]
getty
414 c565a000 f3bc5000 0 1 414 0000000 [SLPQ nanslp 0xc06a20ec][SLP]
cron
401 c54708d4 f3abf000 0 1 401 0000100 [SLPQ select 0xc06a7844][SLP]
sshd
292 c54c3c5c f3b41000 0 1 292 0000000 [SLPQ select 0xc06a7844][SLP]
syslogd
272 c5469c5c f3aaa000 0 1 272 0000000 [SLPQ select 0xc06a7844][SLP]
devd
86 c50d7e20 e9093000 0 0 0 0000204 [SLPQ - 0xe902dd18][SLP]
schedcpu
85 c5385000 f17b2000 0 0 0 0000204 [SLPQ - 0xc06aa34c][SLP] nfsiod
3
84 c53851c4 f17d1000 0 0 0 0000204 [SLPQ - 0xc06aa348][SLP] nfsiod
2
83 c5385388 f17d2000 0 0 0 0000204 [SLPQ - 0xc06aa344][SLP] nfsiod
1
82 c538554c f17d3000 0 0 0 0000204 [SLPQ - 0xc06aa340][SLP] nfsiod
0
81 c5385710 f17d4000 0 0 0 0000204 [SLPQ syncer 0xc06a1e6c][SLP]
syncer
80 c53858d4 f17d5000 0 0 0 0000204 [SLPQ vlruwt 0xc53858d4][SLP]
vnlru
79 c5385a98 f17d6000 0 0 0 0000204 [SLPQ psleep 0xc06a7e0c][SLP]
bufdaemon
78 c5385c5c f17d7000 0 0 0 000020c [SLPQ pgzero 0xc06b0c54][SLP]
pagezero
77 c5385e20 f17d8000 0 0 0 0000204 [SLPQ psleep 0xc06b0ca8][SLP]
vmdaemon
76 c5387000 f17d9000 0 0 0 0000204 [SLPQ psleep 0xc06b0c64][SLP]
pagedaemon
75 c508ea98 e9043000 0 0 0 0000204 [IWAIT] swi0: sio
74 c508ec5c e9044000 0 0 0 0000204 [SLPQ - 0xc509b53c][SLP] fdc0
73 c508ee20 e9045000 0 0 0 0000204 [SLPQ aifthd 0xc508ee20][SLP]
aac0aif
9 c50d3000 e9046000 0 0 0 0000204 [SLPQ actask 0xc078ea2c][SLP]
acpi_task2
8 c50d31c4 e9047000 0 0 0 0000204 [SLPQ actask 0xc078ea2c][SLP]
acpi_task1
7 c50d3388 e9048000 0 0 0 0000204 [SLPQ actask 0xc078ea2c][SLP]
acpi_task0
6 c50d354c e9085000 0 0 0 0000204 [SLPQ - 0xc50fb2c0][SLP] thread
taskq
72 c50d3710 e9086000 0 0 0 0000204 [IWAIT] swi6:+
71 c50d38d4 e9087000 0 0 0 0000204 [IWAIT] swi6: acpitaskq
70 c50d3a98 e9088000 0 0 0 0000204 [IWAIT] swi6: task queue
5 c50d3c5c e9089000 0 0 0 0000204 [SLPQ - 0xc50fb480][SLP] kqueue
taskq
69 c50d3e20 e908a000 0 0 0 0000204 [IWAIT] swi3: cambio
68 c50d7000 e908b000 0 0 0 0000204 [IWAIT] swi2: camnet
67 c50d71c4 e908c000 0 0 0 0000204 [IWAIT] swi6:+
66 c50d7388 e908d000 0 0 0 0000204 [SLPQ - 0xc0699a60][SLP] yarrow
4 c507c1c4 e8ff9000 0 0 0 0000204 [SLPQ - 0xc069c6a8][SLP] g_down
3 c507c388 e8ffa000 0 0 0 0000204 [SLPQ - 0xc069c6a4][SLP] g_up
2 c507c54c e8ffb000 0 0 0 0000204 [SLPQ - 0xc069c69c][SLP]
g_event
65 c507c710 e8ffc000 0 0 0 0000204 [IWAIT] swi1: net
64 c507c8d4 e8ffd000 0 0 0 0000204 [IWAIT] swi4: vm
63 c507ca98 e903a000 0 0 0 000020c [LOCK Giant c5382540] swi5:
clock sio
62 c507cc5c e903b000 0 0 0 0000204 [IWAIT] irq0: clk
61 c507ce20 e903c000 0 0 0 0000204 [IWAIT] irq47:
60 c508e000 e903d000 0 0 0 0000204 [IWAIT] irq46:
59 c508e1c4 e903e000 0 0 0 0000204 [IWAIT] irq45:
58 c508e388 e903f000 0 0 0 0000204 [IWAIT] irq44:
57 c508e54c e9040000 0 0 0 0000204 [IWAIT] irq43:
56 c508e710 e9041000 0 0 0 0000204 [IWAIT] irq42:
55 c508e8d4 e9042000 0 0 0 0000204 [IWAIT] irq41:
54 c5067a98 e8fce000 0 0 0 0000204 [IWAIT] irq40:
53 c5067c5c e8fcf000 0 0 0 0000204 [IWAIT] irq39:
52 c5067e20 e8fd0000 0 0 0 0000204 [IWAIT] irq38:
51 c5079000 e8fd1000 0 0 0 0000204 [IWAIT] irq37:
50 c50791c4 e8ff0000 0 0 0 0000204 [IWAIT] irq36:
49 c5079388 e8ff1000 0 0 0 0000204 [IWAIT] irq35:
48 c507954c e8ff2000 0 0 0 0000204 [IWAIT] irq34:
47 c5079710 e8ff3000 0 0 0 0000204 [IWAIT] irq33:
46 c50798d4 e8ff4000 0 0 0 0000204 [IWAIT] irq32:
45 c5079a98 e8ff5000 0 0 0 0000204 [IWAIT] irq31:
44 c5079c5c e8ff6000 0 0 0 0000204 [IWAIT] irq30: aac0
43 c5079e20 e8ff7000 0 0 0 0000204 [IWAIT] irq29: bge1
42 c507c000 e8ff8000 0 0 0 0000204 [IWAIT] irq28: bge0
41 c505d54c e8fa4000 0 0 0 0000204 [IWAIT] irq27:
40 c505d710 e8fa5000 0 0 0 0000204 [IWAIT] irq26:
39 c505d8d4 e8fa6000 0 0 0 0000204 [IWAIT] irq25:
38 c505da98 e8fc5000 0 0 0 0000204 [IWAIT] irq24:
37 c505dc5c e8fc6000 0 0 0 0000204 [IWAIT] irq23:
36 c505de20 e8fc7000 0 0 0 0000204 [IWAIT] irq22:
35 c5067000 e8fc8000 0 0 0 0000204 [IWAIT] irq21:
34 c50671c4 e8fc9000 0 0 0 0000204 [IWAIT] irq20:
33 c5067388 e8fca000 0 0 0 0000204 [IWAIT] irq19:
32 c506754c e8fcb000 0 0 0 0000204 [IWAIT] irq18:
31 c5067710 e8fcc000 0 0 0 0000204 [IWAIT] irq17:
30 c50678d4 e8fcd000 0 0 0 0000204 [IWAIT] irq16:
29 c50091c4 e7559000 0 0 0 0000204 [IWAIT] irq15: ata1
28 c5009388 e755a000 0 0 0 0000204 [IWAIT] irq14: ata0
27 c500954c e7579000 0 0 0 0000204 [IWAIT] irq13:
26 c5009710 e757a000 0 0 0 0000204 [IWAIT] irq12: psm0
25 c50098d4 e757b000 0 0 0 0000204 [IWAIT] irq11:
24 c5009a98 e757c000 0 0 0 0000204 [IWAIT] irq10:
23 c5009c5c e757d000 0 0 0 0000204 [IWAIT] irq9: acpi0
22 c5009e20 e757e000 0 0 0 0000204 [IWAIT] irq8: rtc
21 c505d000 e8fa1000 0 0 0 0000204 [IWAIT] irq7:
20 c505d1c4 e8fa2000 0 0 0 0000204 [IWAIT] irq6: fdc0
19 c505d388 e8fa3000 0 0 0 0000204 [IWAIT] irq5:
18 c4fff000 e7513000 0 0 0 0000204 [IWAIT] irq4: sio0
17 c4fff1c4 e7550000 0 0 0 0000204 [IWAIT] irq3: sio1
16 c4fff388 e7551000 0 0 0 0000204 [IWAIT] irq2:
15 c4fff54c e7552000 0 0 0 0000204 [IWAIT] irq1: atkbd0
14 c4fff710 e7553000 0 0 0 000020c [CPU 0] idle: cpu0
13 c4fff8d4 e7554000 0 0 0 000020c [CPU 1] idle: cpu1
12 c4fffa98 e7555000 0 0 0 000020c [Can run] idle: cpu2
11 c4fffc5c e7556000 0 0 0 000020c [CPU 3] idle: cpu3
1 c4fffe20 e7557000 0 0 1 0004200 [SLPQ wait 0xc4fffe20][SLP]
init
10 c5009000 e7558000 0 0 0 0000204 [SLPQ ktrace 0xc06a0098][SLP]
ktrace
0 c069c7a0 c081f000 0 0 0 0000200 [SLPQ sched 0xc069c7a0][SLP]
swapper
db> help
print p examine x search set write
w delete d break dwatch watch dhwatch
hwatch step s continue c until next
match trace where call show ps gdb
reset kill watchdog thread panic
db> call sync
forward_wakeup: Idle processor not found
panic: pmap_invalidate_range: interrupts disabled
cpuid = 2
KDB: stack backtrace:
kdb_backtrace(e270ff5c,668bc042,ffc06014,c6206e10,f5) at kdb_backtrace+0x29
mi_switch(1,0,f3d17440,c04d89d8,c6206e10) at mi_switch+0xf6
sched_bind(c6206e10,0) at sched_bind+0x44
boot(104,104,c6206e10,dea1a000,dea1e000) at boot+0x64
panic(c06558ab,c0632bf0,dea1a000,4,d92fc71c) at panic+0x1b9
pmap_invalidate_range(c06c99a0,dea1a000,dea1e000,c6aadd68,f3d174b0) at
pmap_invalidate_range+0x2b
pmap_qremove(dea1a000,4) at pmap_qremove+0x3d
vfs_vmio_release(d92fc71c) at vfs_vmio_release+0x19c
getnewbuf(0,0,4000,4000,c6028000) at getnewbuf+0x2b6
getblk(c6cea528,6e30140,0,4000,0) at getblk+0x409
breadn(c6cea528,6e30140,0,4000,0) at breadn+0x31
bread(c6cea528,6e30140,0,4000,0) at bread+0x20
ffs_update(c6c2ea50,0,4,8000,0) at ffs_update+0x224
ffs_fsync(f3d1767c) at ffs_fsync+0x453
ffs_sync(c51a7c00,2,0,0,c51a7c00) at ffs_sync+0x209
sync(0,0,0,0,0,0,0,0,0,0) at sync+0xcb
db_fncall(0,0,f3d17780,f3d17768,0) at db_fncall+0xf6
db_command(c06986c4,c06609c0,c065ace0,c065ace4,c0639140) at db_command+0x264
db_command_loop(0,0,f3d17820,f3d1780c,f3d17854) at db_command_loop+0x5c
db_trap(c,0,10,f3d178fc,f3d1789c) at db_trap+0xdd
kdb_trap(c,0,f3d178fc) at kdb_trap+0x8b
trap_fatal(f3d178fc,8a8a9304,c6206e10,c5ac5960,c) at trap_fatal+0x279
trap_pfault(f3d178fc,0,8a8a9304) at trap_pfault+0x1d7
trap(18,2000010,10,1,c6b06000) at trap+0x2f1
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0xc05a1913, esp = 0xf3d1793c, ebp = 0xf3d17978 ---
ffs_alloccg(c6c46dac,cc304030,a5a5a5ad,a5a5a5a5,4000) at ffs_alloccg+0x1b
ffs_hashalloc(c6c46dac,cc304030,a5a5a5ad,a5a5a5a5,4000) at ffs_hashalloc+0x28
ffs_alloc(c6c46dac,20d,0,a5a5a5ad,a5a5a5a5) at ffs_alloc+0x175
ffs_balloc_ufs2(c6d73108,834000,0,4000,c64eb800) at ffs_balloc_ufs2+0x14b7
ffs_write(f3d17c10) at ffs_write+0x303
vn_write(c77dce58,f3d17c88,c64eb800,0,c6206e10) at vn_write+0x210
dofilewrite(c6206e10,c77dce58,3,8300000,10000) at dofilewrite+0xa8
write(c6206e10,f3d17d14,3,6eb5,292) at write+0x39
syscall(2f,2f,bfbf002f,0,10000) at syscall+0x283
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0x280e85f3, esp = 0xbfbfe5dc, ebp =
0xbfbfe6f8 ---
db> re
cpu_reset called on cpu#2
cpu_reset: Restarting BSP
cpu_reset_proxy: Stopped CPU 2