Hi,
My Soekris firewall just panic'd
Fatal trap 12: page fault while in kernel mode
fault virtual address = 0x14001d
fault code = supervisor write, page not present
instruction pointer = 0x20:0xc06fd2d3
stack pointer = 0x28:0xd63557bc
frame pointer = 0x28:0xd63558a4
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 1869 (tcpdump)
trap number = 12
panic: page fault
KDB: stack backtrace:
db_trace_self_wrapper(c079bbfc,c07fb800,c078d680,d6355660,d6355660,...) at db_tr
ace_self_wrapper+0x26
panic(c078d680,c07b829c,c36d6d24,1,1,...) at panic+0xed
trap_fatal(c2c352d0,140000,2,8,d63556c8,...) at trap_fatal+0x234
trap_pfault(c2b13620,0,c368f460,4,c36d6b00,...) at trap_pfault+0x27a
trap(d635577c) at trap+0x34e
calltrap() at calltrap+0x6
--- trap 0xc, eip = 0xc06fd2d3, esp = 0xd63557bc, ebp = 0xd63558a4 ---
softdep_disk_io_initiation(c2ade474,1000,c3003738,1,c2aab9b8,...) at softdep_dis
k_io_initiation+0xb3
ffs_geom_strategy(c3003738,c2aab9b8,1000,edb000,0,...) at ffs_geom_strategy+0x10
c
ufs_strategy(d6355910,d6355910,c07f3980,c378bbdc,c2aab9b8,...) at ufs_strategy+0
x64
bufstrategy(c378bc9c,c2aab9b8,c368f460,c2aab9b8,c2bd46b4,...) at bufstrategy+0x2
e
bufwrite(c2aab9b8,c2aabb04,20,c368f460,0,...) at bufwrite+0xf4
cluster_wbuild(c378bbdc,4000,3b7,0,8,...) at cluster_wbuild+0x6c9
cluster_write(c378bbdc,c2bd46b4,edc000,0,7f,...) at cluster_write+0x715
ffs_write(d6355bc0,c07091e4,c378bc34,0,c378bc64,...) at ffs_write+0x837
VOP_WRITE_APV(c07e9500,d6355bc0,c368f460,c07a2aec,252,...) at VOP_WRITE_APV+0xa0
vn_write(c2e2adf4,d6355c54,c36d7500,0,c368f460,...) at vn_write+0x26f
dofilewrite(d6355c54,ffffffff,ffffffff,0,c2e2adf4,...) at dofilewrite+0x84
kern_writev(c368f460,4,d6355c54,d6355c74,1,...) at kern_writev+0x58
write(c368f460,d6355cf8,c,c0798e1b,39bfbd8f,...) at write+0x50
syscall(d6355d38) at syscall+0x1b9
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (4, FreeBSD ELF32, write), eip = 0x28354663, esp = 0xbfbfe85c, ebp
0xbfbfe878 ---
Uptime: 11d5h5m49s
Physical memory: 503 MB
Dumping 112 MB: 97 81 65 49 33 17 1
Dump complete
Automatic reboot in 1 seconds - press a key on the console to abort
fsck prevented the box from coming up automatically
/dev/ufs/varlog: PARTIALLY TRUNCATED INODE I=1496123
/dev/ufs/varlog: UNEXPECTED SOFT UPDATE INCONSISTENCY; RUN fsck MANUALLY.
THE FOLLOWING FILE SYSTEM HAD AN UNEXPECTED INCONSISTENCY:
ufs: /dev/ufs/varlog (/var/log)
Anyone seen this before? Any clues?
I think this has happened on this box before (or a similar incident anyway,
I didn't have the console wired up the last time so I didn't get the
above
trace).
I have a crash dump, although since this is a soekris box I don't have
debugging symbols available on the flash card.
Thanks,
Gary
On Thu, Jan 26, 2012 at 09:48:15PM -0500, Gary Palmer wrote:> Hi, > > My Soekris firewall just panic'd > > Fatal trap 12: page fault while in kernel mode > fault virtual address = 0x14001d > fault code = supervisor write, page not present > instruction pointer = 0x20:0xc06fd2d3 > stack pointer = 0x28:0xd63557bc > frame pointer = 0x28:0xd63558a4 > code segment = base 0x0, limit 0xfffff, type 0x1b > = DPL 0, pres 1, def32 1, gran 1 > processor eflags = interrupt enabled, resume, IOPL = 0 > current process = 1869 (tcpdump) > trap number = 12 > panic: page fault > KDB: stack backtrace: > db_trace_self_wrapper(c079bbfc,c07fb800,c078d680,d6355660,d6355660,...) at db_tr > ace_self_wrapper+0x26 > panic(c078d680,c07b829c,c36d6d24,1,1,...) at panic+0xed > trap_fatal(c2c352d0,140000,2,8,d63556c8,...) at trap_fatal+0x234 > trap_pfault(c2b13620,0,c368f460,4,c36d6b00,...) at trap_pfault+0x27a > trap(d635577c) at trap+0x34e > calltrap() at calltrap+0x6 > --- trap 0xc, eip = 0xc06fd2d3, esp = 0xd63557bc, ebp = 0xd63558a4 --- > softdep_disk_io_initiation(c2ade474,1000,c3003738,1,c2aab9b8,...) at softdep_dis > k_io_initiation+0xb3 > ffs_geom_strategy(c3003738,c2aab9b8,1000,edb000,0,...) at ffs_geom_strategy+0x10 > c > ufs_strategy(d6355910,d6355910,c07f3980,c378bbdc,c2aab9b8,...) at ufs_strategy+0 > x64 > bufstrategy(c378bc9c,c2aab9b8,c368f460,c2aab9b8,c2bd46b4,...) at bufstrategy+0x2 > e > bufwrite(c2aab9b8,c2aabb04,20,c368f460,0,...) at bufwrite+0xf4 > cluster_wbuild(c378bbdc,4000,3b7,0,8,...) at cluster_wbuild+0x6c9 > cluster_write(c378bbdc,c2bd46b4,edc000,0,7f,...) at cluster_write+0x715 > ffs_write(d6355bc0,c07091e4,c378bc34,0,c378bc64,...) at ffs_write+0x837 > VOP_WRITE_APV(c07e9500,d6355bc0,c368f460,c07a2aec,252,...) at VOP_WRITE_APV+0xa0 > vn_write(c2e2adf4,d6355c54,c36d7500,0,c368f460,...) at vn_write+0x26f > dofilewrite(d6355c54,ffffffff,ffffffff,0,c2e2adf4,...) at dofilewrite+0x84 > kern_writev(c368f460,4,d6355c54,d6355c74,1,...) at kern_writev+0x58 > write(c368f460,d6355cf8,c,c0798e1b,39bfbd8f,...) at write+0x50 > syscall(d6355d38) at syscall+0x1b9 > Xint0x80_syscall() at Xint0x80_syscall+0x20 > --- syscall (4, FreeBSD ELF32, write), eip = 0x28354663, esp = 0xbfbfe85c, ebp > 0xbfbfe878 --- > Uptime: 11d5h5m49s > Physical memory: 503 MB > Dumping 112 MB: 97 81 65 49 33 17 1 > Dump complete > Automatic reboot in 1 seconds - press a key on the console to abort > > fsck prevented the box from coming up automatically > > /dev/ufs/varlog: PARTIALLY TRUNCATED INODE I=1496123 > /dev/ufs/varlog: UNEXPECTED SOFT UPDATE INCONSISTENCY; RUN fsck MANUALLY. > THE FOLLOWING FILE SYSTEM HAD AN UNEXPECTED INCONSISTENCY: > ufs: /dev/ufs/varlog (/var/log) > > Anyone seen this before? Any clues?Please shed some light as to your filesystem and disk setup. Partitioning details, any GEOM layers you use, you get the idea. The more verbose the better. Please be sure to state which filesystems use softupdates and which do not (this matters a lot in this situation). As for the actual crash -- I assume the filesystem in question which induced this has softupdates. Have you booted this box into single-user and issued a manual fsck on all the filesystems? If not, please do so ASAP and provide details of the results. These are confirmed cases of background_fsck not properly catching/dealing with all filesystem errors, thus letting some transparently pass through. You are running 7.4-p2, which makes this likelihood even more probable. This is why I advocate (and have for years) background_fsck="no" in rc.conf. I can dig up the (very long and semi-heated) thread discussing this if you want, but I probably won't get to it until next week. I have too much going on with job-related things outside of FreeBSD to spend a lot of time with it now. -- | Jeremy Chadwick jdc@parodius.com | | Parodius Networking http://www.parodius.com/ | | UNIX Systems Administrator Mountain View, CA, US | | Making life hard for others since 1977. PGP 4BD6C0CB |
On Thu, Jan 26, 2012 at 09:48:15PM -0500, Gary Palmer wrote:> Hi, > > My Soekris firewall just panic'd >[snip]> I think this has happened on this box before (or a similar incident anyway, > I didn't have the console wired up the last time so I didn't get the above > trace).Just found old daily security e-mail with the last panic Fatal trap 12: page fault while in kernel mode fault virtual address = 0xc34b6a08 fault code = supervisor write, page not present instruction pointer = 0x20:0xc06faa58 stack pointer = 0x28:0xd5096a34 frame pointer = 0x28:0xd5096b18 code segment = base 0x0, limit 0xfffff, type 0x1b = DPL 0, pres 1, def32 1, gran 1 processor eflags = interrupt enabled, resume, IOPL = 0 current process = 38 (syncer) trap number = 12 panic: page fault KDB: stack backtrace: db_trace_self_wrapper(c079803d,c07f6fa0,c0789da0,d50968e0,d50968e0,...) at db_trace_self_wrapper+0x26 panic(c0789da0,c07b423e,c2d087a4,1,1,...) at panic+0xed trap_fatal(c1154000,c34b6000,2,0,c2b5985c,...) at trap_fatal+0x240 trap_pfault(d5096960,d5096964,0,173a,c2d08580,...) at trap_pfault+0x27a trap(d50969f4) at trap+0x35e calltrap() at calltrap+0x6 --- trap 0xc, eip = 0xc06faa58, esp = 0xd5096a34, ebp = 0xd5096b18 --- softdep_disk_io_initiation(c2b96730,1000,c302d84c,9,c2b96730,...) at softdep_disk_io_initiation+0xd8 ffs_geom_strategy(c302d84c,c2b96730,1000,f9fd3000,ffffffff,...) at ffs_geom_strategy+0x13b ufs_strategy(d5096b84,d5096b84,c07ef140,c37e29b4,c2b96730,...) at ufs_strategy+0x64 bufstrategy(c37e2a74,c2b96730,4000,0,c37e29b4,...) at bufstrategy+0x2e bufwrite(c2b96730,ffffffff,4ab2de0,0,d5096c1c,10,8,4000,0,1,d5096c04,ffffe7f4,ffffffff,4000,d5096c24,c0706a53,c37e2a0c,0,4000,0,0) at bufwrite+0xfe vfs_bio_awrite(c2b96730,200012,0,c2cb7d20,0,...) at vfs_bio_awrite+0x51 ffs_syncvnode(c37e29b4,3,c37e2a74,d5096cf0,c05f5083,...) at ffs_syncvnode+0x2c5 ffs_fsync(d5096cd0,d5096cd0,c2cb7d20,c079e9a0,681,...) at ffs_fsync+0x1c sched_sync(0,d5096d38,aa55aa55,aa55aa55,aa55aa55,...) at sched_sync+0x6f3 fork_exit(c05f4990,0,d5096d38) at fork_exit+0x8d fork_trampoline() at fork_trampoline+0x8 Main difference seems to be the above panic came via the syncer but from bufwrite on up the stack is the same Was on 7.3-RELEASE back in July last year. Thanks, Gary