Hmmm, another one. Not sure if this is hardware as it seems different ?
Fatal trap 12: page fault while in kernel mode
cpuid = 11; apic id = 0b
fault virtual address?? = 0x0
fault code????????????? = supervisor write data, page not present
instruction pointer???? = 0x20:0xffffffff80ca0826
stack pointer?????????? = 0x28:0xfffffe00bc0f8540
frame pointer?????????? = 0x28:0xfffffe00bc0f8590
code segment??????????? = base 0x0, limit 0xfffff, type 0x1b
??????????????????????? = DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags??????? = interrupt enabled, resume, IOPL = 0
current process???????? = 33 (dom0)
trap number???????????? = 12
panic: page fault
cpuid = 11
time = 1608641071
KDB: stack backtrace:
#0 0xffffffff80a3fe85 at kdb_backtrace+0x65
#1 0xffffffff809f406b at vpanic+0x17b
#2 0xffffffff809f3ee3 at panic+0x43
#3 0xffffffff80e3fe71 at trap_fatal+0x391
#4 0xffffffff80e3fecf at trap_pfault+0x4f
#5 0xffffffff80e3f516 at trap+0x286
#6 0xffffffff80e19318 at calltrap+0x8
#7 0xffffffff80ca47d4 at bucket_cache_drain+0x134
#8 0xffffffff80c9e302 at zone_drain_wait+0xa2
#9 0xffffffff80ca2bbd at uma_reclaim_locked+0x6d
#10 0xffffffff80ca2af4 at uma_reclaim+0x34
#11 0xffffffff80cc5321 at vm_pageout_worker+0x421
#12 0xffffffff80cc4ee3 at vm_pageout+0x193
#13 0xffffffff809b55be at fork_exit+0x7e
#14 0xffffffff80e1a34e at fork_trampoline+0xe
Uptime: 5d20h37m16s
Dumping 16057 out of 65398
MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91%
__curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
55????????????? __asm("movq %%gs:%P1,%0" : "=r" (td) :
"n"
(offsetof(struct pcpu,
(kgdb) bt
#0? __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
#1? doadump (textdump=<optimized out>) at
/usr/src/sys/kern/kern_shutdown.c:371
#2? 0xffffffff809f3c85 in kern_reboot (howto=260) at
/usr/src/sys/kern/kern_shutdown.c:451
#3? 0xffffffff809f40c3 in vpanic (fmt=<optimized out>, ap=<optimized
out>) at /usr/src/sys/kern/kern_shutdown.c:880
#4? 0xffffffff809f3ee3 in panic (fmt=<unavailable>) at
/usr/src/sys/kern/kern_shutdown.c:807
#5? 0xffffffff80e3fe71 in trap_fatal (frame=0xfffffe00bc0f8480, eva=0)
at /usr/src/sys/amd64/amd64/trap.c:921
#6? 0xffffffff80e3fecf in trap_pfault (frame=0xfffffe00bc0f8480,
usermode=<optimized out>, signo=<optimized out>, ucode=<optimized
out>)
at /usr/src/sys/amd64/amd64/trap.c:739
#7? 0xffffffff80e3f516 in trap (frame=0xfffffe00bc0f8480) at
/usr/src/sys/amd64/amd64/trap.c:405
#8? <signal handler called>
#9? 0xffffffff80ca0826 in slab_free_item (keg=0xfffff800037fa380,
slab=0xfffff80de4656fb0, item=<optimized out>) at
/usr/src/sys/vm/uma_core.c:3357
#10 zone_release (zone=<optimized out>, bucket=0xfffff801c7fd5218,
cnt=<optimized out>) at /usr/src/sys/vm/uma_core.c:3404
#11 0xffffffff80ca47d4 in bucket_drain (zone=0xfffff800037da000,
bucket=0xfffff801c7fd5200) at /usr/src/sys/vm/uma_core.c:758
#12 bucket_cache_drain (zone=0xfffff800037da000) at
/usr/src/sys/vm/uma_core.c:915
#13 0xffffffff80c9e302 in zone_drain_wait (zone=0xfffff800037da000,
waitok=1) at /usr/src/sys/vm/uma_core.c:1037
#14 0xffffffff80ca2bbd in zone_drain (zone=0xfffff800037da000) at
/usr/src/sys/vm/uma_core.c:1056
#15 zone_foreach (zfunc=<optimized out>) at
/usr/src/sys/vm/uma_core.c:1985
#16 uma_reclaim_locked (kmem_danger=<optimized out>) at
/usr/src/sys/vm/uma_core.c:3737
#17 0xffffffff80ca2af4 in uma_reclaim () at /usr/src/sys/vm/uma_core.c:3757
#18 0xffffffff80cc5321 in vm_pageout_lowmem () at
/usr/src/sys/vm/vm_pageout.c:1890
#19 vm_pageout_worker (arg=<optimized out>) at
/usr/src/sys/vm/vm_pageout.c:1966
#20 0xffffffff80cc4ee3 in vm_pageout () at /usr/src/sys/vm/vm_pageout.c:2126
#21 0xffffffff809b55be in fork_exit (callout=0xffffffff80cc4d50
<vm_pageout>, arg=0x0, frame=0xfffffe00bc0f8b00) at
/usr/src/sys/kern/kern_fork.c:1080
#22 <signal handler called>
(kgdb) bt full
#0? __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
??????? td = <optimized out>
#1? doadump (textdump=<optimized out>) at
/usr/src/sys/kern/kern_shutdown.c:371
??????? error = <error reading variable error (Cannot access memory at
address 0x0)>
??????? coredump = <optimized out>
#2? 0xffffffff809f3c85 in kern_reboot (howto=260) at
/usr/src/sys/kern/kern_shutdown.c:451
??????? once = <error reading variable once (Cannot access memory at
address 0x0)>
#3? 0xffffffff809f40c3 in vpanic (fmt=<optimized out>, ap=<optimized
out>) at /usr/src/sys/kern/kern_shutdown.c:880
??????? buf = "page fault", '\000' <repeats 245 times>
??????? other_cpus = {__bits = {2047, 0, 0, 0}}
??????? td = 0xfffff80004964740
??????? newpanic = <error reading variable newpanic (Cannot access
memory at address 0x0)>
??????? bootopt = <unavailable>
#4? 0xffffffff809f3ee3 in panic (fmt=<unavailable>) at
/usr/src/sys/kern/kern_shutdown.c:807
??????? ap = {{gp_offset = 16, fp_offset = 48, overflow_arg_area
0xfffffe00bc0f82c0, reg_save_area = 0xfffffe00bc0f8260}}
#5? 0xffffffff80e3fe71 in trap_fatal (frame=0xfffffe00bc0f8480, eva=0)
at /usr/src/sys/amd64/amd64/trap.c:921
??????? softseg = {ssd_base = 0, ssd_limit = 1048575, ssd_type = 27,
ssd_dpl = 0, ssd_p = 1, ssd_long = 1, ssd_def32 = 0, ssd_gran = 1}
??????? code = <optimized out>
??????? type = <optimized out>
??????? ss = 40
??????? handled = <optimized out>
#6? 0xffffffff80e3fecf in trap_pfault (frame=0xfffffe00bc0f8480,
usermode=<optimized out>, signo=<optimized out>, ucode=<optimized
out>)
at /usr/src/sys/amd64/amd64/trap.c:739
??????? td = 0xfffff80004964740
??????? p = <optimized out>
??????? eva = 0
??????? map = <optimized out>
??????? ftype = <optimized out>
??????? rv = <optimized out>
#7? 0xffffffff80e3f516 in trap (frame=0xfffffe00bc0f8480) at
/usr/src/sys/amd64/amd64/trap.c:405
??????? ksi = {ksi_link = {tqe_next = 0x5fe1ea2f, tqe_prev = 0x1},
ksi_info = {si_signo = 76957504, si_errno = -2048, si_code -1891515752, si_pid =
506236, si_uid = 0, si_status = 0, si_addr = 0x46,
si_value = {sival_int = 0, sival_ptr = 0x0,
????????????? sigval_int = 0, sigval_ptr = 0x0}, _reason = {_fault {_trapno =
-1891515752}, _timer = {_timerid = -1891515752, _overrun 506236}, _mesgq = {_mqd
= -1891515752}, _poll = {_band 2174269467509400}, __spare__ = {
??????????????? __spare1__ = 2174269467509400, __spare2__ {-1139833872, -512,
-2131811593, -1, 1, 0, 0}}}}, ksi_flags 2126206336, ksi_sigq =
0xfffffe00002e4af8}
??????? td = 0xfffff80004964740
??????? dr6 = <error reading variable dr6 (Cannot access memory at
address 0x0)>
??????? p = <optimized out>
??????? type = 12
??????? addr = <optimized out>
??????? signo = <optimized out>
??????? ucode = <optimized out>
??????? pf = <optimized out>
#8? <signal handler called>
No locals.
#9? 0xffffffff80ca0826 in slab_free_item (keg=0xfffff800037fa380,
slab=0xfffff80de4656fb0, item=<optimized out>) at
/usr/src/sys/vm/uma_core.c:3357
??????? dom = <optimized out>
??????? freei = <optimized out>
#10 zone_release (zone=<optimized out>, bucket=0xfffff801c7fd5218,
cnt=<optimized out>) at /usr/src/sys/vm/uma_core.c:3404
??????? clearfull = 0
??????? keg = 0xfffff800037fa380
??????? i = 18
??????? item = <optimized out>
??????? mem = <optimized out>
??????? slab = 0xfffff80de4656fb0
#11 0xffffffff80ca47d4 in bucket_drain (zone=0xfffff800037da000,
bucket=0xfffff801c7fd5200) at /usr/src/sys/vm/uma_core.c:758
??????? i = <optimized out>
#12 bucket_cache_drain (zone=0xfffff800037da000) at
/usr/src/sys/vm/uma_core.c:915
??????? i = 0
??????? bucket = 0xfffff801c7fd5200
??????? zdom = <optimized out>
#13 0xffffffff80c9e302 in zone_drain_wait (zone=0xfffff800037da000,
waitok=1) at /usr/src/sys/vm/uma_core.c:1037
No locals.
#14 0xffffffff80ca2bbd in zone_drain (zone=0xfffff800037da000) at
/usr/src/sys/vm/uma_core.c:1056
--Type <RET> for more, q to quit, c to continue without paging--
No locals.
#15 zone_foreach (zfunc=<optimized out>) at
/usr/src/sys/vm/uma_core.c:1985
??????? keg = 0xfffff800037fa380
??????? zone = 0xfffff800037da000
#16 uma_reclaim_locked (kmem_danger=<optimized out>) at
/usr/src/sys/vm/uma_core.c:3737
No locals.
#17 0xffffffff80ca2af4 in uma_reclaim () at /usr/src/sys/vm/uma_core.c:3757
No locals.
#18 0xffffffff80cc5321 in vm_pageout_lowmem () at
/usr/src/sys/vm/vm_pageout.c:1890
??????? last = <optimized out>
??????? _el = <optimized out>
??????? _ep = <optimized out>
??????? _t = <optimized out>
??????? _v = <optimized out>
??????? _tid = <optimized out>
??????? _v = <optimized out>
??????? _v = <optimized out>
#19 vm_pageout_worker (arg=<optimized out>) at
/usr/src/sys/vm/vm_pageout.c:1966
??????? domain = <optimized out>
??????? target_met = true
??????? shortage = 107
??????? vmd = <optimized out>
??????? ofree = 348095
??????? addl_shortage = <optimized out>
#20 0xffffffff80cc4ee3 in vm_pageout () at /usr/src/sys/vm/vm_pageout.c:2126
??????? td = <optimized out>
??????? p = 0xfffff800049f8530
??????? i = <optimized out>
??????? first = 0
??????? error = <optimized out>
#21 0xffffffff809b55be in fork_exit (callout=0xffffffff80cc4d50
<vm_pageout>, arg=0x0, frame=0xfffffe00bc0f8b00) at
/usr/src/sys/kern/kern_fork.c:1080
??????? td = 0xfffff80004964740
??????? p = 0xfffff800049f8530
??????? dtd = <optimized out>
#22 <signal handler called>
No locals.
(kgdb)
On 12/15/2020 4:39 PM, mike tancsa wrote:> Was doing a backup via zfs send | zfs recv when the box panic'd.? Its a
> not so old RELENG_12 box from last week. Any ideas if this is a hardware
> issue or a bug ? Its r368493 from last Wednesday. I dont see an ECC
> errors logged, so dont think its hardware.
>
> Reading symbols from /boot/kernel/kernel...
> Reading symbols from /usr/lib/debug//boot/kernel/kernel.debug...
>
> Unread portion of the kernel message buffer:
>
>
> Fatal trap 12: page fault while in kernel mode
> cpuid = 1; apic id = 01
> fault virtual address?? = 0x0
> fault code????????????? = supervisor read data, page not present
> instruction pointer???? = 0x20:0xffffffff823a554b
> stack pointer?????????? = 0x28:0xfffffe0343231000
> frame pointer?????????? = 0x28:0xfffffe03432310c0
> code segment??????????? = base 0x0, limit 0xfffff, type 0x1b
> ??????????????????????? = DPL 0, pres 1, long 1, def32 0, gran 1
> processor eflags??????? = interrupt enabled, resume, IOPL = 0
> current process???????? = 87427 (zfs)
> trap number???????????? = 12
> panic: page fault
> cpuid = 1
> time = 1608065221
> KDB: stack backtrace:
> #0 0xffffffff80a3fa05 at kdb_backtrace+0x65
> #1 0xffffffff809f3beb at vpanic+0x17b
> #2 0xffffffff809f3a63 at panic+0x43
> #3 0xffffffff80e400d1 at trap_fatal+0x391
> #4 0xffffffff80e4012f at trap_pfault+0x4f
> #5 0xffffffff80e3f776 at trap+0x286
> #6 0xffffffff80e19568 at calltrap+0x8
> #7 0xffffffff82393a5e at dmu_object_info+0x1e
> #8 0xffffffff823983a5 at dmu_recv_stream+0x7b5
> #9 0xffffffff8244b706 at zfs_ioc_recv+0xac6
> #10 0xffffffff8244dd3d at zfsdev_ioctl+0x62d
> #11 0xffffffff808a35e0 at devfs_ioctl+0xb0
> #12 0xffffffff80f3becb at VOP_IOCTL_APV+0x7b
> #13 0xffffffff80ad1b0a at vn_ioctl+0x16a
> #14 0xffffffff808a3bce at devfs_ioctl_f+0x1e
> #15 0xffffffff80a5d807 at kern_ioctl+0x2b7
> #16 0xffffffff80a5d4aa at sys_ioctl+0xfa
> #17 0xffffffff80e40c87 at amd64_syscall+0x387
> Uptime: 3d14h59m52s
> Dumping 17213 out of 65366
> MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91%
>
> __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
> 55????????????? __asm("movq %%gs:%P1,%0" : "=r" (td) :
"n"
> (offsetof(struct pcpu,
> (kgdb) #0? __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
> #1? doadump (textdump=<optimized out>)
> ??? at /usr/src/sys/kern/kern_shutdown.c:371
> #2? 0xffffffff809f3805 in kern_reboot (howto=260)
> ??? at /usr/src/sys/kern/kern_shutdown.c:451
> #3? 0xffffffff809f3c43 in vpanic (fmt=<optimized out>,
ap=<optimized out>)
> ??? at /usr/src/sys/kern/kern_shutdown.c:880
> #4? 0xffffffff809f3a63 in panic (fmt=<unavailable>)
> ??? at /usr/src/sys/kern/kern_shutdown.c:807
> #5? 0xffffffff80e400d1 in trap_fatal (frame=0xfffffe0343230f40, eva=0)
> ??? at /usr/src/sys/amd64/amd64/trap.c:921
> #6? 0xffffffff80e4012f in trap_pfault (frame=0xfffffe0343230f40,
> ??? usermode=<optimized out>, signo=<optimized out>,
ucode=<optimized out>)
> ??? at /usr/src/sys/amd64/amd64/trap.c:739
> #7? 0xffffffff80e3f776 in trap (frame=0xfffffe0343230f40)
> ??? at /usr/src/sys/amd64/amd64/trap.c:405
> #8? <signal handler called>
> #9? 0xffffffff823a554b in dnode_hold_impl (os=0xfffff805e1d2b800,
> ??? object=<optimized out>, flag=<optimized out>,
slots=<optimized out>,
> ??? tag=<optimized out>, dnp=0xfffffe03432310d8)
> ??? at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c:1370
> #10 0xffffffff82393a5e in dmu_object_info (os=0xfffff80777890070,
> ??? object=18446744071600721588, doi=0xfffffe03432312e0)
> ??? at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c:2615
> #11 0xffffffff823983a5 in receive_read_record (ra=<optimized out>)
> ??? at
> /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_send.c:2821
> #12 dmu_recv_stream (drc=0xfffffe0343231430, fp=<optimized out>,
> ??? voffp=<optimized out>, cleanup_fd=8, action_handlep=<optimized
out>)
> ??? at
> /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_send.c:3203
> #13 0xffffffff8244b706 in zfs_ioc_recv (zc=<optimized out>)
> ??? at
> /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c:4745
> #14 0xffffffff8244dd3d in zfsdev_ioctl (dev=<optimized out>,
> ??? zcmd=<optimized out>, arg=<optimized out>,
flag=<optimized out>,
> ??? td=<optimized out>)
> ??? at
> /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c:6956
> #15 0xffffffff808a35e0 in devfs_ioctl (ap=0xfffffe0343231778)
> ??? at /usr/src/sys/fs/devfs/devfs_vnops.c:797
> #16 0xffffffff80f3becb in VOP_IOCTL_APV (
> ??? vop=0xffffffff816a2fe0 <devfs_specops>, a=0xfffffe0343231778)
> ??? at vnode_if.c:1067
> #17 0xffffffff80ad1b0a in vn_ioctl (fp=0xfffff8001802b5a0,
> ??? com=<optimized out>, data=0xfffffe0343231910,
> ??? active_cred=0xfffff80032214300, td=0x20700000)
> ??? at /usr/src/sys/kern/vfs_vnops.c:1508
> #18 0xffffffff808a3bce in devfs_ioctl_f (fp=0xfffff80777890070,
> ??? com=18446744071600721588, data=0xffffffff824e34ed <.L.str+1>,
cred=0x0,
> ??? td=0xfffff80298850000) at /usr/src/sys/fs/devfs/devfs_vnops.c:755
> #19 0xffffffff80a5d807 in fo_ioctl (fp=0xfffff8001802b5a0, com=3222821403,
> ??? data=0xffffffff824e34ed <.L.str+1>, active_cred=0x0,
> ??? td=0xfffff80298850000) at /usr/src/sys/sys/file.h:337
> #20 kern_ioctl (td=0x20700000, fd=<optimized out>, com=3222821403,
> ??? data=0xffffffff824e34ed <.L.str+1> "zrl->zr_mtx")
> ??? at /usr/src/sys/kern/sys_generic.c:805
> #21 0xffffffff80a5d4aa in sys_ioctl (td=0xfffff80298850000,
> ??? uap=0xfffff802988503c0) at /usr/src/sys/kern/sys_generic.c:713
> #22 0xffffffff80e40c87 in syscallenter (td=0xfffff80298850000)
> ??? at /usr/src/sys/amd64/amd64/../../kern/subr_syscall.c:144
> #23 amd64_syscall (td=0xfffff80298850000, traced=0)
> ??? at /usr/src/sys/amd64/amd64/trap.c:1163
> #24 <signal handler called>
> #25 0x00000008004a8cba in ?? ()
> Backtrace stopped: Cannot access memory at address 0x7fffffffa988
> (kgdb)
>
>
> _______________________________________________
> freebsd-stable at freebsd.org mailing list
> https://lists.freebsd.org/mailman/listinfo/freebsd-stable
> To unsubscribe, send any mail to "freebsd-stable-unsubscribe at
freebsd.org"
>