Marcel Moolenaar
2009-Sep-25 18:39 UTC
8.0-RC1: kernel page fault in NLM master thread (VIMAGE or ZFS related?)
All,
I just got this overnight on my server:
Fatal trap 12: page fault while in kernel mode
fault virtual address = 0x90
fault code = supervisor read, page not present
instruction pointer = 0x20:0xc05ba39d
stack pointer = 0x28:0xf31077bc
frame pointer = 0x28:0xf31077c8
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 928 (NLM: master)
(kgdb) bt
#0 doadump () at pcpu.h:246
#1 0xc05e03f3 in boot (howto=260) at /zmirror/nfs/freebsd/base/stable/
8/sys/kern/kern_shutdown.c:416
#2 0xc05e062d in panic (fmt=Variable "fmt" is not available.
) at /zmirror/nfs/freebsd/base/stable/8/sys/kern/kern_shutdown.c:579
#3 0xc04ac807 in db_panic (addr=Could not find the frame base for
"db_panic".
) at /zmirror/nfs/freebsd/base/stable/8/sys/ddb/db_command.c:478
#4 0xc04acd91 in db_command (last_cmdp=0xc0881c3c, cmd_table=0x0,
dopager=1) at /zmirror/nfs/freebsd/base/stable/8/sys/ddb/db_command.c:
445
#5 0xc04aceea in db_command_loop () at /zmirror/nfs/freebsd/base/
stable/8/sys/ddb/db_command.c:498
#6 0xc04aed5d in db_trap (type=12, code=0) at /zmirror/nfs/freebsd/
base/stable/8/sys/ddb/db_main.c:229
#7 0xc0608a14 in kdb_trap (type=12, code=0, tf=0xf310777c) at /
zmirror/nfs/freebsd/base/stable/8/sys/kern/subr_kdb.c:535
#8 0xc07c53af in trap_fatal (frame=0xf310777c, eva=144) at /zmirror/
nfs/freebsd/base/stable/8/sys/i386/i386/trap.c:924
#9 0xc07c5650 in trap_pfault (frame=0xf310777c, usermode=0, eva=144)
at /zmirror/nfs/freebsd/base/stable/8/sys/i386/i386/trap.c:846
#10 0xc07c5ff2 in trap (frame=0xf310777c) at /zmirror/nfs/freebsd/base/
stable/8/sys/i386/i386/trap.c:528
#11 0xc07ac50b in calltrap () at /zmirror/nfs/freebsd/base/stable/8/
sys/i386/i386/exception.s:165
#12 0xc05ba39d in prison_priv_check (cred=0xc61e4880, priv=334) at /
zmirror/nfs/freebsd/base/stable/8/sys/kern/kern_jail.c:3568
#13 0xc05d39ee in priv_check_cred (cred=0xc61e4880, priv=334, flags=0)
at /zmirror/nfs/freebsd/base/stable/8/sys/kern/kern_priv.c:92
#14 0xc09dbffc in secpolicy_fs_owner (mp=0xc4112284, cred=0xc61e4880)
at /zmirror/nfs/freebsd/base/stable/8/sys/modules/zfs/../../cddl/
compat/opensolaris/kern/opensolaris_policy.c:86
#15 0xc09dc527 in secpolicy_vnode_access (cred=0xc61e4880,
vp=0xc4bb6d9c, owner=501, accmode=128)
at /zmirror/nfs/freebsd/base/stable/8/sys/modules/zfs/../../cddl/
compat/opensolaris/kern/opensolaris_policy.c:125
#16 0xc0a56c5c in zfs_zaccess (zp=0xd4be8658, mode=2, flags=Variable
"flags" is not available.
) at /zmirror/nfs/freebsd/base/stable/8/sys/modules/zfs/../../cddl/
contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c:2445
#17 0xc0a56edb in zfs_zaccess_rwx (zp=0xd4be8658, mode=Variable "mode"
is not available.
) at /zmirror/nfs/freebsd/base/stable/8/sys/modules/zfs/../../cddl/
contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c:2484
#18 0xc0a6bfa4 in zfs_freebsd_access (ap=0xf31078d4) at /zmirror/nfs/
freebsd/base/stable/8/sys/modules/zfs/../../cddl/contrib/opensolaris/
uts/common/fs/zfs/zfs_vnops.c:1068
#19 0xc07cfeb2 in VOP_ACCESS_APV (vop=0xc0acfac0, a=0xf31078d4) at
vnode_if.c:571
#20 0xc0718c93 in nlm_get_vfs_state (host=Variable "host" is not
available.
) at vnode_if.h:254
#21 0xc0718e30 in nlm_do_unlock (argp=0xf31079c8, result=0xf3107a08,
rqstp=0xcb199800, rpcp=0x0) at /zmirror/nfs/freebsd/base/stable/8/sys/
nlm/nlm_prot_impl.c:2227
#22 0xc071ac87 in nlm4_unlock_4_svc (argp=0xf31079c8,
result=0xf3107a08, rqstp=0xcb199800) at /zmirror/nfs/freebsd/base/
stable/8/sys/nlm/nlm_prot_server.c:540
#23 0xc071bce3 in nlm_prog_4 (rqstp=0xcb199800, transp=0xc652de00) at /
zmirror/nfs/freebsd/base/stable/8/sys/nlm/nlm_prot_svc.c:512
#24 0xc07284bf in svc_run_internal (pool=0xc61e4c80, ismaster=1) at /
zmirror/nfs/freebsd/base/stable/8/sys/rpc/svc.c:893
#25 0xc072943d in svc_run (pool=0xc61e4c80) at /zmirror/nfs/freebsd/
base/stable/8/sys/rpc/svc.c:1233
#26 0xc071a348 in nlm_syscall (td=0xc6551000, uap=0xf3107cf8) at /
zmirror/nfs/freebsd/base/stable/8/sys/nlm/nlm_prot_impl.c:1593
#27 0xc07c5977 in syscall (frame=0xf3107d38) at /zmirror/nfs/freebsd/
base/stable/8/sys/i386/i386/trap.c:1073
#28 0xc07ac570 in Xint0x80_syscall () at /zmirror/nfs/freebsd/base/
stable/8/sys/i386/i386/exception.s:261
#29 0x00000033 in ?? ()
(kgdb) frame 12
#12 0xc05ba39d in prison_priv_check (cred=0xc61e4880, priv=334) at /
zmirror/nfs/freebsd/base/stable/8/sys/kern/kern_jail.c:3568
3568 switch (priv) {
(kgdb) l 3567
3562 */
3563 if (cred->cr_prison->pr_flags & PR_VNET)
3564 return (0);
3565 }
3566 #endif /* VIMAGE */
3567
3568 switch (priv) {
3569
3570 /*
3571 * Allow ktrace privileges for root in jail.
(kgdb) p cred->cr_prison
$4 = (struct prison *) 0x0
--
Marcel Moolenaar
xcllnt@mac.com
Jamie Gritton
2009-Sep-25 23:13 UTC
8.0-RC1: kernel page fault in NLM master thread (VIMAGE or ZFS related?)
Marcel Moolenaar wrote:> All, > > I just got this overnight on my server: > > Fatal trap 12: page fault while in kernel mode > fault virtual address = 0x90 > fault code = supervisor read, page not present > instruction pointer = 0x20:0xc05ba39d > stack pointer = 0x28:0xf31077bc > frame pointer = 0x28:0xf31077c8 > code segment = base 0x0, limit 0xfffff, type 0x1b > = DPL 0, pres 1, def32 1, gran 1 > processor eflags = interrupt enabled, resume, IOPL = 0 > current process = 928 (NLM: master) > > (kgdb) bt > #0 doadump () at pcpu.h:246 > #1 0xc05e03f3 in boot (howto=260) at > /zmirror/nfs/freebsd/base/stable/8/sys/kern/kern_shutdown.c:416 > #2 0xc05e062d in panic (fmt=Variable "fmt" is not available. > ) at /zmirror/nfs/freebsd/base/stable/8/sys/kern/kern_shutdown.c:579 > #3 0xc04ac807 in db_panic (addr=Could not find the frame base for > "db_panic". > ) at /zmirror/nfs/freebsd/base/stable/8/sys/ddb/db_command.c:478 > #4 0xc04acd91 in db_command (last_cmdp=0xc0881c3c, cmd_table=0x0, > dopager=1) at /zmirror/nfs/freebsd/base/stable/8/sys/ddb/db_command.c:445 > #5 0xc04aceea in db_command_loop () at > /zmirror/nfs/freebsd/base/stable/8/sys/ddb/db_command.c:498 > #6 0xc04aed5d in db_trap (type=12, code=0) at > /zmirror/nfs/freebsd/base/stable/8/sys/ddb/db_main.c:229 > #7 0xc0608a14 in kdb_trap (type=12, code=0, tf=0xf310777c) at > /zmirror/nfs/freebsd/base/stable/8/sys/kern/subr_kdb.c:535 > #8 0xc07c53af in trap_fatal (frame=0xf310777c, eva=144) at > /zmirror/nfs/freebsd/base/stable/8/sys/i386/i386/trap.c:924 > #9 0xc07c5650 in trap_pfault (frame=0xf310777c, usermode=0, eva=144) at > /zmirror/nfs/freebsd/base/stable/8/sys/i386/i386/trap.c:846 > #10 0xc07c5ff2 in trap (frame=0xf310777c) at > /zmirror/nfs/freebsd/base/stable/8/sys/i386/i386/trap.c:528 > #11 0xc07ac50b in calltrap () at > /zmirror/nfs/freebsd/base/stable/8/sys/i386/i386/exception.s:165 > #12 0xc05ba39d in prison_priv_check (cred=0xc61e4880, priv=334) at > /zmirror/nfs/freebsd/base/stable/8/sys/kern/kern_jail.c:3568 > #13 0xc05d39ee in priv_check_cred (cred=0xc61e4880, priv=334, flags=0) > at /zmirror/nfs/freebsd/base/stable/8/sys/kern/kern_priv.c:92 > #14 0xc09dbffc in secpolicy_fs_owner (mp=0xc4112284, cred=0xc61e4880) at > /zmirror/nfs/freebsd/base/stable/8/sys/modules/zfs/../../cddl/compat/opensolaris/kern/opensolaris_policy.c:86 > > #15 0xc09dc527 in secpolicy_vnode_access (cred=0xc61e4880, > vp=0xc4bb6d9c, owner=501, accmode=128) > at > /zmirror/nfs/freebsd/base/stable/8/sys/modules/zfs/../../cddl/compat/opensolaris/kern/opensolaris_policy.c:125 > > #16 0xc0a56c5c in zfs_zaccess (zp=0xd4be8658, mode=2, flags=Variable > "flags" is not available. > ) at > /zmirror/nfs/freebsd/base/stable/8/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c:2445 > > #17 0xc0a56edb in zfs_zaccess_rwx (zp=0xd4be8658, mode=Variable "mode" > is not available. > ) at > /zmirror/nfs/freebsd/base/stable/8/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c:2484 > > #18 0xc0a6bfa4 in zfs_freebsd_access (ap=0xf31078d4) at > /zmirror/nfs/freebsd/base/stable/8/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c:1068 > > #19 0xc07cfeb2 in VOP_ACCESS_APV (vop=0xc0acfac0, a=0xf31078d4) at > vnode_if.c:571 > #20 0xc0718c93 in nlm_get_vfs_state (host=Variable "host" is not available. > ) at vnode_if.h:254 > #21 0xc0718e30 in nlm_do_unlock (argp=0xf31079c8, result=0xf3107a08, > rqstp=0xcb199800, rpcp=0x0) at > /zmirror/nfs/freebsd/base/stable/8/sys/nlm/nlm_prot_impl.c:2227 > #22 0xc071ac87 in nlm4_unlock_4_svc (argp=0xf31079c8, result=0xf3107a08, > rqstp=0xcb199800) at > /zmirror/nfs/freebsd/base/stable/8/sys/nlm/nlm_prot_server.c:540 > #23 0xc071bce3 in nlm_prog_4 (rqstp=0xcb199800, transp=0xc652de00) at > /zmirror/nfs/freebsd/base/stable/8/sys/nlm/nlm_prot_svc.c:512 > #24 0xc07284bf in svc_run_internal (pool=0xc61e4c80, ismaster=1) at > /zmirror/nfs/freebsd/base/stable/8/sys/rpc/svc.c:893 > #25 0xc072943d in svc_run (pool=0xc61e4c80) at > /zmirror/nfs/freebsd/base/stable/8/sys/rpc/svc.c:1233 > #26 0xc071a348 in nlm_syscall (td=0xc6551000, uap=0xf3107cf8) at > /zmirror/nfs/freebsd/base/stable/8/sys/nlm/nlm_prot_impl.c:1593 > #27 0xc07c5977 in syscall (frame=0xf3107d38) at > /zmirror/nfs/freebsd/base/stable/8/sys/i386/i386/trap.c:1073 > #28 0xc07ac570 in Xint0x80_syscall () at > /zmirror/nfs/freebsd/base/stable/8/sys/i386/i386/exception.s:261 > #29 0x00000033 in ?? () > > (kgdb) frame 12 > #12 0xc05ba39d in prison_priv_check (cred=0xc61e4880, priv=334) at > /zmirror/nfs/freebsd/base/stable/8/sys/kern/kern_jail.c:3568 > 3568 switch (priv) { > (kgdb) l 3567 > 3562 */ > 3563 if (cred->cr_prison->pr_flags & PR_VNET) > 3564 return (0); > 3565 } > 3566 #endif /* VIMAGE */ > 3567 > 3568 switch (priv) { > 3569 > 3570 /* > 3571 * Allow ktrace privileges for root in jail. > (kgdb) p cred->cr_prison > $4 = (struct prison *) 0x0It seems to be NFS related. I think the null pointer in question is from the export's anonymous credential. Try the patch below and see if it helps (which I guess means run it overnight and see if it crashes again). I've also patched a similar missing cred prison in GSS_SVC, since I'm not versed enough in NFS/RPC stuff to know if it might be the problem. - Jamie Index: kern/vfs_export.c ==================================================================--- kern/vfs_export.c (revision 197506) +++ kern/vfs_export.c (working copy) @@ -122,6 +122,8 @@ np->netc_anon->cr_uid = argp->ex_anon.cr_uid; crsetgroups(np->netc_anon, argp->ex_anon.cr_ngroups, argp->ex_anon.cr_groups); + np->netc_anon->cr_prison = &prison0; + prison_hold(np->netc_anon->cr_prison); np->netc_numsecflavors = argp->ex_numsecflavors; bcopy(argp->ex_secflavors, np->netc_secflavors, sizeof(np->netc_secflavors)); @@ -206,6 +208,8 @@ np->netc_anon->cr_uid = argp->ex_anon.cr_uid; crsetgroups(np->netc_anon, argp->ex_anon.cr_ngroups, np->netc_anon->cr_groups); + np->netc_anon->cr_prison = &prison0; + prison_hold(np->netc_anon->cr_prison); np->netc_numsecflavors = argp->ex_numsecflavors; bcopy(argp->ex_secflavors, np->netc_secflavors, sizeof(np->netc_secflavors)); Index: rpc/rpcsec_gss/svc_rpcsec_gss.c ==================================================================--- rpc/rpcsec_gss/svc_rpcsec_gss.c (revision 197506) +++ rpc/rpcsec_gss/svc_rpcsec_gss.c (working copy) @@ -449,6 +449,8 @@ cr->cr_uid = cr->cr_ruid = cr->cr_svuid = uc->uid; cr->cr_rgid = cr->cr_svgid = uc->gid; crsetgroups(cr, uc->gidlen, uc->gidlist); + cr->cr_prison = &prison0; + prison_hold(cr->cr_prison); *crp = crhold(cr); return (TRUE);