Florian Smeets
2010-Jan-21 10:23 UTC
7.2-STABLE page fault with kernel from 12.01.2010 / crashinfo available
Hi,
this firewall has been running happily with a kernel from August 8th
2009 and has only been rebooted to upgrade world/kernel on Jan 12th
2010, after only 7 days of uptime i got the page fault further down.
There are quite a few services on this firewall it has 4 physical
(sis(4)) interfaces of which all are used, 3 for Ethernet and one for
PPPoE via mpd5, it uses a gif interface for IPv6 tunneling via IPv4, and
a few IPsec tunnels are also configured and heavily used. arpwatch is
beeing run on 2 of the ethernet interfaces so they are always in
promiscuous mode, as a packet filter pf with altq is used.
The crashinfo file can be found here
http://webmail.smeets.im/~flo/crashinfo.txt
Fatal trap 12: page fault while in kernel mode
fault virtual address = 0xc
fault code = supervisor read, page not present
instruction pointer = 0x20:0xc0572e48
stack pointer = 0x28:0xc1f15b24
frame pointer = 0x28:0xc1f15b40
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 21 (irq5: sis0)
trap number = 12
panic: page fault
Uptime: 7d21h44m23s
Physical memory: 245 MB
Dumping 65 MB: 50 34 18 2
#0 doadump () at pcpu.h:196
196 pcpu.h: No such file or directory.
in pcpu.h
(kgdb) where
#0 doadump () at pcpu.h:196
#1 0xc0525703 in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:418
#2 0xc052590e in panic (fmt=Variable "fmt" is not available.
) at /usr/src/sys/kern/kern_shutdown.c:574
#3 0xc06f110c in trap_fatal (frame=0xc1f15ae4, eva=12) at
/usr/src/sys/i386/i386/trap.c:950
#4 0xc06f1390 in trap_pfault (frame=0xc1f15ae4, usermode=0, eva=12) at
/usr/src/sys/i386/i386/trap.c:863
#5 0xc06f1d65 in trap (frame=0xc1f15ae4) at
/usr/src/sys/i386/i386/trap.c:541
#6 0xc06d910b in calltrap () at /usr/src/sys/i386/i386/exception.s:166
#7 0xc0572e48 in m_copydata (m=0x0, off=0, len=40, cp=0xc23cced8
"\203??b??\237\f)h?M\220\224?\023?\205K(e??s?\"???k?oQ?~\223\020g\030")
at /usr/src/sys/kern/uipc_mbuf.c:815
#8 0xc05f8b28 in ip_forward (m=0xc23dc900, srcrt=0) at
/usr/src/sys/netinet/ip_input.c:1307
#9 0xc05fa30c in ip_input (m=0xc23dc900) at
/usr/src/sys/netinet/ip_input.c:609
#10 0xc05c83d5 in netisr_dispatch (num=2, m=0xc23dc900) at
/usr/src/sys/net/netisr.c:185
#11 0xc05bf581 in ether_demux (ifp=0xc20a4800, m=0xc23dc900) at
/usr/src/sys/net/if_ethersubr.c:834
#12 0xc05bf973 in ether_input (ifp=0xc20a4800, m=0xc23dc900) at
/usr/src/sys/net/if_ethersubr.c:692
#13 0xc04b8749 in sis_rxeof (sc=0xc2093800) at
/usr/src/sys/dev/sis/if_sis.c:1476
#14 0xc04b8973 in sis_intr (arg=0xc2093800) at
/usr/src/sys/dev/sis/if_sis.c:1667
#15 0xc050344b in ithread_loop (arg=0xc20ab410) at
/usr/src/sys/kern/kern_intr.c:1126
#16 0xc04ffe36 in fork_exit (callout=0xc05032a0 <ithread_loop>,
arg=0xc20ab410, frame=0xc1f15d38) at /usr/src/sys/kern/kern_fork.c:811
#17 0xc06d9180 in fork_trampoline () at
/usr/src/sys/i386/i386/exception.s:271
(kgdb) list *0xc0572e48
0xc0572e48 is in m_copydata (libkern.h:61).
56 static __inline int imax(int a, int b) { return (a > b ? a : b); }
57 static __inline int imin(int a, int b) { return (a < b ? a : b); }
58 static __inline long lmax(long a, long b) { return (a > b ? a : b); }
59 static __inline long lmin(long a, long b) { return (a < b ? a : b); }
60 static __inline u_int max(u_int a, u_int b) { return (a > b ? a : b); }
61 static __inline u_int min(u_int a, u_int b) { return (a < b ? a : b); }
62 static __inline quad_t qmax(quad_t a, quad_t b) { return (a > b ? a :
b); }
63 static __inline quad_t qmin(quad_t a, quad_t b) { return (a < b ? a :
b); }
64 static __inline u_long ulmax(u_long a, u_long b) { return (a > b ? a
: b); }
65 static __inline u_long ulmin(u_long a, u_long b) { return (a < b ? a
: b); }
(kgdb) frame 7
#7 0xc0572e48 in m_copydata (m=0x0, off=0, len=40, cp=0xc23cced8
"\203??b??\237\f)h?M\220\224?\023?\205K(e??s?\"???k?oQ?~\223\020g\030")
at /usr/src/sys/kern/uipc_mbuf.c:815
815 count = min(m->m_len - off, len);
(kgdb) l
810 off -= m->m_len;
811 m = m->m_next;
812 }
813 while (len > 0) {
814 KASSERT(m != NULL, ("m_copydata, length > size of mbuf
chain"));
815 count = min(m->m_len - off, len);
816 bcopy(mtod(m, caddr_t) + off, cp, count);
817 len -= count;
818 cp += count;
819 off = 0;
Looking forward to suggestions what may be causing this.
Thanks,
Florian
John Baldwin
2010-Jan-21 13:13 UTC
7.2-STABLE page fault with kernel from 12.01.2010 / crashinfo available
On Thursday 21 January 2010 5:10:20 am Florian Smeets wrote:> Hi, > > this firewall has been running happily with a kernel from August 8th > 2009 and has only been rebooted to upgrade world/kernel on Jan 12th > 2010, after only 7 days of uptime i got the page fault further down. > > There are quite a few services on this firewall it has 4 physical > (sis(4)) interfaces of which all are used, 3 for Ethernet and one for > PPPoE via mpd5, it uses a gif interface for IPv6 tunneling via IPv4, and > a few IPsec tunnels are also configured and heavily used. arpwatch is > beeing run on 2 of the ethernet interfaces so they are always in > promiscuous mode, as a packet filter pf with altq is used. > > The crashinfo file can be found here > http://webmail.smeets.im/~flo/crashinfo.txt > > Fatal trap 12: page fault while in kernel mode > fault virtual address = 0xc > fault code = supervisor read, page not present > instruction pointer = 0x20:0xc0572e48 > stack pointer = 0x28:0xc1f15b24 > frame pointer = 0x28:0xc1f15b40 > code segment = base 0x0, limit 0xfffff, type 0x1b > = DPL 0, pres 1, def32 1, gran 1 > processor eflags = interrupt enabled, resume, IOPL = 0 > current process = 21 (irq5: sis0) > trap number = 12 > panic: page fault > Uptime: 7d21h44m23s > Physical memory: 245 MB > Dumping 65 MB: 50 34 18 2 > > #0 doadump () at pcpu.h:196 > 196 pcpu.h: No such file or directory. > in pcpu.h > (kgdb) where > #0 doadump () at pcpu.h:196 > #1 0xc0525703 in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:418 > #2 0xc052590e in panic (fmt=Variable "fmt" is not available. > ) at /usr/src/sys/kern/kern_shutdown.c:574 > #3 0xc06f110c in trap_fatal (frame=0xc1f15ae4, eva=12) at > /usr/src/sys/i386/i386/trap.c:950 > #4 0xc06f1390 in trap_pfault (frame=0xc1f15ae4, usermode=0, eva=12) at > /usr/src/sys/i386/i386/trap.c:863 > #5 0xc06f1d65 in trap (frame=0xc1f15ae4) at > /usr/src/sys/i386/i386/trap.c:541 > #6 0xc06d910b in calltrap () at /usr/src/sys/i386/i386/exception.s:166 > #7 0xc0572e48 in m_copydata (m=0x0, off=0, len=40, cp=0xc23cced8 > "\203??b??\237\f)h?M\220\224?\023?\205K(e??s?\"???k?oQ?~\223\020g\030") > at /usr/src/sys/kern/uipc_mbuf.c:815 > #8 0xc05f8b28 in ip_forward (m=0xc23dc900, srcrt=0) at > /usr/src/sys/netinet/ip_input.c:1307 > #9 0xc05fa30c in ip_input (m=0xc23dc900) at > /usr/src/sys/netinet/ip_input.c:609 > #10 0xc05c83d5 in netisr_dispatch (num=2, m=0xc23dc900) at > /usr/src/sys/net/netisr.c:185 > #11 0xc05bf581 in ether_demux (ifp=0xc20a4800, m=0xc23dc900) at > /usr/src/sys/net/if_ethersubr.c:834 > #12 0xc05bf973 in ether_input (ifp=0xc20a4800, m=0xc23dc900) at > /usr/src/sys/net/if_ethersubr.c:692 > #13 0xc04b8749 in sis_rxeof (sc=0xc2093800) at > /usr/src/sys/dev/sis/if_sis.c:1476 > #14 0xc04b8973 in sis_intr (arg=0xc2093800) at > /usr/src/sys/dev/sis/if_sis.c:1667 > #15 0xc050344b in ithread_loop (arg=0xc20ab410) at > /usr/src/sys/kern/kern_intr.c:1126 > #16 0xc04ffe36 in fork_exit (callout=0xc05032a0 <ithread_loop>, > arg=0xc20ab410, frame=0xc1f15d38) at /usr/src/sys/kern/kern_fork.c:811 > #17 0xc06d9180 in fork_trampoline () at > /usr/src/sys/i386/i386/exception.s:271 > (kgdb) list *0xc0572e48 > 0xc0572e48 is in m_copydata (libkern.h:61). > 56 static __inline int imax(int a, int b) { return (a > b ? a : b); } > 57 static __inline int imin(int a, int b) { return (a < b ? a : b); } > 58 static __inline long lmax(long a, long b) { return (a > b ? a : b); } > 59 static __inline long lmin(long a, long b) { return (a < b ? a : b); } > 60 static __inline u_int max(u_int a, u_int b) { return (a > b ? a : b); } > 61 static __inline u_int min(u_int a, u_int b) { return (a < b ? a : b); } > 62 static __inline quad_t qmax(quad_t a, quad_t b) { return (a > b ? a : > b); } > 63 static __inline quad_t qmin(quad_t a, quad_t b) { return (a < b ? a : > b); } > 64 static __inline u_long ulmax(u_long a, u_long b) { return (a > b ? a > : b); } > 65 static __inline u_long ulmin(u_long a, u_long b) { return (a < b ? a > : b); } > (kgdb) frame 7 > #7 0xc0572e48 in m_copydata (m=0x0, off=0, len=40, cp=0xc23cced8 > "\203??b??\237\f)h?M\220\224?\023?\205K(e??s?\"???k?oQ?~\223\020g\030") > at /usr/src/sys/kern/uipc_mbuf.c:815 > 815 count = min(m->m_len - off, len); > (kgdb) l > 810 off -= m->m_len; > 811 m = m->m_next; > 812 } > 813 while (len > 0) { > 814 KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain"));I think you would have hit this assertion if INVARIANTS were enabled. Can you go up to frame 8 and do an 'l'? Maybe 'p *m' as well? -- John Baldwin