This was with a kernel from Aug 6th. (r253988). The box functions as an
LNS. Similarly loaded RELENG_8 boxes with ipv6 disabled are quite
stable. Not sure if thats where the issue is as this is RELENG9
---Mike
--
-------------------
Mike Tancsa, tel +1 519 651 3400
Sentex Communications, mike at sentex.net
Providing Internet services since 1994 www.sentex.net
Cambridge, Ontario Canada http://www.tancsa.com/
-------------- next part --------------
Fatal trap 12: page fault while in kernel mode
cpuid = 1; apic id = 02
fault virtual address = 0x34646dc7
fault code = supervisor read data, page not present
instruction pointer = 0x20:0xffffffff80a02d60
stack pointer = 0x28:0xffffff8000320710
frame pointer = 0x28:0xffffff8000320750
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 12 (irq265: igb0:que 1)
trap number = 12
panic: page fault
cpuid = 1
KDB: stack backtrace:
#0 0xffffffff8092d686 at kdb_backtrace+0x66
#1 0xffffffff808f639e at panic+0x1ce
#2 0xffffffff80ca1800 at trap_fatal+0x290
#3 0xffffffff80ca1b38 at trap_pfault+0x1e8
#4 0xffffffff80ca213e at trap+0x3be
#5 0xffffffff80c8b76f at calltrap+0x8
#6 0xffffffff80a08ce2 at ng_iface_send+0xc2
#7 0xffffffff80a091c6 at ng_iface_output+0x1c6
#8 0xffffffff80a2d044 at ip_fastforward+0x824
#9 0xffffffff809b3f0c at ether_demux+0x19c
#10 0xffffffff809b419e at ether_nh_input+0x1fe
#11 0xffffffff809bd2a8 at netisr_dispatch_src+0x218
#12 0xffffffff804d0460 at igb_rxeof+0x3f0
#13 0xffffffff804d08f1 at igb_msix_que+0xe1
#14 0xffffffff808c998d at intr_event_execute_handlers+0xfd
#15 0xffffffff808cb17d at ithread_loop+0x9d
#16 0xffffffff808c697f at fork_exit+0x11f
#17 0xffffffff80c8bc9e at fork_trampoline+0xe
Uptime: 75d19h28m41s
Dumping 1028 out of 7895 MB:..2%..11%..21%..32%..41%..52%..61%..71%..81%..91%
#0 doadump (textdump=<value optimized out>) at pcpu.h:234
234 pcpu.h: No such file or directory.
in pcpu.h
(kgdb) #0 doadump (textdump=<value optimized out>) at pcpu.h:234
#1 0xffffffff808f5e76 in kern_reboot (howto=260)
at /usr/src/sys/kern/kern_shutdown.c:449
#2 0xffffffff808f6377 in panic (fmt=0x1 <Address 0x1 out of bounds>)
at /usr/src/sys/kern/kern_shutdown.c:637
#3 0xffffffff80ca1800 in trap_fatal (frame=0xc, eva=<value optimized
out>)
at /usr/src/sys/amd64/amd64/trap.c:879
#4 0xffffffff80ca1b38 in trap_pfault (frame=0xffffff8000320660, usermode=0)
at /usr/src/sys/amd64/amd64/trap.c:795
#5 0xffffffff80ca213e in trap (frame=0xffffff8000320660)
at /usr/src/sys/amd64/amd64/trap.c:463
#6 0xffffffff80c8b76f in calltrap ()
at /usr/src/sys/amd64/amd64/exception.S:232
#7 0xffffffff80a02d60 in ng_address_hook (here=0x0, item=0xfffffe0085c57c80,
hook=0x34646d63, retaddr=0) at /usr/src/sys/netgraph/ng_base.c:3583
#8 0xffffffff80a08ce2 in ng_iface_send (ifp=0xfffffe00091b2800,
m=<value optimized out>, sa=<value optimized out>)
at /usr/src/sys/netgraph/ng_iface.c:473
#9 0xffffffff80a091c6 in ng_iface_output (ifp=0xfffffe00091b2800,
m=0xfffffe002638e400, dst=0xffffff800032082c, ro=<value optimized
out>)
at /usr/src/sys/netgraph/ng_iface.c:408
#10 0xffffffff80a2d044 in ip_fastforward (m=0xfffffe002638e400)
at /usr/src/sys/netinet/ip_fastfwd.c:541
#11 0xffffffff809b3f0c in ether_demux (ifp=0xfffffe000451d000,
m=0xfffffe002638e400) at /usr/src/sys/net/if_ethersubr.c:879
#12 0xffffffff809b419e in ether_nh_input (m=<value optimized out>)
at /usr/src/sys/net/if_ethersubr.c:762
#13 0xffffffff809bd2a8 in netisr_dispatch_src (proto=9,
source=<value optimized out>, m=<value optimized out>)
at /usr/src/sys/net/netisr.c:1013
#14 0xffffffff804d0460 in igb_rxeof (que=0xfffffe00045f8068, count=98,
done=0x0) at /usr/src/sys/dev/e1000/if_igb.c:4724
#15 0xffffffff804d08f1 in igb_msix_que (arg=<value optimized out>)
at /usr/src/sys/dev/e1000/if_igb.c:1591
#16 0xffffffff808c998d in intr_event_execute_handlers (
p=<value optimized out>, ie=0xfffffe00045fb500)
at /usr/src/sys/kern/kern_intr.c:1272
#17 0xffffffff808cb17d in ithread_loop (arg=0xfffffe0004629680)
at /usr/src/sys/kern/kern_intr.c:1285
#18 0xffffffff808c697f in fork_exit (
callout=0xffffffff808cb0e0 <ithread_loop>, arg=0xfffffe0004629680,
frame=0xffffff8000320b00) at /usr/src/sys/kern/kern_fork.c:988
#19 0xffffffff80c8bc9e in fork_trampoline ()
at /usr/src/sys/amd64/amd64/exception.S:606
#20 0x0000000000000000 in ?? ()
------------------------------------------------------------------------
I upgraded to r256844, and still getting crashes under load. (about 500
connections). RELENG_8 sans ipv6 is nice and stable under similar and
higher loads. I will try without inet6 for now
Unread portion of the kernel message buffer:
current process = 1158 (mpd5)
trap number = 12
panic: page fault
cpuid = 2
KDB: stack backtrace:
#0 0xffffffff8092e5a6 at kdb_backtrace+0x66
#1 0xffffffff808f6e9e at panic+0x1ce
#2 0xffffffff80ca2cc0 at trap_fatal+0x290
#3 0xffffffff80ca2ff8 at trap_pfault+0x1e8
#4 0xffffffff80ca35fe at trap+0x3be
#5 0xffffffff80c8cc0f at calltrap+0x8
#6 0xffffffff80a75470 at sctp_add_addr_to_vrf+0x60
#7 0xffffffff80a3db63 at sctp_addr_change+0x113
#8 0xffffffff809c51cd at rt_newaddrmsg_fib+0x4d
#9 0xffffffff809c3ce7 at rtinit+0x1b7
#10 0xffffffff80a222d8 at in_ifinit+0x3a8
#11 0xffffffff80a239c2 at in_control+0x1072
#12 0xffffffff809b1f83 at ifioctl+0x803
#13 0xffffffff809404a6 at kern_ioctl+0x106
#14 0xffffffff809406ed at sys_ioctl+0xfd
#15 0xffffffff80ca25a0 at amd64_syscall+0x540
#16 0xffffffff80c8cef7 at Xfast_syscall+0xf7
Uptime: 8d9h15m51s
Dumping 1022 out of 7895
MB:..2%..11%..21%..32%..41%..51%..62%..71%..82%..91%
#0 doadump (textdump=<value optimized out>) at pcpu.h:234
234 pcpu.h: No such file or directory.
in pcpu.h
(kgdb) #0 doadump (textdump=<value optimized out>) at pcpu.h:234
#1 0xffffffff808f6976 in kern_reboot (howto=260)
at /usr/src/sys/kern/kern_shutdown.c:449
#2 0xffffffff808f6e77 in panic (fmt=0x1 <Address 0x1 out of bounds>)
at /usr/src/sys/kern/kern_shutdown.c:637
#3 0xffffffff80ca2cc0 in trap_fatal (frame=0xc, eva=<value optimized
out>)
at /usr/src/sys/amd64/amd64/trap.c:879
#4 0xffffffff80ca2ff8 in trap_pfault (frame=0xffffff823bcaa320, usermode=0)
at /usr/src/sys/amd64/amd64/trap.c:795
#5 0xffffffff80ca35fe in trap (frame=0xffffff823bcaa320)
at /usr/src/sys/amd64/amd64/trap.c:463
#6 0xffffffff80c8cc0f in calltrap ()
at /usr/src/sys/amd64/amd64/exception.S:232
#7 0xffffffff80a6a300 in sctp_find_ifn (ifn=0xfffffe0009406800,
ifn_index=359) at /usr/src/sys/netinet/sctp_pcb.c:205
#8 0xffffffff80a75470 in sctp_add_addr_to_vrf (vrf_id=0,
ifn=0xfffffe0009406800, ifn_index=359, ifn_type=53,
if_name=0xfffffe0009406828 "ng347", ifa=0xfffffe0200cb1a00,
addr=0xfffffe0200cb1b38, ifa_flags=0, dynamic_add=1)
at /usr/src/sys/netinet/sctp_pcb.c:508
#9 0xffffffff80a3db63 in sctp_addr_change (ifa=0xfffffe0200cb1a00,
cmd=<value optimized out>) at /usr/src/sys/netinet/sctp_bsd_addr.c:346
#10 0xffffffff809c51cd in rt_newaddrmsg_fib (cmd=1, ifa=0xfffffe0200cb1a00,
error=0, rt=0xfffffe019c395000, fibnum=0)
at /usr/src/sys/net/rtsock.c:1299
#11 0xffffffff809c3ce7 in rtinit (ifa=0xfffffe0200cb1a00, cmd=1, flags=5)
at /usr/src/sys/net/route.c:1626
#12 0xffffffff80a222d8 in in_ifinit (ifp=0xfffffe0009406800,
ia=0xfffffe0200cb1a00, sin=<value optimized out>, scrub=0)
at /usr/src/sys/netinet/in.c:1068
#13 0xffffffff80a239c2 in in_control (so=<value optimized out>, cmd=0,
data=0xfffffe01724ff340 "ng347", ifp=0xfffffe0009406800,
td=<value optimized out>) at /usr/src/sys/netinet/in.c:550
#14 0xffffffff809b1f83 in ifioctl (so=0xfffffe01091c77f8, cmd=2151704858,
data=0xfffffe01724ff340 "ng347", td=0xfffffe00096b3490)
at /usr/src/sys/net/if.c:2569
#15 0xffffffff809404a6 in kern_ioctl (td=0xfffffe00096b3490, fd=149,
com=2151704858, data=0xfffffe01724ff340 "ng347") at file.h:311
#16 0xffffffff809406ed in sys_ioctl (td=0xfffffe00096b3490,
uap=0xffffff823bcaaa70) at /usr/src/sys/kern/sys_generic.c:692
#17 0xffffffff80ca25a0 in amd64_syscall (td=0xfffffe00096b3490, traced=0)
at subr_syscall.c:135
#18 0xffffffff80c8cef7 in Xfast_syscall ()
at /usr/src/sys/amd64/amd64/exception.S:391
#19 0x000000080227bbac in ?? ()
Previous frame inner to this frame (corrupt stack?)
(kgdb)
On 10/21/2013 9:38 AM, Mike Tancsa wrote:> This was with a kernel from Aug 6th. (r253988). The box functions as an
> LNS. Similarly loaded RELENG_8 boxes with ipv6 disabled are quite
> stable. Not sure if thats where the issue is as this is RELENG9
>
> ---Mike
>
>
>
>
>
> _______________________________________________
> freebsd-stable at freebsd.org mailing list
> http://lists.freebsd.org/mailman/listinfo/freebsd-stable
> To unsubscribe, send any mail to "freebsd-stable-unsubscribe at
freebsd.org"
--
-------------------
Mike Tancsa, tel +1 519 651 3400
Sentex Communications, mike at sentex.net
Providing Internet services since 1994 www.sentex.net
Cambridge, Ontario Canada http://www.tancsa.com/