Hi List, Anybody know how to debug this crash. It happen every time I try to configure several tunnels with gre and quagga/zebra running. If I wait til the tunnel are built before starting zebra - it does not panic. The panic is always at the same place. line 186 in if_ether.c Also what would cause a page fault in the kernel - what exactly does this mean. Thanks, Steve PS: This freebsd 4.9 - which is old - but we have 1000's of these deployed and other than this issue they are rock solid. IdlePTD at phsyical address 0x00398000 initial pcb at physical address 0x002fbf20 panicstr: page fault panic messages: --- Fatal trap 12: page fault while in kernel mode fault virtual address = 0x4 fault code = supervisor read, page not present instruction pointer = 0x8:0xc01bc035 stack pointer = 0x10:0xc98a0d1c frame pointer = 0x10:0xc98a0d28 code segment = base 0x0, limit 0xfffff, type 0x1b = DPL 0, pres 1, def32 1, gran 1 processor eflags = interrupt enabled, resume, IOPL = 0 current process = 1249 (zebra) interrupt mask trap number = 12 panic: page fault syncing disks... 47 2 done Uptime: 14m30s dumping to dev #ad/1, offset 1851392 dump ata0: resetting devices .. done 120 119 118 117 116 115 114 113 112 111 110 109 108 107 106 105 104 103 102 101 100 99 98 97 96 95 94 93 92 91 90 89 88 87 86 85 84 83 82 81 80 79 78 77 76 75 74 73 72 71 70 69 68 67 66 65 64 63 62 61 60 59 58 57 56 55 54 53 52 51 50 49 48 47 46 45 44 43 42 41 40 39 38 37 36 35 34 33 32 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 --- #0 dumpsys () at ../../kern/kern_shutdown.c:487 487 if (dumping++) { (kgdb) bt #0 dumpsys () at ../../kern/kern_shutdown.c:487 #1 0xc0165eff in boot (howto=256) at ../../kern/kern_shutdown.c:316 #2 0xc0166324 in poweroff_wait (junk=0xc02c96ac, howto=-1070820945) at ../../kern/kern_shutdown.c:595 #3 0xc0280a4e in trap_fatal (frame=0xc98a0cdc, eva=4) at ../../i386/i386/trap.c:974 #4 0xc0280721 in trap_pfault (frame=0xc98a0cdc, usermode=0, eva=4) at ../../i386/i386/trap.c:867 #5 0xc02802df in trap (frame={tf_fs = 16, tf_es = 16, tf_ds = 16, tf_edi = -1055827712, tf_esi = -1056169104, tf_ebp = -913699544, tf_isp = -913699576, tf_ebx = 0, tf_edx = -1055827712, tf_ecx = 1, tf_eax = 0, tf_trapno = 12, tf_err = 0, tf_eip = -1071923147, tf_cs = 8, tf_eflags = 66118, tf_esp = -913699432, tf_ss = -1057684480}) at ../../i386/i386/trap.c:466 #6 0xc01bc035 in arp_rtrequest (req=1, rt=0xc1115900, info=0xc98a0d98) at ../../netinet/if_ether.c:186 #7 0xc01acfe2 in rtrequest1 (req=1, info=0xc98a0d98, ret_nrt=0xc98a0d94) at ../../net/route.c:750 #8 0xc01ada25 in route_output (m=0xc0705800, so=0xc8b90140) at ../../net/rtsock.c:341 #9 0xc01ac466 in raw_usend (so=0xc8b90140, flags=0, m=0xc0705800, nam=0x0, control=0x0, p=0xc979e5a0) at ../../net/raw_usrreq.c:258 #10 0xc01ad7bc in rts_send (so=0xc8b90140, flags=0, m=0xc0705800, nam=0x0, control=0x0, p=0xc979e5a0) at ../../net/rtsock.c:236 #11 0xc01852e7 in sosend (so=0xc8b90140, addr=0x0, uio=0xc98a0ed4, top=0xc0705800, control=0x0, flags=0, p=0xc979e5a0) at ../../kern/uipc_socket.c:609 #12 0xc0178780 in soo_write (fp=0xc0fe83c0, uio=0xc98a0ed4, cred=0xc1088280, flags=0, p=0xc979e5a0) at ../../kern/sys_socket.c:81 #13 0xc01752cd in dofilewrite (p=0xc979e5a0, fp=0xc0fe83c0, fd=5, buf=0xbfbfe714, nbyte=128, offset=-1, flags=0) at ../../sys/file.h:163 #14 0xc0175186 in write (p=0xc979e5a0, uap=0xc98a0f80) at ../../kern/sys_generic.c:329 #15 0xc0280cfd in syscall2 (frame={tf_fs = 47, tf_es = 47, tf_ds = 47, tf_edi = 128, tf_esi = 134882612, tf_ebp = -1077941904, tf_isp = -913698860, ---Type <return> to continue, or q <return> to quit--- tf_ebx = 16, tf_edx = 128, tf_ecx = 0, tf_eax = 4, tf_trapno = 7, tf_err = 2, tf_eip = 672644132, tf_cs = 31, tf_eflags = 659, tf_esp = -1077942556, tf_ss = 47}) at ../../i386/i386/trap.c:1175 #16 0xc0274ad5 in Xint0x80_syscall () #17 0x8059231 in ?? () #18 0x80592a5 in ?? () #19 0x80508c2 in ?? () #20 0x8050b5b in ?? () #21 0x280b2b8b in ?? () #22 0x2809cc8e in ?? () #23 0x804cffd in ?? () #24 0x804ac12 in ?? () (kgdb) frame 6 #6 0xc01bc035 in arp_rtrequest (req=1, rt=0xc1115900, info=0xc98a0d98) at ../../netinet/if_ether.c:186 186 if ((rt->rt_flags & RTF_HOST) == 0 && (kgdb) list 180 175 } 176 if (rt->rt_flags & RTF_GATEWAY) 177 return; 178 switch (req) { 179 180 case RTM_ADD: 181 /* 182 * XXX: If this is a manually added route to interface 183 * such as older version of routed or gated might provide, 184 * restore cloning bit. (kgdb) print *rt $1 = {rt_nodes = {{rn_mklist = 0x0, rn_parent = 0xc1115918, rn_bit = -1, rn_bmask = 0 '\000', rn_flags = 4 '\004', rn_u = {rn_leaf = { rn_Key = 0xc10c2360 "\020\002", rn_Mask = 0x0, rn_Dupedkey = 0x0}, rn_node = {rn_Off = -1056169120, rn_L = 0x0, rn_R = 0x0}}}, { rn_mklist = 0x0, rn_parent = 0xc0f3d718, rn_bit = 60, rn_bmask = 8 '\b', rn_flags = 4 '\004', rn_u = {rn_leaf = { rn_Key = 0x7 <Address 0x7 out of bounds>, rn_Mask = 0xc1115900 "", rn_Dupedkey = 0xc0f3d700}, rn_node = {rn_Off = 7, rn_L = 0xc1115900, rn_R = 0xc0f3d700}}}}, rt_gateway = 0xc10c2370, rt_refcnt = 0, rt_flags = 98305, rt_ifp = 0xc0703400, rt_ifa = 0xc0f50400, rt_genmask = 0x0, rt_llinfo = 0x0, rt_rmx = {rmx_locks = 0, rmx_mtu = 1500, rmx_hopcount = 0, rmx_expire = 0, rmx_recvpipe = 0, rmx_sendpipe = 0, rmx_ssthresh = 0, rmx_rtt = 0, rmx_rttvar = 0, rmx_pksent = 0, rmx_filler = {0, 0, 0, 0}}, rt_gwroute = 0x0, rt_output = 0, rt_parent = 0x0, rt_filler2 = 0x0} (kgdb) print rt->rt_flags $2 = 98305 (kgdb) print *info $3 = {rti_addrs = 7, rti_info = {0xc1040fdc, 0xc02e39ac, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, rti_flags = 32769, rti_ifa = 0xc0f50400, rti_ifp = 0xc0703400} (kgdb) quit