Hi list,
I caught a panic this night on my RELENG_5. The kernel was compiled on
2005/05/21. Please, feel free to ask for further informations (and
include me explicitely in the recipients list since I'm not subscribed
to this list).
kgdb stacktrace:
%%%
#22 0xc0566d1d in panic (
fmt=0xc0728d5d "Duplicate free of item %p from zone %p(%s)\n")
at ../../../kern/kern_shutdown.c:550
td = (struct thread *) 0xc205ec00
bootopt = 256
newpanic = 1
ap = 0xd6d3f968 ""
buf = "Duplicate free of item 0xc1be8800 from zone
0xc1045ae0(Mbuf)\n",
'\0' <repeats 194 times>
#23 0xc069e280 in uma_dbg_free (zone=0xc1045ae0, slab=0xc1be8fa8,
item=0xc1be8800) at ../../../vm/uma_dbg.c:301
keg = 0xc101f3c0
slabref = 0x0
freei = 8
#24 0xc069cc39 in uma_zfree_arg (zone=0xc1045ae0, item=0xc1be8800, udata=0x0)
at ../../../vm/uma_core.c:2273
keg = 0xc101f3c0
cache = 0xc1045b18
bucket = 0xc1be2000
bflags = 0
cpu = 0
skip = SKIP_DTOR
#25 0xc05a0a0b in m_freem (mb=0x0) at uma.h:304
No locals.
#26 0xc05ee0d5 in arpresolve (ifp=0xc1a5b000, rt0=0xc1d44000, m=0xc1be7200,
dst=0xd6d3fa94, desten=0xd6d3fa2c "/?]???w?")
at ../../../netinet/if_ether.c:442
la = (struct llinfo_arp *) 0xc1a75a00
sdl = (struct sockaddr_dl *) 0xc2128910
error = -1038972656
rt = (struct rtentry *) 0xc1d44000
#27 0xc05dac65 in ether_output (ifp=0xc1a5b000, m=0xc1be7200, dst=0xd6d3fa94,
rt0=0x0) at ../../../net/if_ethersubr.c:165
type = -10541
error = 50
hdrcmplt = 0
esrc = "K\000\000\000\214z"
edst = "/?]???"
eh = (struct ether_header *) 0x32
loop_copy = 0
#28 0xc060150c in ip_output (m=0xc1be7200, opt=0xc1be7240, ro=0xd6d3fa90,
flags=0, imo=0x0, inp=0xc40f7a8c) at ../../../netinet/ip_output.c:770
ip = (struct ip *) 0xc1be7240
ifp = (struct ifnet *) 0xc1a5b000
m0 = (struct mbuf *) 0xc1be7240
hlen = 20
len = 1
error = 0
dst = (struct sockaddr_in *) 0xd6d3fa94
ia = (struct in_ifaddr *) 0xc1c2b300
isbroadcast = 0
sw_csum = 1
iproute = {ro_rt = 0xc1d44000, ro_dst = {sa_len = 16 '\020',
sa_family = 2 '\002',
sa_data = "\000\000??\001?\000\000\000\000\000\000\000"}}
odst = {s_addr = 1}
fwd_tag = (struct m_tag *) 0x0
__func__ = "ip_output"
#29 0xc060aba1 in tcp_output (tp=0xc1d75534)
at ../../../netinet/tcp_output.c:1119
so = (struct socket *) 0xc2afe000
len = 144
recwin = 66608
sendwin = -1044483500
flags = 24
error = -1044483500
m = (struct mbuf *) 0xc1be7200
ip = (struct ip *) 0xc1be7240
th = (struct tcphdr *) 0xc1be7254
opt =
"\001\001\b\n\002?>m\003?J?+\001\000\000????????\000???\204???\203
~Z?????"
ipoptlen = 0
optlen = 12
hdrlen = 52
idle = 1
sendalot = 0
i = 299
sack_rxmit = 0
sack_bytes_rxmt = 0
p = (struct sackhole *) 0x0
tao = {tao_cc = 767, tao_ccsent = 3228670914, tao_mssopt = 64356}
__func__ = "tcp_output"
#30 0xc061167c in tcp_usr_send (so=0xc2afe000, flags=0, m=0xc1be7600, nam=0x0,
control=0x0, td=0xc205ec00) at ../../../netinet/tcp_usrreq.c:699
error = 0
inp = (struct inpcb *) 0xc40f7a8c
tp = (struct tcpcb *) 0xc1d75534
#31 0xc05a41e8 in sosend (so=0xc2afe000, addr=0x0, uio=0xd6d3fc70,
top=0xc1be7600, control=0x0, flags=0, td=0xc205ec00)
at ../../../kern/uipc_socket.c:835
mp = (struct mbuf **) 0xc1be7600
m = (struct mbuf *) 0xc1be7600
space = 33160
len = 144
resid = 0
clen = -1044482560
error = 0
dontroute = 0
atomic = 0
#32 0xc05928bf in soo_write (fp=0x0, uio=0xd6d3fc70, active_cred=0xc4211e80,
flags=0, td=0xc205ec00) at ../../../kern/sys_socket.c:118
so = (struct socket *) 0xc2afe000
error = 144
#33 0xc058bc0b in dofilewrite (td=0xc205ec00, fp=0xc2aff83c, fd=0, buf=0x0,
nbyte=3228877920, offset=Unhandled dwarf expression opcode 0x93
) at file.h:245
auio = {uio_iov = 0xd6d3fc68, uio_iovcnt = 1, uio_offset = 143,
uio_resid = 0, uio_segflg = UIO_USERSPACE, uio_rw = UIO_WRITE,
uio_td = 0xc205ec00}
aiov = {iov_base = 0x807d090, iov_len = 0}
cnt = 144
error = -1066089376
ktruio = (struct uio *) 0x0
#34 0xc058ba74 in write (td=0xc205ec00, uap=0xd6d3fd04)
at ../../../kern/sys_generic.c:300
fp = (struct file *) 0xc2aff83c
error = 0
#35 0xc06d2a12 in syscall (frame {tf_fs = -1078001617, tf_es = 47, tf_ds =
-1078001617, tf_edi = 134671528,
tf_esi = 144, tf_ebp = -1077943016, tf_isp = -690750108, tf_ebx = 671922152, tf
_edx = 134671528, tf_ecx = 4, tf_eax = 4, tf_trapno = 12, tf_err = 2, tf_eip = 6
73631499, tf_cs = 31, tf_eflags = 518, tf_esp = -1077943044, tf_ss = 47})
at ../../../i386/i386/trap.c:1009
params = 0xbfbfe500 <Address 0xbfbfe500 out of bounds>
callp = (struct sysent *) 0xc0744960
td = (struct thread *) 0xc205ec00
p = (struct proc *) 0xc2b04a98
orig_tf_eflags = 518
sticks = 113
error = 0
narg = 3
args = {4, 134729728, 144, 0, -1077942928, 0, 113, -1028633960}
code = 4
#36 0xc06c46bf in Xint0x80_syscall () at ../../../i386/i386/exception.s:201
%%%
Regards,
--
Jeremie Le Hen
< jeremie at le-hen dot org >< ttz at chchile dot org >
Hi,> I caught a panic this night on my RELENG_5. The kernel was compiled on > 2005/05/21. Please, feel free to ask for further informations (and > include me explicitely in the recipients list since I'm not subscribed > to this list). > > kgdb stacktrace: > %%% > [snip] > %%%I was a little bit sleepy earlier this morning. I forgot to tell that my kernel is compiled with INVARIANTS and PREEMPTION. %%% (kgdb) up 26 #26 0xc05ee0d5 in arpresolve (ifp=0xc1a5b000, rt0=0xc1d44000, m=0xc1be7200, dst=0xd6d3fa94, desten=0xd6d3fa2c "/?]???w?") at ../../../netinet/if_ether.c:442 442 m_freem(la->la_hold); (kgdb) l 437 * There is an arptab entry, but no ethernet address 438 * response yet. Replace the held mbuf with this 439 * latest one. 440 */ 441 if (la->la_hold) 442 m_freem(la->la_hold); 443 la->la_hold = m; 444 if (rt->rt_expire) { 445 RT_LOCK(rt); 446 rt->rt_flags &= ~RTF_REJECT; (kgdb) print *la $1 = {la_le = {le_next = 0xc1e74400, le_prev = 0xc077aa68}, la_rt = 0xc1d44000, la_hold = 0x0, la_preempt = 5, la_asked = 0} %%% -- Jeremie Le Hen < jeremie at le-hen dot org >< ttz at chchile dot org >
On Tue, Jun 21, 2005 at 09:04:27AM +0200, Jeremie Le Hen wrote:
J> #25 0xc05a0a0b in m_freem (mb=0x0) at uma.h:304
J> No locals.
J> #26 0xc05ee0d5 in arpresolve (ifp=0xc1a5b000, rt0=0xc1d44000,
m=0xc1be7200,
J> dst=0xd6d3fa94, desten=0xd6d3fa2c "/??]??????w??")
J> at ../../../netinet/if_ether.c:442
J> la = (struct llinfo_arp *) 0xc1a75a00
J> sdl = (struct sockaddr_dl *) 0xc2128910
J> error = -1038972656
J> rt = (struct rtentry *) 0xc1d44000
IMHO, this looks like a race. The route is not locked, when
its llinfo is edited.
Probably the mbuf was freed when arp reply arrived and la_hold was send.
Look into in_arpinput() near 736:
(*ifp->if_output)(ifp, la->la_hold, rt_key(rt),
rt);
la->la_hold = 0;
Yeah, I have just triggered another panic running 15 instances of this script on
SMP box:
(
while (true); do
arp -d 81.19.64.111 >/dev/null 2>&1;
ping -c 1 -t 1 81.19.64.111 >/dev/null 2>&1;
done
) &
But my duplicate free is in fxp_txeof(). This means that output thread has
won the race.
--
Totus tuus, Glebius.
GLEBIUS-RIPN GLEB-RIPE
Hi Gleb,> IMHO, this looks like a race. The route is not locked, when > its llinfo is edited. > > Probably the mbuf was freed when arp reply arrived and la_hold was send. > Look into in_arpinput() near 736: > > (*ifp->if_output)(ifp, la->la_hold, rt_key(rt), rt); > la->la_hold = 0; > > Yeah, I have just triggered another panic running 15 instances of this > script on SMP box: > > ( > while (true); do > arp -d 81.19.64.111 >/dev/null 2>&1; > ping -c 1 -t 1 81.19.64.111 >/dev/null 2>&1; > done > ) & > > But my duplicate free is in fxp_txeof(). This means that output thread has > won the race.This explanation sounds good but my box is an UP with PREEMPTION. Is is supposed to be also possible in this case ? Regards, -- Jeremie Le Hen < jeremie at le-hen dot org >< ttz at chchile dot org >
On Tue, Jun 21, 2005 at 11:28:36AM +0200, Jeremie Le Hen wrote: J> > IMHO, this looks like a race. The route is not locked, when J> > its llinfo is edited. J> > J> > Probably the mbuf was freed when arp reply arrived and la_hold was send. J> > Look into in_arpinput() near 736: J> > J> > (*ifp->if_output)(ifp, la->la_hold, rt_key(rt), rt); J> > la->la_hold = 0; J> > J> > Yeah, I have just triggered another panic running 15 instances of this J> > script on SMP box: J> > J> > ( J> > while (true); do J> > arp -d 81.19.64.111 >/dev/null 2>&1; J> > ping -c 1 -t 1 81.19.64.111 >/dev/null 2>&1; J> > done J> > ) & J> > J> > But my duplicate free is in fxp_txeof(). This means that output thread has J> > won the race. J> J> This explanation sounds good but my box is an UP with PREEMPTION. J> Is is supposed to be also possible in this case ? I guess yes, because of preemption. -- Totus tuus, Glebius. GLEBIUS-RIPN GLEB-RIPE