thr3ads.net - freebsd stable - got a panic on 5.4-STABLE [Aug 2005]

If this information is useful, please help other people find it:
Share via:

Rong-En Fan

2005-Aug-28 13:53 UTC

got a panic on 5.4-STABLE

Hi,

I got a panic on an i386 5.4-STABLE around Aug 28 with SMP enabled.
It has 2 physical CPU with HTT enabled (so, total 4 cpus). 
This is a NFS server only with external scsi raid attached.

The console log, kgdb output and sysctl.conf are as below.
I'll keep this core and if someone is interested, I can send any other
information requested.

Regards,
Rong-En Fan

[sysctl.conf]
net.link.ether.inet.log_arp_wrong_iface=0
net.inet.tcp.sendspace=65536
net.inet.tcp.recvspace=65536
net.inet.udp.recvspace=65536
kern.ipc.somaxconn=4096
kern.maxfiles=65535
kern.ipc.shmmax=104857600
kern.ipc.shmall=25600
net.inet.ip.random_id=1
kern.cam.da.retry_count=20
kern.cam.da.default_timeout=300
kern.maxvnodes=100000
vfs.read_max=16

[console log]
Fatal trap 12: page fault while in kernel mode
cpuid = 0; apic id = 00
fault virtual address   = 0xc
fault code              = supervisor read, page not present
instruction pointer     = 0x8:0xc051d62f
stack pointer           = 0x10:0xe7088a64
frame pointer           = 0x10:0xe7088a98
code segment            = base 0x0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, def32 1, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 523 (nfsd)
trap number             = 12
panic: page fault
cpuid = 0
boot() called on cpu#0
Uptime: 2h59m59s
Dumping 1023 MB
 16 32 48 64 80 96 112 128 144 160 176 192 208 224 240 256 272 288 304
320 336 352 368 384 400 416 432 448 464 480 496 512 528 544 560 576
592 608 624 640 656 672 688 704 720 736 752 768 784 800 816 832 848
864 880 896 912 928 944 960 976 992 1008
Dump complete
Automatic reboot in 1 seconds - press a key on the console to abort
Rebooting...
cpu_reset called on cpu#0
cpu_reset: Stopping other CPUs

[kgdb output]
Unread portion of the kernel message buffer:
@???????? ??
????D0???"?p'?@???0?????0??p'

#0  doadump () at pcpu.h:160
160             __asm __volatile("movl %%fs:0,%0" : "=r"
(td));
(kgdb) up
#1  0xc04e0385 in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:410
410                     doadump();
(kgdb)
#2  0xc04e0733 in panic (fmt=0xc065d085 "%s") at
/usr/src/sys/kern/kern_shutdown.c:566
566             boot(bootopt);
(kgdb)
#3  0xc064503d in trap_fatal (frame=0xe7088a24, eva=0) at
/usr/src/sys/i386/i386/trap.c:817
817                     panic("%s", trap_msg[type]);
(kgdb)
#4  0xc0644d34 in trap_pfault (frame=0xe7088a24, usermode=0, eva=12)
    at /usr/src/sys/i386/i386/trap.c:735
735                     trap_fatal(frame, eva);
(kgdb)
#5  0xc0644919 in trap (frame      {tf_fs = -1038614504, tf_es = -418906096,
tf_ds = 16, tf_edi 1480, tf_esi = 0, tf_ebp = -418870632, tf_isp = -418870704,
tf_ebx -1039299264, tf_edx = 20, tf_ecx = 1500, tf_eax = -1039299328,
tf_trapno = 12, tf_err = 0, tf_eip = -1068378577, tf_cs = 8, tf_eflags
= 66050, tf_esp = -1039299328, tf_ss = 1480}) at
/usr/src/sys/i386/i386/trap.c:425
425                             (void) trap_pfault(&frame, FALSE, eva);
(kgdb)
#6  0xc063122a in calltrap () at /usr/src/sys/i386/i386/exception.s:140
140             call    trap
Current language:  auto; currently asm
(kgdb)
#7  0xc2180018 in ?? ()
(kgdb)
#8  0xe7080010 in ?? ()
(kgdb)
#9  0x00000010 in ?? ()
(kgdb)
#10 0x000005c8 in ?? ()
(kgdb)
#11 0x00000000 in ?? ()
(kgdb)
#12 0xe7088a98 in ?? ()
(kgdb)
#13 0xe7088a50 in ?? ()
(kgdb)
#14 0xc20d8d40 in ?? ()
(kgdb)
#15 0x00000014 in ?? ()
(kgdb)
#16 0x000005dc in ?? ()
(kgdb)
#17 0xc20d8d00 in ?? ()
(kgdb)
#18 0x0000000c in ?? ()
(kgdb)
#19 0x00000000 in ?? ()
(kgdb)
#20 0xc051d62f in m_copym (m=0x0, off0=1500, len=1480, wait=1)
    at /usr/src/sys/kern/uipc_mbuf.c:389
389                     m = m->m_next;
Current language:  auto; currently c
(kgdb)
#21 0xc0576835 in ip_fragment (ip=0xc20d8de4, m_frag=0xe7088b48,
mtu=-1039299264,
    if_hwassist_flags=6, sw_csum=1) at /usr/src/sys/netinet/ip_output.c:967
967                     m->m_next = m_copy(m0, off, len);
(kgdb)
#22 0xc05764a6 in ip_output (m=0xc20d8d00, opt=0xc20d8de4,
ro=0xe7088b14, flags=0, imo=0x0,
    inp=0xc24317bc) at /usr/src/sys/netinet/ip_output.c:796
796             error = ip_fragment(ip, &m, ifp->if_mtu,
ifp->if_hwassist, sw_csum);
(kgdb)
#23 0xc0589954 in udp_output (inp=0xc24317bc, m=0xc20d8d00,
addr=0xc36fe6d0, control=0x0,
    td=0xc243ea80) at /usr/src/sys/netinet/udp_usrreq.c:870
870             error = ip_output(m, inp->inp_options, NULL, ipflags,
(kgdb)
#24 0xc058a5b2 in udp_send (so=0x0, flags=0, m=0x0, addr=0x0,
control=0x0, td=0x0)
    at /usr/src/sys/netinet/udp_usrreq.c:1047
1047            return udp_output(inp, m, addr, control, td);
(kgdb)
#25 0xc0520aa6 in sosend (so=0xc2443144, addr=0xc36fe6d0, uio=0x0,
top=0xc5b87b00,
    control=0x0, flags=0, td=0xc243ea80) at /usr/src/sys/kern/uipc_socket.c:835
835                         error =
(*so->so_proto->pr_usrreqs->pru_send)(so,
(kgdb)
#26 0xc05c09f4 in nfsrv_send (so=0xc2443144, nam=0xc36fe6d0, top=0x0)
at pcpu.h:157
157     {
(kgdb)
#27 0xc05c3b5a in nfssvc_nfsd (td=0x0) at
/usr/src/sys/nfsserver/nfs_syscalls.c:509
509                                 error = nfsrv_send(slp->ns_so,
nd->nd_nam2, m);
(kgdb)
#28 0xc05c2f83 in nfssvc (td=0xc243ea80, uap=0xe7088d04)
    at /usr/src/sys/nfsserver/nfs_syscalls.c:181
181                     error = nfssvc_nfsd(td);
(kgdb)
#29 0xc06453ef in syscall (frame      {tf_fs = 47, tf_es = 47, tf_ds = 47,
tf_edi = 0, tf_esi = 0,
tf_ebp = -1077941416, tf_isp = -418869916, tf_ebx = 0, tf_edx 1746232856, tf_ecx
= 25, tf_eax = 155, tf_trapno = 12, tf_err = 2,
tf_eip = 1745692359, tf_cs = 31, tf_eflags = 642, tf_esp -1077941444, tf_ss =
47}) at /usr/src/sys/i386/i386/trap.c:1009
1009                    error = (*callp->sy_call)(td, args);
(kgdb)
#30 0xc063127f in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception.s:201
201             call    syscall
Current language:  auto; currently asm
(kgdb)
#31 0x0000002f in ?? ()
(kgdb)
#32 0x0000002f in ?? ()
(kgdb)
#33 0x0000002f in ?? ()
(kgdb)
#34 0x00000000 in ?? ()
(kgdb)
#35 0x00000000 in ?? ()
(kgdb)
#36 0xbfbfeb58 in ?? ()
(kgdb)
#37 0xe7088d64 in ?? ()
(kgdb)
#38 0x00000000 in ?? ()
(kgdb)
#39 0x68156618 in ?? ()
(kgdb)
#40 0x00000019 in ?? ()
(kgdb)
#41 0x0000009b in ?? ()
(kgdb)
#42 0x0000000c in ?? ()
(kgdb)
#43 0x00000002 in ?? ()
(kgdb)
#44 0x680d26c7 in ?? ()
(kgdb)
#45 0x0000001f in ?? ()
(kgdb)
#46 0x00000282 in ?? ()
(kgdb)
#47 0xbfbfeb3c in ?? ()
(kgdb)
#48 0x0000002f in ?? ()
(kgdb)
#49 0x00000000 in ?? ()
(kgdb)
#50 0x00000000 in ?? ()
(kgdb)
#51 0x00000000 in ?? ()
(kgdb)
#52 0x00000000 in ?? ()
(kgdb)
#53 0x31efd000 in ?? ()
(kgdb)
#54 0xc244154c in ?? ()
(kgdb)
#55 0xc243ea80 in ?? ()
(kgdb)
#56 0xe7088b88 in ?? ()
(kgdb)
#57 0xe7088b70 in ?? ()
(kgdb)
#58 0xc1e95480 in ?? ()
(kgdb)
#59 0xc04f3460 in sched_switch (td=0x0, newtd=0x0, flags=Cannot access
memory at address 0xbfbfeb68
)
    at /usr/src/sys/kern/sched_4bsd.c:881
881                     cpu_switch(td, newtd);
Current language:  auto; currently c
(kgdb)
Previous frame inner to this frame (corrupt stack?)

Rong-En Fan

2005-Aug-28 14:29 UTC

head link

got a panic on 5.4-STABLE

On 8/28/05, Rong-En Fan <grafan@gmail.com> wrote:> Hi,
> 
> I got a panic on an i386 5.4-STABLE around Aug 28 with SMP enabled.
> It has 2 physical CPU with HTT enabled (so, total 4 cpus).
> This is a NFS server only with external scsi raid attached.
> 
> The console log, kgdb output and sysctl.conf are as below.
> I'll keep this core and if someone is interested, I can send any other
> information requested.
I have the following in make.conf:
CPUTYPE?= p4
CFLAGS= -O -pipe
COPTFLAGS= -O -pipe

The difference between GENERIC and my kernel is here:
http://www.rafan.org/FreeBSD/panic/m_copym/kernel-diff-against-GENERIC.txt

And I also looked at the dump file, looks like that when calling
m_copym(), m->m_len is 20, off is 1500, m->m_next is NULL
After first iteration, m becomes NULL...

#20 0xc051d62f in m_copym (m=0x0, off0=1500, len=1480, wait=1)
    at /usr/src/sys/kern/uipc_mbuf.c:389
389                     m = m->m_next;
(kgdb) l
384             while (off > 0) {
385                     KASSERT(m != NULL, ("m_copym, offset > size of
mbuf chain"));
386                     if (off < m->m_len)
387                             break;
388                     off -= m->m_len;
389                     m = m->m_next;
390             }
391             np = &top;
392             top = 0;
393             while (len > 0) {
(kgdb) p off
$15 = 1480
(kgdb) up
#21 0xc0576835 in ip_fragment (ip=0xc20d8de4, m_frag=0xe7088b48,
mtu=-1039299264,
    if_hwassist_flags=6, sw_csum=1) at /usr/src/sys/netinet/ip_output.c:967
967                     m->m_next = m_copy(m0, off, len);
(kgdb) p off
$9 = 1500
(kgdb) p len
$10 = 1480
(kgdb) p m0
$11 = (struct mbuf *) 0xc20d8d00
(kgdb) p *m0
$12 = {m_hdr = {mh_next = 0x0, mh_nextpkt = 0x0, mh_data = 0xc20d8d40
"E", mh_len = 20,
    mh_flags = 2050, mh_type = 2}, M_dat = {MH = {MH_pkthdr = {rcvif 0x0, len =
8348,
        header = 0xc5b25010, csum_flags = 0, csum_data = 6, tags {slh_first =
0x0}},
      MH_dat = {MH_ext = {
          ext_buf = 0xc33ae000 "oker</a>\nonline casinos <a
href=http://www.lucky-777-casinos.com>online casinos</a>\nviagra <a
href=http://www.ALL-VIAGRA.INFO>vip?", ext_free = 0,
          ext_args = 0x0, ext_size = 2048, ref_cnt = 0xdc050045,
ext_type = 549021963},
        MH_databuf
"\000?:?\000\000\000\000\000\000\000\000\000\b\000\000E\000\005?\vi?
@\0                                                                   
                   
021??\214p\0360\214p\036\034@\021??\214p\0360\214p\036%\b\001\003
\000xU?mJ\207?\000\000\000\0                                          
                                           001", '\0' <repeats
23
times>, "\002\000\000\001?\000\000\000\002\000\000q?\000\000\001?\000\0
                                                                      
                
000\000\000\000\000\002\000\000\000\000\000\000\000\b\000\000\000\000\001\001\217\000\021\000\000\000\000\000\000\004\034\000\000\000\000\000\031\024?C\021m?\000\000\000\000B?[?\000\000\0
                                                                      
               
000\000B?[?\000\000\000\016\fo\2117\000\016\f^?\b\000E\000\234
\vi\000\000@\021??\214p\0360\214                                      
                                                 4p\036\034\b\001\003?
\210N?"}},
    M_databuf = "\000\000\000\000\234
\000\000\020P??\000\000\000\000\006\000\000\000\000\000               
                                                                      
   0\000\000\000?:?\000\000\000\000\000\000\000\000\000\b\000\000E\000\005?\vi?
@\021??\214p\0360                                                     
                               
0\214p\036\034@\021??\214p\0360\214p\036%\b\001\003
\000xU?mJ\207?\000\000\000\001", '\0' <rep
                                                           peats 23
times>,
"\002\000\000\001?\000\000\000\002\000\000q?\000\000\001?\000\000\000\000\000
                                                                      
                
0\000\002\000\000\000\000\000\000\000\b\000\000\000\000\001\001\217\000\021\000\000\000\000\000\000\004\034\000\000\000\000\000\031\024?C\021m?\000\000\000\000B?[?\000\000\000\000B?[?\000
                                                                      
              0\000\000\016\fo\2117\000\016\f^?\b\000E\000\234
\vi\000\000@"...}}
(kgdb) l
962                             len = ip->ip_len - off;
963                             m->m_flags |= M_LASTFRAG;
964                     } else
965                             mhip->ip_off |= IP_MF;
966                     mhip->ip_len = htons((u_short)(len + mhlen));
967                     m->m_next = m_copy(m0, off, len);
968                     if (m->m_next == NULL) {        /* copy failed */
969                             m_free(m);
970                             error = ENOBUFS;        /* ??? */
971                             ipstat.ips_odropped++;
(kgdb) up
#22 0xc05764a6 in ip_output (m=0xc20d8d00, opt=0xc20d8de4,
ro=0xe7088b14, flags=0, imo=0x0,
    inp=0xc24317bc) at /usr/src/sys/netinet/ip_output.c:796
796             error = ip_fragment(ip, &m, ifp->if_mtu,
ifp->if_hwassist, sw_csum);
(kgdb) p ip
$16 = (struct ip *) 0xc20d8de4
(kgdb) p *ip
$19 = {ip_hl = 5, ip_v = 4, ip_tos = 0 '\0', ip_len = 8348, ip_id 26891,
ip_off = 0,
  ip_ttl = 64 '@', ip_p = 17 '\021', ip_sum = 54716, ip_src =
{s_addr
= 807301260},
  ip_dst = {s_addr = 471756940}}

Thanks,
Rong-En fan

freebsd stable - Aug 2005 - got a panic on 5.4-STABLE

got a panic on 5.4-STABLE

got a panic on 5.4-STABLE