Fabian Keil
2005-Dec-14 04:35 UTC
FreeBSD 6.0 panic: kmem_malloc(16384): kmem_map too small: 172728320 total allocated
I triggered a few reproducible panics on FreeBSD 6.0-STABLE.
I created a ramdisk with:
/sbin/mdconfig -a -t malloc -s 256M -u 10
/sbin/newfs -U /dev/md10
/sbin/mount /dev/md10 /mnt/ramdisk
The system has "avail memory = 515932160 (492 MB)"
and 1GB swap space.
While copying to /mnt/ramdisk trough ftp localhost
it got:
root@africanqueen ~/crashdump #kgdb kernel-GENERIC.debug vmcore.3
[GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so:
Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
[...]
Unread portion of the kernel message buffer:
panic: kmem_malloc(16384): kmem_map too small: 172728320 total allocated
Uptime: 2m57s
Dumping 511 MB (2 chunks)
chunk 0: 1MB (158 pages) ... ok
chunk 1: 511MB (130800 pages) 495 479 463 447 431 415 399 383 367 351 335 319
303 287 271 255 239 223 207 191 175 159 143 127 111 95 79 63 47 31 15
#0 doadump () at pcpu.h:165
165 pcpu.h: No such file or directory.
in pcpu.h
(kgdb) where
#0 doadump () at pcpu.h:165
#1 0xc063a4ee in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:399
#2 0xc063a784 in panic (fmt=0xc0880846 "kmem_malloc(%ld): kmem_map too
small: %ld total allocated")
at /usr/src/sys/kern/kern_shutdown.c:555
#3 0xc07a44bd in kmem_malloc (map=0xc10430c0, size=16384, flags=1026) at
/usr/src/sys/vm/vm_kern.c:299
#4 0xc079c0c6 in page_alloc (zone=0x0, bytes=16384, pflag=0x0, wait=1026) at
/usr/src/sys/vm/uma_core.c:958
#5 0xc079e41f in uma_large_malloc (size=16384, wait=1026) at
/usr/src/sys/vm/uma_core.c:2702
#6 0xc0630085 in malloc (size=16384, mtp=0xc08ffe40, flags=1026) at
/usr/src/sys/kern/kern_malloc.c:329
#7 0xc078365e in softdep_disk_io_initiation (bp=0xcd899658) at
/usr/src/sys/ufs/ffs/ffs_softdep.c:3630
#8 0xc078b1fe in ffs_geom_strategy (bo=0xc3593e90, bp=0xcd899658) at buf.h:422
#9 0xc0796869 in ufs_strategy (ap=0x0) at /usr/src/sys/ufs/ufs/ufs_vnops.c:1926
#10 0xc081c645 in VOP_STRATEGY_APV (vop=0xc09012a0, a=0xdd93ec0c) at
vnode_if.c:1796
#11 0xc06841d0 in bufstrategy (bo=0xc35f7720, bp=0x0) at vnode_if.h:928
#12 0xc067eda8 in bufwrite (bp=0xcd899658) at buf.h:415
#13 0xc067f397 in bawrite (bp=0x0) at buf.h:399
#14 0xc078b53d in ffs_syncvnode (vp=0xc35f7660, waitfor=1) at
/usr/src/sys/ufs/ffs/ffs_vnops.c:256
#15 0xc078b28e in ffs_fsync (ap=0xdd93ecc0) at
/usr/src/sys/ufs/ffs/ffs_vnops.c:179
#16 0xc081c05c in VOP_FSYNC_APV (vop=0x0, a=0x0) at vnode_if.c:1020
#17 0xc0698278 in fsync (td=0xc3460d80, uap=0x0) at vnode_if.h:537
#18 0xc080b6eb in syscall (frame {tf_fs = 59, tf_es = 59, tf_ds = 59,
tf_edi = 64, tf_esi = 134572032, tf_ebp = -1077940680, tf_isp = -5775079
96, tf_ebx = 134561920, tf_edx = 1, tf_ecx = 6, tf_eax = 95, tf_trapno = 0,
tf_err = 2, tf_eip = 672366947, tf_cs =
51, tf_eflags = 662, tf_esp = -1077945572, tf_ss = 59}) at
/usr/src/sys/i386/i386/trap.c:981
#19 0xc07fa57f in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception.s:200
#20 0x00000033 in ?? ()
Previous frame inner to this frame (corrupt stack?)
By simply copying to /mnt/ramdisk with cp I got:
root@africanqueen ~/crashdump #kgdb kernel-GENERIC.debug vmcore.4
[GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so:
Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
[...]
Unread portion of the kernel message buffer:
g_vfs_done():md10[WRITE(offset=206372864, length=131072)]error = 28
g_vfs_done():md10[WRITE(offset=206503936, length=131072)]error = 28
g_vfs_done():md10[WRITE(offset=206635008, length=131072)]error = 28
g_vfs_done():md10[WRITE(offset=206766080, length=131072)]error = 28
g_vfs_done():md10[WRITE(offset=206897152, length=131072)]error = 28
g_vfs_done():md10[WRITE(offset=207028224, length=131072)]error = 28
g_vfs_done():md10[WRITE(offset=207159296, length=131072)]error = 28
g_vfs_done():md10[WRITE(offset=207290368, length=131072)]error = 28
g_vfs_done():md10[WRITE(offset=207421440, length=131072)]error = 28
g_vfs_done():md10[WRITE(offset=207552512, length=131072)]error = 28
g_vfs_done():md10[WRITE(offset=207683584, length=131072)]error = 28
g_vfs_done():md10[WRITE(offset=207814656, length=131072)]error = 28
g_vfs_done():md10[WRITE(offset=207945728, length=131072)]error = 28
g_vfs_done():md10[WRITE(offset=208076800, length=131072)]error = 28
g_vfs_done():md10[WRITE(offset=208207872, length=131072)]error = 28
g_vfs_done():md10[WRITE(offset=208338944, length=131072)]error = 28
panic: kmem_malloc(4096): kmem_map too small: 172728320 total allocated
Uptime: 11m23s
Dumping 511 MB (2 chunks)
chunk 0: 1MB (158 pages) ... ok
chunk 1: 511MB (130800 pages) 495 479 463 447 431 415 399 383 367 351 335 319
303 287 271 255 239 223 207 191 175 159 143 127 111 95 79 63 47 31 15
#0 doadump () at pcpu.h:165
165 pcpu.h: No such file or directory.
in pcpu.h
#0 doadump () at pcpu.h:165
#1 0xc063a4ee in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:399
#2 0xc063a784 in panic (fmt=0xc0880846 "kmem_malloc(%ld): kmem_map too
small: %ld total allocated")
at /usr/src/sys/kern/kern_shutdown.c:555
#3 0xc07a44bd in kmem_malloc (map=0xc10430c0, size=4096, flags=1026) at
/usr/src/sys/vm/vm_kern.c:299
#4 0xc079c0c6 in page_alloc (zone=0xc104d800, bytes=4096, pflag=0x0, wait=1026)
at /usr/src/sys/vm/uma_core.c:958
#5 0xc079bbfd in slab_zalloc (zone=0xc104d800, wait=1026) at
/usr/src/sys/vm/uma_core.c:823
#6 0xc079d39c in uma_zone_slab (zone=0xc104d800, flags=1282) at
/usr/src/sys/vm/uma_core.c:2025
#7 0xc079d5b8 in uma_zalloc_bucket (zone=0xc104d800, flags=1282) at
/usr/src/sys/vm/uma_core.c:2134
#8 0xc079d22d in uma_zalloc_arg (zone=0xc104d800, udata=0x0, flags=1282) at
/usr/src/sys/vm/uma_core.c:1942
#9 0xc0630042 in malloc (size=64, mtp=0xc08ffec0, flags=1282) at uma.h:275
#10 0xc077fff5 in newallocindir (ip=0xc924a6b4, ptrno=0, newblkno=Unhandled
dwarf expression opcode 0x93
) at /usr/src/sys/ufs/ffs/ffs_softdep.c:1818
#11 0xc0780060 in softdep_setup_allocindir_page (ip=0xc924a6b4, lbn=4410,
bp=0xcd82e550, ptrno=302,
newblkno=101840, oldblkno=0, nbp=0xcd7c4118) at
/usr/src/sys/ufs/ffs/ffs_softdep.c:1847
#12 0xc0776562 in ffs_balloc_ufs2 (vp=0xc37a7220, startoffset=Unhandled dwarf
expression opcode 0x93
) at /usr/src/sys/ufs/ffs/ffs_balloc.c:822
#13 0xc078be30 in ffs_write (ap=0xdeb8fbec) at
/usr/src/sys/ufs/ffs/ffs_vnops.c:662
#14 0xc081bdbe in VOP_WRITE_APV (vop=0xc0900d60, a=0xdeb8fbec) at vnode_if.c:698
#15 0xc069c336 in vn_write (fp=0xc3573a68, uio=0xdeb8fcbc,
active_cred=0xc35f7c80, flags=0, td=0xc374dc00)
at vnode_if.h:372
#16 0xc065bb1b in dofilewrite (td=0xc374dc00, fd=4, fp=0xc3573a68,
auio=0xdeb8fcbc, offset=Unhandled dwarf expression opcode 0x93
) at file.h:246
#17 0xc065b9bf in kern_writev (td=0xc374dc00, fd=4, auio=0xdeb8fcbc) at
/usr/src/sys/kern/sys_generic.c:402
#18 0xc065b8e5 in write (td=0xc374dc00, uap=0x0) at
/usr/src/sys/kern/sys_generic.c:326
#19 0xc080b6eb in syscall (frame {tf_fs = 59, tf_es = 59, tf_ds = 59,
tf_edi = 134526528, tf_esi = 65536, tf_ebp = -1077941656, tf_isp = -558301852,
tf_ebx = 65536, tf_edx = 0, tf_ecx = 134607176, tf_eax = 4, tf_trapno = 32,
tf_err = 2, tf_eip = 672311759, tf_cs = 51, tf_eflags = 518, tf_esp =
-1077941732, tf_ss = 59}) at /usr/src/sys/i386/i386/trap.c:981
#20 0xc07fa57f in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception.s:200
#21 0x00000033 in ?? ()
Previous frame inner to this frame (corrupt stack?)
For the last two panics I used GENERIC, sources are from yesterday.
Fabian
--
http://www.fabiankeil.de/
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 187 bytes
Desc: not available
Url :
http://lists.freebsd.org/pipermail/freebsd-stable/attachments/20051214/4c0185da/signature.bin
Gleb Smirnoff
2005-Dec-14 05:20 UTC
FreeBSD 6.0 panic: kmem_malloc(16384): kmem_map too small: 172728320 total allocated
On Wed, Dec 14, 2005 at 01:25:30PM +0100, Fabian Keil wrote: F> I triggered a few reproducible panics on FreeBSD 6.0-STABLE. F> F> I created a ramdisk with: F> F> /sbin/mdconfig -a -t malloc -s 256M -u 10 F> /sbin/newfs -U /dev/md10 F> /sbin/mount /dev/md10 /mnt/ramdisk F> F> The system has "avail memory = 515932160 (492 MB)" F> and 1GB swap space. F> F> While copying to /mnt/ramdisk trough ftp localhost F> it got: This usually exposes some memory leak in kernel. Can you please do the following - copy some amount of data to /mnt/ramdisk trough ftp localhost, and cancel the operation before it panics. Then run vmstat -m and vmstat -z, to determine what kind of memory allocation is leaking. -- Totus tuus, Glebius. GLEBIUS-RIPN GLEB-RIPE
Fabian Keil
2005-Dec-14 06:23 UTC
FreeBSD 6.0 panic: kmem_malloc(16384): kmem_map too small: 172728320 total allocated
Gleb Smirnoff <glebius@FreeBSD.org> wrote:> On Wed, Dec 14, 2005 at 01:25:30PM +0100, Fabian Keil wrote: > F> I triggered a few reproducible panics on FreeBSD 6.0-STABLE. > F> > F> I created a ramdisk with: > F> > F> /sbin/mdconfig -a -t malloc -s 256M -u 10 > F> /sbin/newfs -U /dev/md10 > F> /sbin/mount /dev/md10 /mnt/ramdisk > F> > F> The system has "avail memory = 515932160 (492 MB)" > F> and 1GB swap space. > F> > F> While copying to /mnt/ramdisk trough ftp localhost > F> it got: > > This usually exposes some memory leak in kernel. Can you please do the > following - copy some amount of data to /mnt/ramdisk trough ftp > localhost, and cancel the operation before it panics. > > Then run vmstat -m and vmstat -z, to determine what kind of memory > allocation is leaking.I had loops with vmstat -m and vmstat -z in the background while copying to /mnt/ramdisk. The last output before the panic was: Type InUse MemUse HighUse Requests Size(s) DEVFS 22 1K - 23 16,128 pfs_nodes 20 3K - 20 128 GEOM 189 26K - 858 16,32,64,128,256,512,1024,2048,4096 isadev 17 2K - 17 64 ATA DMA 4 1K - 4 128 cdev 27 4K - 27 128 AR driver 0 0K - 11 512,2048 ACD driver 3 6K - 3 2048 file desc 120 46K - 1611 16,32,256,512,2048 sigio 2 1K - 3 32 kenv 96 7K - 97 16,32,64,4096 kqueue 0 0K - 62 256,1024 proc-args 43 2K - 797 16,32,64,128 zombie 0 0K - 907 128 ithread 48 5K - 49 64,128 KTRACE 100 13K - 100 128 CAM SIM 1 1K - 1 64 linker 68 3K - 99 16,32,256 CAM XPT 10 1K - 17 16,64,512 lockf 3 1K - 3 64 devbuf 1346 3177K - 1816 16,32,64,128,256,512,1024,2048,4096 temp 16 171K - 6266 16,32,64,128,256,512,1024,2048,4096 ip6opt 1 1K - 1 128 ip6ndp 6 1K - 7 64,128 module 371 24K - 371 64,128 mtx_pool 1 8K - 1 pgrp 36 3K - 623 64 session 29 4K - 47 128 proc 2 4K - 2 2048 subproc 209 413K - 1116 256,4096 cred 35 5K - 4132 128 plimit 18 5K - 400 256 uidinfo 4 1K - 20 32,512 sysctl 0 0K - 619 16,32,64 sysctloid 2567 77K - 2567 16,32,64 sysctltmp 0 0K - 280 16,32,128 umtx 120 8K - 120 64 SWAP 2 141K - 2 64 bus 959 38K - 3599 16,32,64,128,1024 bus-sc 57 27K - 1537 16,32,64,128,256,512,1024,2048,4096 devstat 18 37K - 18 16,4096 eventhandler 37 3K - 37 32,128 kobj 248 496K - 299 2048 MD disk 294 7K - 294 16,2048 MD sectors 293 1172K - 293 4096 rman 149 10K - 570 16,64 sbuf 0 0K - 440 16,32,64,128,256,512,1024,2048,4096 sleep queues 121 4K - 121 32 taskqueue 6 1K - 6 128 turnstiles 121 8K - 121 64 Unitno 7 1K - 9 16,64 ioctlops 0 0K - 2757 16,32,64,256,512,1024,4096 iov 0 0K - 487 16,64,128 msg 4 25K - 4 1024,4096 sem 4 7K - 4 512,1024,4096 shm 1 12K - 1 ttys 1228 174K - 3223 128,1024 ptys 3 1K - 3 128 mbuf_tag 0 0K - 6 32,64 soname 6 1K - 735 16,32,128 pcb 29 5K - 81 16,32,64,2048 BIO buffer 0 0K - 99 2048 vfscache 1 256K - 1 cluster_save buffer 0 0K - 19 32,64 Export Host 1 1K - 2 256 VFS hash 1 128K - 1 vnodes 1 1K - 1 128 mount 130 12K - 641 16,32,64,128,512,1024,2048 CAM periph 1 1K - 1 128 BPF 4 1K - 4 64 ifnet 5 5K - 5 256,1024 ifaddr 40 10K - 40 16,32,64,256,512,2048 ether_multi 40 2K - 46 16,32,64 clone 4 16K - 4 4096 arpcom 2 1K - 2 16 lo 1 1K - 1 16 ATA generic 5 5K - 22 16,512,1024 CAM queue 3 1K - 3 16 CAM dev queue 1 1K - 1 64 routetbl 30 3K - 62 16,32,64,128,256 in_multi 2 1K - 2 32 hostcache 1 24K - 1 syncache 1 8K - 1 in6_multi 12 1K - 12 16,64 ip6_moptions 1 1K - 1 16 NFS srvsock 2 1K - 2 128 NFS daemon 5 3K - 5 256,512 agp 2 257K - 2 16 p1003.1b 1 1K - 1 16 pagedep 2 17K - 32 64 inodedep 11 130K - 116 128 newblk 1 1K - 15816 64,256 bmsafemap 3 1K - 76 32 allocdirect 14 2K - 573 128 indirdep 3 1K - 311 32 allocindir 3333 209K - 15242 64 freefrag 0 0K - 91 32 freeblks 2 1K - 32 256 freefile 1 1K - 36 32 diradd 1 1K - 78 32 mkdir 0 0K - 12 32 dirrem 0 0K - 68 32 savedino 0 0K - 34 256 UFS dirhash 48 9K - 48 16,32,512 UFS mount 18 38K - 30 64,256,512,1024,2048,4096 UMAHash 2 257K - 13 256,512,1024,2048,4096 AD driver 2 1K - 2 32 ppbusdev 3 1K - 3 128 entropy 1024 64K - 1024 64 VM pgdata 2 33K - 2 64 atkbddev 2 1K - 2 32 USB 31 3K - 31 16,32,64,128,256 USBdev 2 1K - 8 128,512 DEVFS2 105 2K - 105 16 memdesc 1 4K - 1 4096 nexusdev 3 1K - 3 16 DEVFS3 237 30K - 238 128 DEVFS1 112 28K - 112 256 DEVFS_RULE 34 8K - 34 32,256 linux 10 1K - 10 32 acpica 1507 84K - 19881 16,32,64,128,256,512,1024,2048 acpitask 0 0K - 2 32 acpidev 41 2K - 41 32 acpisem 18 2K - 18 64 PCI Link 8 1K - 8 64,128 ITEM SIZE LIMIT USED FREE REQUESTS UMA Kegs: 140, 0, 63, 9, 63 UMA Zones: 120, 0, 63, 27, 63 UMA Slabs: 64, 0, 37254, 34, 38294 UMA RCntSlabs: 104, 0, 67, 7, 67 UMA Hash: 128, 0, 2, 28, 4 16 Bucket: 76, 0, 24, 26, 28 32 Bucket: 140, 0, 20, 8, 25 64 Bucket: 268, 0, 22, 6, 29 128 Bucket: 524, 0, 67, 3, 70 VM OBJECT: 132, 0, 1518, 135, 20178 MAP: 192, 0, 7, 33, 7 KMAP ENTRY: 68, 31976, 15, 153, 41589 MAP ENTRY: 68, 0, 1265, 191, 59199 PV ENTRY: 24, 933800, 12742, 2193, 513804 DP fakepg: 72, 0, 0, 0, 0 mt_zone: 64, 0, 195, 100, 195 16: 16, 0, 2869, 176, 24798 32: 32, 0, 1998, 149, 4847 64: 64, 0, 3973, 3461, 38988 128: 128, 0, 1994, 106, 12191 256: 256, 0, 425, 25, 2784 512: 512, 0, 51, 5, 301 1024: 1024, 0, 61, 71, 1511 2048: 2048, 0, 283, 57, 862 4096: 4096, 0, 442, 13, 5057 Files: 72, 0, 114, 98, 5926 PROC: 524, 0, 95, 17, 1004 THREAD: 372, 0, 112, 8, 112 KSEGRP: 88, 0, 112, 48, 112 UPCALL: 44, 0, 0, 0, 0 VMSPACE: 300, 0, 43, 22, 952 mbuf_packet: 256, 0, 21, 107, 118780 mbuf: 256, 0, 1, 141, 160524 mbuf_cluster: 2048, 17088, 128, 6, 128 ACL UMA zone: 388, 0, 0, 0, 0 g_bio: 132, 0, 0, 348, 162122 ata_request: 200, 0, 0, 95, 40954 ata_composite: 192, 0, 0, 0, 0 VNODE: 272, 0, 792, 20, 830 VNODEPOLL: 76, 0, 0, 0, 0 S VFS Cache: 68, 0, 758, 82, 1542 L VFS Cache: 291, 0, 0, 0, 0 NAMEI: 1024, 0, 3, 9, 17240 DIRHASH: 1024, 0, 47, 9, 47 NFSMOUNT: 480, 0, 0, 0, 0 NFSNODE: 460, 0, 0, 0, 0 PIPE: 408, 0, 4, 14, 216 KNOTE: 68, 0, 0, 112, 62 socket: 356, 17094, 48, 7, 294 unpcb: 140, 17108, 14, 42, 135 udpcb: 180, 17094, 14, 30, 95 inpcb: 180, 17094, 23, 43, 63 tcpcb: 460, 17088, 20, 12, 63 tcptw: 48, 3432, 3, 153, 10 syncache: 100, 15366, 0, 78, 18 hostcache: 76, 15400, 2, 98, 2 tcpreass: 20, 1183, 0, 0, 0 sackhole: 20, 0, 0, 0, 0 ripcb: 180, 17094, 0, 0, 0 rtentry: 132, 0, 14, 44, 15 SWAPMETA: 276, 63266, 0, 0, 0 FFS inode: 132, 0, 740, 43, 777 FFS1 dinode: 128, 0, 0, 0, 0 FFS2 dinode: 256, 0, 740, 10, 777 md10: 512, 0, 292351, 41, 292418 Fabian -- http://www.fabiankeil.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 187 bytes Desc: not available Url : http://lists.freebsd.org/pipermail/freebsd-stable/attachments/20051214/36724338/signature.bin