Hi, guys. I have a panic running squid as web accelerator on
6.2-PRERELEASE. Here is a backtrace:
root@accel1:/usr/obj/usr/src/sys/ACCEL# kgdb kernel.debug /var/crash/vmcore.0
kgdb: kvm_nlist(_stopped_cpus):
kgdb: kvm_nlist(_stoppcbs):
[GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so:
Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for
details.
This GDB was configured as "i386-marcel-freebsd".
Unread portion of the kernel message buffer:
panic: kmem_malloc(4096): kmem_map too small: 335544320 total allocated
KDB: stack backtrace:
kdb_backtrace(100,cb83b000,c1040500,c1040500,0,...) at kdb_backtrace+0x29
panic(c06e0826,1000,14000000,dd22b6e8,1000,...) at panic+0xa8
kmem_malloc(c104b0c0,1000,102,ebc75aec,c063c285,...) at kmem_malloc+0x89
page_alloc(c103d080,1000,ebc75adf,102,38,...) at page_alloc+0x1a
slab_zalloc(c103d080,102,c103d0c8,c103d080,c1032d9c,...) at slab_zalloc+0xa1
uma_zone_slab(c103d080,2) at uma_zone_slab+0xe8
uma_zalloc_bucket(c103d080,2) at uma_zalloc_bucket+0x11c
uma_zalloc_arg(c103d080,dd18d100,2) at uma_zalloc_arg+0x2dc
mb_zinit_pack(dd18d100,100,2) at mb_zinit_pack+0x18
uma_zalloc_bucket(c103d100,3) at uma_zalloc_bucket+0x168
uma_zalloc_arg(c103d100,ebc75bfc,2) at uma_zalloc_arg+0x2dc
sosend(cbb7842c,0,ebc75cbc,0,0,0,cb83b000) at sosend+0x3c5
soo_write(daa2f870,ebc75cbc,cb68d100,0,cb83b000) at soo_write+0xa1
dofilewrite(cb83b000,1174,daa2f870,ebc75cbc,ffffffff,...) at dofilewrite+0x77
kern_writev(cb83b000,1174,ebc75cbc,cfa4000,1000,...) at kern_writev+0x3b
write(cb83b000,ebc75d04) at write+0x45
syscall(82e003b,3b,bfbe003b,e894ea0,e2e40,...) at syscall+0x25b
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip = 0xa8319bab, esp = 0xbfbe8cac, ebp =
0xbfbe8cd8 ---
KDB: enter: panic
panic: from debugger
Uptime: 5m34s
Dumping 3967 MB (3 chunks)
chunk 0: 1MB (159 pages) ... ok
chunk 1: 3966MB (1015280 pages) 3950 3934 3918 3902 3886 3870 3854 3838 3822
3806
3790 3774 3758 3742 3726 3710 3694 3678 3662 3646 3630 3614 3598 3582 3566 3550
3534 3518 3502 3486 3470 3454 3438 3422 3406 3390 3374 3358 3342 3326 3310 3294
3278 3262 3246 3230 3214 3198 3182 3166 3150 3134 3118 3102 3086 3070 3054 3038
3022 3006 2990 2974 2958 2942 2926 2910 2894 2878 2862 2846 2830 2814 2798 2782
2766 2750 2734 2718 2702 2686 2670 2654 2638 2622 2606 2590 2574 2558 2542 2526
2510 2494 2478 2462 2446 2430 2414 2398 2382 2366 2350 2334 2318 2302 2286 2270
2254 2238 2222 2206 2190 2174 2158 2142 2126 2110 2094 2078 2062 2046 2030 2014
1998 1982 1966 1950 1934 1918 1902 1886 1870 1854 1838 1822 1806 1790 1774 1758
1742 1726 1710 1694 1678 1662 1646 1630 1614 1598 1582 1566 1550 1534 1518 1502
1486 1470 1454 1438 1422 1406 1390 1374 1358 1342 1326 1310 1294 1278 1262 1246
1230 1214 1198 1182 1166 1150 1134 1118 1102 1086 1070 1054 1038 1022 1006 990
974 958 942 926 910 894 878 862 846 830 814 798 782 766 750 734 718 702 686 670
654 638 622 606 590 574 558 542 526 510 494 478 462 446 430 414 398 382 366 350
334 318 302 286 270 254 238 222 206 190 174 158 142 126 110 94 78 62 46 30 14
... ok
chunk 2: 1MB (128 pages)
#0 doadump () at pcpu.h:165
165 __asm __volatile("movl %%fs:0,%0" : "=r"
(td));
(kgdb) quit
Some additional information:
root@accel1:~# cat /boot/loader.conf
# --- Loader settings ---
autoboot_delay="5" # Delay in seconds before autobooting
# --- Kernel tunables ---
kern.ipc.nmbclusters="131072" # Set the number of mbuf clusters
kern.cam.scsi_delay="5000" # Delay (in ms) before probing SCSI
kern.maxdsiz="2560M" # Allow more memory allocation for
squid
kern.dfldsiz="2560M" # Allow more memory allocation for
squid
kern.maxssiz="128M" # Allow more memory allocation for
squid
# --- Networking modules ---
pf_load="YES" # Packet filter
# --- Other modules ---
accf_data_load="YES" # Wait for data accept filter
accf_http_load="YES" # Wait for full HTTP request accept
filter
root@accel1:~# cat /etc/sysctl.conf
# --- MAC access for squid ---
net.inet.ip.portrange.reservedlow=0
net.inet.ip.portrange.reservedhigh=0
security.mac.portacl.rules=uid:100:tcp:80
# --- Kernel tunning ---
kern.ipc.somaxconn=8192
# --- Network tunning and protection ---
kern.ipc.maxsockbuf=1048576
net.inet.tcp.sendspace=262144
net.inet.tcp.recvspace=131072
net.inet.tcp.msl=3000
net.inet.icmp.icmplim=50
net.inet.icmp.drop_redirect=1
net.inet.icmp.log_redirect=1
net.inet.ip.redirect=0
net.inet6.ip6.redirect=0
Kernel was complied with this additional options:
options KDB
options KDB_TRACE
options DDB
options ALT_BREAK_TO_DEBUGGER
options INVARIANTS
options INVARIANT_SUPPORT
options WITNESS
options DEBUG_LOCKS
options DEBUG_VFS_LOCKS
options DIAGNOSTIC
options DEBUG_MEMGUARD
options MAC
options MAC_PORTACL
If you need additional information let me know.
--
======================================================================
- Best regards, Nikolay Pavlov. <<<-----------------------------------
======================================================================
On Friday, 3 November 2006 at 17:33:42 +0200, Nikolay Pavlov wrote:> Hi, guys. I have a panic running squid as web accelerator on > 6.2-PRERELEASE. Here is a backtrace: > > root@accel1:/usr/obj/usr/src/sys/ACCEL# kgdb kernel.debug /var/crash/vmcore.0 > kgdb: kvm_nlist(_stopped_cpus): > kgdb: kvm_nlist(_stoppcbs): > [GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"] > GNU gdb 6.1.1 [FreeBSD] > Copyright 2004 Free Software Foundation, Inc. > GDB is free software, covered by the GNU General Public License, and you are > welcome to change it and/or distribute copies of it under certain conditions. > Type "show copying" to see the conditions. > There is absolutely no warranty for GDB. Type "show warranty" for details. > This GDB was configured as "i386-marcel-freebsd". > > Unread portion of the kernel message buffer: > panic: kmem_malloc(4096): kmem_map too small: 335544320 total allocated > KDB: stack backtrace: > kdb_backtrace(100,cb83b000,c1040500,c1040500,0,...) at kdb_backtrace+0x29 > panic(c06e0826,1000,14000000,dd22b6e8,1000,...) at panic+0xa8 > kmem_malloc(c104b0c0,1000,102,ebc75aec,c063c285,...) at kmem_malloc+0x89 > page_alloc(c103d080,1000,ebc75adf,102,38,...) at page_alloc+0x1a > slab_zalloc(c103d080,102,c103d0c8,c103d080,c1032d9c,...) at slab_zalloc+0xa1 > uma_zone_slab(c103d080,2) at uma_zone_slab+0xe8 > uma_zalloc_bucket(c103d080,2) at uma_zalloc_bucket+0x11c > uma_zalloc_arg(c103d080,dd18d100,2) at uma_zalloc_arg+0x2dc > mb_zinit_pack(dd18d100,100,2) at mb_zinit_pack+0x18 > uma_zalloc_bucket(c103d100,3) at uma_zalloc_bucket+0x168 > uma_zalloc_arg(c103d100,ebc75bfc,2) at uma_zalloc_arg+0x2dc > sosend(cbb7842c,0,ebc75cbc,0,0,0,cb83b000) at sosend+0x3c5 > soo_write(daa2f870,ebc75cbc,cb68d100,0,cb83b000) at soo_write+0xa1 > dofilewrite(cb83b000,1174,daa2f870,ebc75cbc,ffffffff,...) at dofilewrite+0x77 > kern_writev(cb83b000,1174,ebc75cbc,cfa4000,1000,...) at kern_writev+0x3b > write(cb83b000,ebc75d04) at write+0x45 > syscall(82e003b,3b,bfbe003b,e894ea0,e2e40,...) at syscall+0x25b > Xint0x80_syscall() at Xint0x80_syscall+0x1f > --- syscall (4, FreeBSD ELF32, write), eip = 0xa8319bab, esp = 0xbfbe8cac, ebp = 0xbfbe8cd8 --- > KDB: enter: panic > panic: from debugger > Uptime: 5m34s > Dumping 3967 MB (3 chunks) > chunk 0: 1MB (159 pages) ... ok > chunk 1: 3966MB (1015280 pages) 3950 3934 3918 3902 3886 3870 3854 3838 3822 3806 > 3790 3774 3758 3742 3726 3710 3694 3678 3662 3646 3630 3614 3598 3582 3566 3550 > 3534 3518 3502 3486 3470 3454 3438 3422 3406 3390 3374 3358 3342 3326 3310 3294 > 3278 3262 3246 3230 3214 3198 3182 3166 3150 3134 3118 3102 3086 3070 3054 3038 > 3022 3006 2990 2974 2958 2942 2926 2910 2894 2878 2862 2846 2830 2814 2798 2782 > 2766 2750 2734 2718 2702 2686 2670 2654 2638 2622 2606 2590 2574 2558 2542 2526 > 2510 2494 2478 2462 2446 2430 2414 2398 2382 2366 2350 2334 2318 2302 2286 2270 > 2254 2238 2222 2206 2190 2174 2158 2142 2126 2110 2094 2078 2062 2046 2030 2014 > 1998 1982 1966 1950 1934 1918 1902 1886 1870 1854 1838 1822 1806 1790 1774 1758 > 1742 1726 1710 1694 1678 1662 1646 1630 1614 1598 1582 1566 1550 1534 1518 1502 > 1486 1470 1454 1438 1422 1406 1390 1374 1358 1342 1326 1310 1294 1278 1262 1246 > 1230 1214 1198 1182 1166 1150 1134 1118 1102 1086 1070 1054 1038 1022 1006 990 > 974 958 942 926 910 894 878 862 846 830 814 798 782 766 750 734 718 702 686 670 > 654 638 622 606 590 574 558 542 526 510 494 478 462 446 430 414 398 382 366 350 > 334 318 302 286 270 254 238 222 206 190 174 158 142 126 110 94 78 62 46 30 14 ... ok > chunk 2: 1MB (128 pages) > > #0 doadump () at pcpu.h:165 > 165 __asm __volatile("movl %%fs:0,%0" : "=r" (td)); > (kgdb) quit > > > Some additional information: > > root@accel1:~# cat /boot/loader.conf > # --- Loader settings --- > autoboot_delay="5" # Delay in seconds before autobooting > > # --- Kernel tunables --- > kern.ipc.nmbclusters="131072" # Set the number of mbuf clusters > kern.cam.scsi_delay="5000" # Delay (in ms) before probing SCSI > kern.maxdsiz="2560M" # Allow more memory allocation for squid > kern.dfldsiz="2560M" # Allow more memory allocation for squid > kern.maxssiz="128M" # Allow more memory allocation for squid > > # --- Networking modules --- > pf_load="YES" # Packet filter > > # --- Other modules --- > accf_data_load="YES" # Wait for data accept filter > accf_http_load="YES" # Wait for full HTTP request accept filter > > root@accel1:~# cat /etc/sysctl.conf > > # --- MAC access for squid --- > net.inet.ip.portrange.reservedlow=0 > net.inet.ip.portrange.reservedhigh=0 > security.mac.portacl.rules=uid:100:tcp:80 > > > # --- Kernel tunning --- > kern.ipc.somaxconn=8192 > > # --- Network tunning and protection --- > kern.ipc.maxsockbuf=1048576 > net.inet.tcp.sendspace=262144 > net.inet.tcp.recvspace=131072 > net.inet.tcp.msl=3000 > net.inet.icmp.icmplim=50 > net.inet.icmp.drop_redirect=1 > net.inet.icmp.log_redirect=1 > net.inet.ip.redirect=0 > net.inet6.ip6.redirect=0 > > Kernel was complied with this additional options: > > options KDB > options KDB_TRACE > options DDB > options ALT_BREAK_TO_DEBUGGER > > options INVARIANTS > options INVARIANT_SUPPORT > options WITNESS > options DEBUG_LOCKS > options DEBUG_VFS_LOCKS > options DIAGNOSTIC > options DEBUG_MEMGUARD > > options MAC > options MAC_PORTACL > > If you need additional information let me know. >Forget to add this: root@accel1:~# sysctl -a | grep mem memdesc 1 4K - 1 4096 vm.memguard_divisor: 10 vm.kmem_size: 335544320 vm.kmem_size_max: 335544320 vm.kmem_size_scale: 3 vfs.ufs.dirhash_maxmem: 2097152 vfs.ufs.dirhash_mem: 33557 hw.physmem: 4151218176 hw.usermem: 4127731712 hw.realmem: 4160225280 hw.pci.host_mem_start: 2147483648 p1003_1b.memlock: 0 p1003_1b.memlock_range: 0 p1003_1b.memory_protection: 0 p1003_1b.shared_memory_objects: 1 -- ====================================================================== - Best regards, Nikolay Pavlov. <<<----------------------------------- ======================================================================
On Fri, Nov 03, 2006 at 05:33:42PM +0200, Nikolay Pavlov wrote:> Hi, guys. I have a panic running squid as web accelerator on > 6.2-PRERELEASE. Here is a backtrace: > > root@accel1:/usr/obj/usr/src/sys/ACCEL# kgdb kernel.debug /var/crash/vmcore.0 > kgdb: kvm_nlist(_stopped_cpus): > kgdb: kvm_nlist(_stoppcbs): > [GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"] > GNU gdb 6.1.1 [FreeBSD] > Copyright 2004 Free Software Foundation, Inc. > GDB is free software, covered by the GNU General Public License, and you are > welcome to change it and/or distribute copies of it under certain conditions. > Type "show copying" to see the conditions. > There is absolutely no warranty for GDB. Type "show warranty" for details. > This GDB was configured as "i386-marcel-freebsd". > > Unread portion of the kernel message buffer: > panic: kmem_malloc(4096): kmem_map too small: 335544320 total allocatedYour kernel ran out of memory. This comes up quite a lot so you can google for the solution. Kris -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 187 bytes Desc: not available Url : http://lists.freebsd.org/pipermail/freebsd-stable/attachments/20061103/99902c6a/attachment.pgp
On Friday, 3 November 2006 at 17:49:28 +0200, Nikolay Pavlov wrote:> On Friday, 3 November 2006 at 17:33:42 +0200, Nikolay Pavlov wrote: > > Hi, guys. I have a panic running squid as web accelerator on > > 6.2-PRERELEASE. Here is a backtrace: > > > > root@accel1:/usr/obj/usr/src/sys/ACCEL# kgdb kernel.debug /var/crash/vmcore.0 > > kgdb: kvm_nlist(_stopped_cpus): > > kgdb: kvm_nlist(_stoppcbs): > > [GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"] > > GNU gdb 6.1.1 [FreeBSD] > > Copyright 2004 Free Software Foundation, Inc. > > GDB is free software, covered by the GNU General Public License, and you are > > welcome to change it and/or distribute copies of it under certain conditions. > > Type "show copying" to see the conditions. > > There is absolutely no warranty for GDB. Type "show warranty" for details. > > This GDB was configured as "i386-marcel-freebsd". > > > > Unread portion of the kernel message buffer: > > panic: kmem_malloc(4096): kmem_map too small: 335544320 total allocated > > KDB: stack backtrace: > > kdb_backtrace(100,cb83b000,c1040500,c1040500,0,...) at kdb_backtrace+0x29 > > panic(c06e0826,1000,14000000,dd22b6e8,1000,...) at panic+0xa8 > > kmem_malloc(c104b0c0,1000,102,ebc75aec,c063c285,...) at kmem_malloc+0x89 > > page_alloc(c103d080,1000,ebc75adf,102,38,...) at page_alloc+0x1a > > slab_zalloc(c103d080,102,c103d0c8,c103d080,c1032d9c,...) at slab_zalloc+0xa1 > > uma_zone_slab(c103d080,2) at uma_zone_slab+0xe8 > > uma_zalloc_bucket(c103d080,2) at uma_zalloc_bucket+0x11c > > uma_zalloc_arg(c103d080,dd18d100,2) at uma_zalloc_arg+0x2dc > > mb_zinit_pack(dd18d100,100,2) at mb_zinit_pack+0x18 > > uma_zalloc_bucket(c103d100,3) at uma_zalloc_bucket+0x168 > > uma_zalloc_arg(c103d100,ebc75bfc,2) at uma_zalloc_arg+0x2dc > > sosend(cbb7842c,0,ebc75cbc,0,0,0,cb83b000) at sosend+0x3c5 > > soo_write(daa2f870,ebc75cbc,cb68d100,0,cb83b000) at soo_write+0xa1 > > dofilewrite(cb83b000,1174,daa2f870,ebc75cbc,ffffffff,...) at dofilewrite+0x77 > > kern_writev(cb83b000,1174,ebc75cbc,cfa4000,1000,...) at kern_writev+0x3b > > write(cb83b000,ebc75d04) at write+0x45 > > syscall(82e003b,3b,bfbe003b,e894ea0,e2e40,...) at syscall+0x25b > > Xint0x80_syscall() at Xint0x80_syscall+0x1f > > --- syscall (4, FreeBSD ELF32, write), eip = 0xa8319bab, esp = 0xbfbe8cac, ebp = 0xbfbe8cd8 --- > > KDB: enter: panic > > panic: from debugger > > Uptime: 5m34s > > Dumping 3967 MB (3 chunks) > > chunk 0: 1MB (159 pages) ... ok > > chunk 1: 3966MB (1015280 pages) 3950 3934 3918 3902 3886 3870 3854 3838 3822 3806 > > 3790 3774 3758 3742 3726 3710 3694 3678 3662 3646 3630 3614 3598 3582 3566 3550 > > 3534 3518 3502 3486 3470 3454 3438 3422 3406 3390 3374 3358 3342 3326 3310 3294 > > 3278 3262 3246 3230 3214 3198 3182 3166 3150 3134 3118 3102 3086 3070 3054 3038 > > 3022 3006 2990 2974 2958 2942 2926 2910 2894 2878 2862 2846 2830 2814 2798 2782 > > 2766 2750 2734 2718 2702 2686 2670 2654 2638 2622 2606 2590 2574 2558 2542 2526 > > 2510 2494 2478 2462 2446 2430 2414 2398 2382 2366 2350 2334 2318 2302 2286 2270 > > 2254 2238 2222 2206 2190 2174 2158 2142 2126 2110 2094 2078 2062 2046 2030 2014 > > 1998 1982 1966 1950 1934 1918 1902 1886 1870 1854 1838 1822 1806 1790 1774 1758 > > 1742 1726 1710 1694 1678 1662 1646 1630 1614 1598 1582 1566 1550 1534 1518 1502 > > 1486 1470 1454 1438 1422 1406 1390 1374 1358 1342 1326 1310 1294 1278 1262 1246 > > 1230 1214 1198 1182 1166 1150 1134 1118 1102 1086 1070 1054 1038 1022 1006 990 > > 974 958 942 926 910 894 878 862 846 830 814 798 782 766 750 734 718 702 686 670 > > 654 638 622 606 590 574 558 542 526 510 494 478 462 446 430 414 398 382 366 350 > > 334 318 302 286 270 254 238 222 206 190 174 158 142 126 110 94 78 62 46 30 14 ... ok > > chunk 2: 1MB (128 pages) > > > > #0 doadump () at pcpu.h:165 > > 165 __asm __volatile("movl %%fs:0,%0" : "=r" (td)); > > (kgdb) quit > > > > > > Some additional information: > > > > root@accel1:~# cat /boot/loader.conf > > # --- Loader settings --- > > autoboot_delay="5" # Delay in seconds before autobooting > > > > # --- Kernel tunables --- > > kern.ipc.nmbclusters="131072" # Set the number of mbuf clusters > > kern.cam.scsi_delay="5000" # Delay (in ms) before probing SCSI > > kern.maxdsiz="2560M" # Allow more memory allocation for squid > > kern.dfldsiz="2560M" # Allow more memory allocation for squid > > kern.maxssiz="128M" # Allow more memory allocation for squid > > > > # --- Networking modules --- > > pf_load="YES" # Packet filter > > > > # --- Other modules --- > > accf_data_load="YES" # Wait for data accept filter > > accf_http_load="YES" # Wait for full HTTP request accept filter > > > > root@accel1:~# cat /etc/sysctl.conf > > > > # --- MAC access for squid --- > > net.inet.ip.portrange.reservedlow=0 > > net.inet.ip.portrange.reservedhigh=0 > > security.mac.portacl.rules=uid:100:tcp:80 > > > > > > # --- Kernel tunning --- > > kern.ipc.somaxconn=8192 > > > > # --- Network tunning and protection --- > > kern.ipc.maxsockbuf=1048576 > > net.inet.tcp.sendspace=262144 > > net.inet.tcp.recvspace=131072 > > net.inet.tcp.msl=3000 > > net.inet.icmp.icmplim=50 > > net.inet.icmp.drop_redirect=1 > > net.inet.icmp.log_redirect=1 > > net.inet.ip.redirect=0 > > net.inet6.ip6.redirect=0 > > > > Kernel was complied with this additional options: > > > > options KDB > > options KDB_TRACE > > options DDB > > options ALT_BREAK_TO_DEBUGGER > > > > options INVARIANTS > > options INVARIANT_SUPPORT > > options WITNESS > > options DEBUG_LOCKS > > options DEBUG_VFS_LOCKS > > options DIAGNOSTIC > > options DEBUG_MEMGUARD > > > > options MAC > > options MAC_PORTACL > > > > If you need additional information let me know. > > > > Forget to add this: > > root@accel1:~# sysctl -a | grep mem > memdesc 1 4K - 1 4096 > vm.memguard_divisor: 10 > vm.kmem_size: 335544320 > vm.kmem_size_max: 335544320 > vm.kmem_size_scale: 3 > vfs.ufs.dirhash_maxmem: 2097152 > vfs.ufs.dirhash_mem: 33557 > hw.physmem: 4151218176 > hw.usermem: 4127731712 > hw.realmem: 4160225280 > hw.pci.host_mem_start: 2147483648 > p1003_1b.memlock: 0 > p1003_1b.memlock_range: 0 > p1003_1b.memory_protection: 0 > p1003_1b.shared_memory_objects: 1 >For history: http://www.freebsd.org/doc/en_US.ISO8859-1/books/faq/book.html#KMEM-MAP-TOO-SMALL is the answer for such panics. -- ====================================================================== - Best regards, Nikolay Pavlov. <<<----------------------------------- ======================================================================