This panic comes from the system where NFS is served from NULLFS. See http://bugs.freebsd.org/164261 for the full description. Wed Jan 18 11:33:51 MSK 2012 FreeBSD XXX 10.0-CURRENT FreeBSD 10.0-CURRENT #12 r230080M: Wed Jan 18 11:25:12 MSK 2012 root@XXX:/sys/amd64/compile/SHADOW-CURRENT amd64 panic: from debugger GNU gdb 6.1.1 [FreeBSD] Copyright 2004 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "amd64-marcel-freebsd"... Unread portion of the kernel message buffer: svc_thread_start() at svc_run() at svc_thread_start+0xb svc_run+0x8b fork_exit() at nfsrvd_nfsd() at fork_exit+0x114 nfsrvd_nfsd+0x177 fork_trampoline() at nfssvc_nfsd() at fork_trampoline+0xe --- trap 0xc, rip = 0x8008ab6ac, rsp = 0x7fffffffd698, rbp = 0x5 --- insmntque: mp-safe fs and non-locked vp: 0xfffffe0006ca5000 is not exclusive locked but should be KDB: enter: lock violation nUptime: 48s Dumping 301 out of 4083 MB: Reading symbols from /boot/kernel/zfs.ko...Reading symbols from /boot/kernel/zfs.ko.symbols...done. done. Loaded symbols for /boot/kernel/zfs.ko Reading symbols from /boot/kernel/opensolaris.ko...Reading symbols from /boot/kernel/opensolaris.ko.symbols...done. done. Loaded symbols for /boot/kernel/opensolaris.ko Reading symbols from /boot/kernel/wpifw.ko...Reading symbols from /boot/kernel/wpifw.ko.symbols...done. done. Loaded symbols for /boot/kernel/wpifw.ko #0 doadump (textdump=1) at pcpu.h:224 224 pcpu.h: No such file or directory. in pcpu.h (kgdb) #0 doadump (textdump=1) at pcpu.h:224 #1 0xffffffff80532218 in kern_reboot (howto=260) at ../../../kern/kern_shutdown.c:455 #2 0xffffffff8053271d in panic (fmt=0x0) at ../../../kern/kern_shutdown.c:643 #3 0xffffffff802ece90 in db_panic (addr=Variable "addr" is not available. ) at ../../../ddb/db_command.c:482 #4 0xffffffff802ed281 in db_command (last_cmdp=0xffffffff80b3d980, cmd_table=Variable "cmd_table" is not available. ) at ../../../ddb/db_command.c:449 #5 0xffffffff802ed4c9 in db_command_loop () at ../../../ddb/db_command.c:502 #6 0xffffffff802ef4a7 in db_trap (type=Variable "type" is not available. ) at ../../../ddb/db_main.c:229 #7 0xffffffff80565d51 in kdb_trap (type=3, code=0, tf=0xffffff811deef170) at ../../../kern/subr_kdb.c:629 #8 0xffffffff807421a2 in trap (frame=0xffffff811deef170) at ../../../amd64/amd64/trap.c:591 #9 0xffffffff8072d37f in calltrap () at ../../../amd64/amd64/exception.S:228 #10 0xffffffff80565aff in kdb_enter (why=0xffffffff8084bfea "vfslock", msg=0x780
) at cpufunc.h:63 #11 0xffffffff805c4ad4 in assert_vop_elocked (vp=0xfffffe0006ca5000, str=0xffffffff8084cc80 "insmntque: mp-safe fs and non-locked vp") at ../../../kern/vfs_subr.c:3848 #12 0xffffffff805c4e7d in insmntque1 (vp=0xfffffe0006ca5000, mp=0xfffffe0006b0bc00, dtr=0xffffffff804b9ff4 , dtr_arg=0xfffffe0006053720) at ../../../kern/vfs_subr.c:1124 #13 0xffffffff804b9e4b in null_nodeget (mp=0xfffffe0006b0bc00, lowervp=0xfffffe000623b960, vpp=0xffffff811deef6f0) at ../../../fs/nullfs/null_subr.c:246 #14 0xffffffff804b138e in nfsrvd_readdirplus (nd=0xffffff811deef990, isdgram=1, vp=0xfffffe0006df81e0, p=0xfffffe0006bbf000, exp=0xffffff811deef830) at ../../../fs/nfsserver/nfs_nfsdport.c:2044 #15 0xffffffff804913b0 in nfsrvd_dorpc (nd=0xffffff811deef990, isdgram=1, p=0xfffffe0006bbf000) at ../../../fs/nfsserver/nfs_nfsdsocket.c:440 #16 0xffffffff8049f9a0 in nfssvc_program (rqst=0xfffffe0006487800, xprt=0xfffffe0006b23600) at ../../../fs/nfsserver/nfs_nfsdkrpc.c:328 #17 0xffffffff806ac817 in svc_run_internal (pool=0xfffffe0001df6500, ismaster=0) at ../../../rpc/svc.c:895 #18 0xffffffff806acbe5 in svc_thread_start (arg=Variable "arg" is not available. ) at ../../../rpc/svc.c:1200 #19 0xffffffff80507d8f in fork_exit ( callout=0xffffffff806acbda , arg=0xfffffe0001df6500, frame=0xffffff811deefc50) at ../../../kern/kern_fork.c:995 #20 0xffffffff8072d83e in fork_trampoline () at ../../../amd64/amd64/exception.S:602 #21 0x0000000000000080 in ?? () #22 0x00007fffffffd6d0 in ?? () #23 0x000000000000002e in ?? () #24 0x0000000000000000 in ?? () #25 0xfffffffffffffef4 in ?? () #26 0x0000000000000002 in ?? () #27 0x000000000000009b in ?? () #28 0x00007fffffffd6d0 in ?? () #29 0x0000000000000005 in ?? () #30 0x0000000000000003 in ?? () #31 0x00000000ffffffff in ?? () #32 0x00007fffffffd6f0 in ?? () #33 0x0000000000000000 in ?? () #34 0x0000000000000001 in ?? () #35 0x0000000000000001 in ?? () #36 0x001b00130000000c in ?? () #37 0x00007fffffffefe8 in ?? () #38 0x003b003b00000001 in ?? () #39 0x0000000000000002 in ?? () #40 0x00000008008ab6ac in ?? () #41 0x0000000000000043 in ?? () #42 0x0000000000000206 in ?? () #43 0x00007fffffffd698 in ?? () #44 0x000000000000003b in ?? () #45 0xffffffff80b7b700 in tdq_cpu () #46 0xffffffff80b7b700 in tdq_cpu () #47 0x0000000000000000 in ?? () #48 0xfffffe0006bbf000 in ?? () #49 0xffffff811deeed00 in ?? () #50 0xffffff811deeeca8 in ?? () #51 0xffffffff80b7b700 in tdq_cpu () #52 0xffffffff80559793 in sched_switch (td=0xffffffff806acbda, newtd=Variable "newtd" is not available. ) at ../../../kern/sched_ule.c:1854 Previous frame inner to this frame (corrupt stack?) (kgdb) ------------------------------------------------------------------------ ps -axl UID PID PPID CPU PRI NI VSZ RSS MWCHAN STAT TT TIME COMMAND 0 0 0 0 -8 0 0 0 - DLs - 0:00.15 [kernel] 0 1 0 0 52 0 6280 0 wait DLs - 0:00.00 [init] 0 2 0 0 -16 0 0 0 crypto_w DL - 0:00.00 [crypto] 0 3 0 0 -16 0 0 0 crypto_r DL - 0:00.00 [crypto returns] 0 4 0 0 -16 0 0 0 - DL - 0:00.00 [fw0_probe] 0 5 0 0 -16 0 0 0 pftm DL - 0:00.00 [pfpurge] 0 6 0 0 -16 0 0 0 ccb_scan DL - 0:00.00 [xpt_thrd] 0 7 0 0 -16 0 0 0 psleep DL - 0:00.00 [pagedaemon] 0 8 0 0 -16 0 0 0 psleep DL - 0:00.00 [vmdaemon] 0 9 0 0 155 0 0 0 pgzero DL - 0:00.00 [pagezero] 0 10 0 0 -16 0 0 0 audit_wo DL - 0:00.00 [audit] 0 11 0 0 155 0 0 0 - RL - 0:16.65 [idle] 0 12 0 0 -84 0 0 0 - WL - 0:00.04 [intr] 0 13 0 0 -16 0 0 0 sleep DL - 0:00.00 [ng_queue] 0 14 0 0 -8 0 0 0 - DL - 0:00.04 [geom] 0 15 0 0 -16 0 0 0 - DL - 0:00.00 [yarrow] 0 16 0 0 -68 0 0 0 - DL - 0:00.01 [usb] 0 17 0 0 -16 0 0 0 tzpoll DL - 0:00.00 [acpi_thermal] 0 18 0 0 -16 0 0 0 cooling DL - 0:00.00 [acpi_cooling0] 0 19 0 0 -16 0 0 0 psleep DL - 0:00.00 [bufdaemon] 0 20 0 0 -16 0 0 0 vlruwt DL - 0:00.00 [vnlru] 0 21 0 0 16 0 0 0 syncer DL - 0:00.00 [syncer] 0 22 0 0 -16 0 0 0 sdflush DL - 0:00.00 [softdepflush] 0 49 0 0 -8 0 0 0 tx->tx_s DL - 0:00.00 [zfskern] 0 403 1 0 20 0 18376 0 select Ds - 0:00.00 [wpa_supplicant] 0 795 1 0 24 0 10372 0 select Ds - 0:00.00 [devd] 0 1114 1 0 20 0 12208 0 select Ds - 0:00.01 [syslogd] 53 1211 1 0 52 0 47224 0 kqread Ds - 0:00.07 [named] 0 1226 1 0 20 0 14288 0 select Ds - 0:00.01 [rpcbind] 0 1297 1 0 21 0 12204 0 select Ds - 0:00.00 [mountd] 0 1307 1 0 52 0 10072 0 accept Ds - 0:00.02 [nfsd] 0 1308 1307 0 52 0 10072 0 - R - 0:00.00 [nfsd] 0 1314 1 0 20 0 274312 0 select Ds - 0:00.00 [rpc.statd] 0 1320 1 0 52 0 14288 0 rpcsvc Ds - 0:00.00 [rpc.lockd] 0 1385 1 0 20 0 22308 0 select Ds - 0:00.01 [ntpd] 0 1392 1 0 20 0 12204 0 select Ds - 0:00.00 [powerd] 0 1530 1 0 52 0 18532 0 kqread Ds - 0:00.01 [master] 125 1540 1530 0 52 0 18536 0 kqread D - 0:00.01 [pickup] 125 1541 1530 0 52 0 18532 0 kqread D - 0:00.01 [qmgr] 5999 1573 1 0 52 0 10060 0 select Ds - 0:00.01 [cvsupd] 0 1620 1 0 20 0 14288 0 nanslp Ds - 0:00.00 [cron] 0 1653 1 0 52 0 16368 0 select Ds - 0:00.00 [inetd] 0 1683 1 0 52 0 6276 0 wait D - 0:00.00 [sh] 0 1684 1 0 52 0 10072 0 piperd D - 0:00.00 [logger] 0 1685 1683 0 52 0 6276 0 nanslp D - 0:00.00 [sleep] 0 1688 1 0 22 0 41320 0 wait Ds - 0:00.00 [login] 0 1689 1 0 52 0 12208 0 ttyin Ds+ - 0:00.00 [getty] 0 1690 1 0 52 0 12208 0 ttyin Ds+ - 0:00.00 [getty] 0 1691 1 0 52 0 12208 0 ttyin Ds+ - 0:00.00 [getty] 0 1692 1688 0 24 0 6276 0 pause D - 0:00.03 [csh] 0 1704 1692 0 35 0 41316 0 wait D - 0:00.00 [su] 1001 1705 1704 0 52 0 6276 0 wait D - 0:00.14 [bash] 1001 1732 1 0 52 0 36188 0 select Ds - 0:00.00 [ssh-agent] 1001 1747 1 0 29 0 24000 0 select Ds - 0:00.00 [gpg-agent] 1001 1753 1705 0 52 0 6276 0 wait D+ - 0:00.00 [sh] 1001 1755 1753 0 52 0 10076 0 wait D+ - 0:00.00 [find] 1001 1756 1753 0 52 0 10076 0 wait D+ - 0:00.00 [find] 1001 1757 1753 0 52 0 10076 0 wait D+ - 0:00.00 [find] 1001 1758 1756 0 52 0 6276 0 newnfsre D+ - 0:00.00 [ls] 1001 1759 1755 0 52 0 6276 0 newnfsre D+ - 0:00.00 [ls] 1001 1760 1757 0 52 0 6276 0 newnfsre D+ - 0:00.00 [ls] ------------------------------------------------------------------------ vmstat -s 80383 cpu context switches 246930 device interrupts 8269 software interrupts 174112 traps 236116 system calls 23 kernel threads created 1720 fork() calls 17 vfork() calls 0 rfork() calls 0 swap pager pageins 0 swap pager pages paged in 0 swap pager pageouts 0 swap pager pages paged out 768 vnode pager pageins 6767 vnode pager pages paged in 1 vnode pager pageouts 1 vnode pager pages paged out 0 page daemon wakeups 0 pages examined by the page daemon 197 pages reactivated 48261 copy-on-write faults 113 copy-on-write optimized faults 95304 zero fill pages zeroed 0 zero fill pages prezeroed 6 intransit blocking page faults 188080 total VM faults taken 0 pages affected by kernel thread creation 517098 pages affected by fork() 6938 pages affected by vfork() 0 pages affected by rfork() 0 pages cached 172607 pages freed 0 pages freed by daemon 0 pages freed by exiting processes 7834 pages active 6944 pages inactive 53 pages in VM cache 35064 pages wired down 956731 pages free 4096 bytes per page 37180 total name lookups cache hits (84% pos + 6% neg) system 0% per-directory deletions 0%, falsehits 0%, toolong 0% ------------------------------------------------------------------------ vmstat -m Type InUse MemUse HighUse Requests Size(s) kbdmux 6 18K - 6 16,512,1024,2048 LED 18 2K - 18 16,128 CAM queue 22 1K - 181 16,32,64,128,256 GEOM 159 41K - 1764 16,32,64,128,256,512,1024,2048 CAM SIM 6 2K - 6 256 acpisem 41 6K - 41 128 scsi_cd 0 0K - 6 16 CAM periph 6 2K - 27 16,32,64,128,256 CAM XPT 80 93K - 248 32,64,128,256,1024,2048 ata_generic 1 1K - 1 1024 ata_pci 1 1K - 1 64 isadev 7 1K - 7 128 acd_driver 1 2K - 1 2048 ata_cam 1 1K - 22 64,256,1024 entropy 1024 64K - 1024 64 cdev 9 3K - 9 256 CAM dev queue 6 1K - 6 128 sigio 1 1K - 1 64 filedesc 80 75K - 1873 16,32,64,128,512,1024,2048,4096 kenv 78 11K - 84 16,32,64,128 kqueue 8 15K - 33 256,2048 proc-args 37 3K - 762 16,32,64,128,256 hhook 2 1K - 2 128 hdac 10 32K - 10 64,128,256,512,2048,4096 ithread 82 13K - 82 32,128,256 KTRACE 100 13K - 100 128 linker 168 807K - 194 16,32,64,128,256,512,1024,2048,4096 lockf 29 3K - 65 64,128 loginclass 2 1K - 4 64 drm_ctxbitmap 1 4K - 1 4096 temp 58 52K - 8202 16,32,64,128,256,512,1024,2048,4096 devbuf 22698 47656K - 23447 16,32,64,128,256,512,1024,2048,4096 module 278 35K - 278 128 mtx_pool 2 16K - 2 osd 3 1K - 6 16,64,128 feeder 14 2K - 17 32,128 subproc 135 281K - 1835 512,4096 proc 2 16K - 2 session 23 3K - 24 128 pgrp 27 4K - 33 128 cred 172 27K - 7936 64,256 uidinfo 6 3K - 9 128,2048 plimit 15 4K - 130 256 sysctltmp 0 0K - 337 16,32,64,128 sysctloid 4747 234K - 4858 16,32,64,128 sysctl 0 0K - 1200 16,32,64 tidhash 1 16K - 1 callout 1 512K - 1 umtx 612 77K - 612 128 p1003.1b 1 1K - 1 16 SWAP 2 821K - 2 64 drm_maps 1 1K - 1 128 bus-sc 83 389K - 1410 16,32,64,128,256,512,1024,2048,4096 bus 841 75K - 3742 16,32,64,128,256,1024 devstat 20 41K - 20 32,4096 eventhandler 87 7K - 87 64,128 mixer 2 8K - 2 4096 kobj 178 712K - 205 4096 Per-cpu 1 1K - 1 32 drm_driver 4 3K - 8 16,32,64,128,512,2048 drm_sarea 1 1K - 1 16 rman 253 29K - 601 16,32,128 sbuf 0 0K - 1607 16,32,64,128,256,512,1024,2048,4096 fw_xfer 0 0K - 1 256 stack 0 0K - 2 256 taskqueue 57 6K - 87 16,32,64,128,1024 Unitno 19 2K - 599 32,64 iov 0 0K - 506 16,64,128,256,512 select 25 4K - 25 128 ioctlops 0 0K - 1654 16,32,64,128,256,512,1024,2048,4096 msg 4 30K - 4 2048,4096 sem 4 106K - 4 2048,4096 shm 1 20K - 1 tty 17 17K - 19 1024,2048 mbuf_tag 0 0K - 18 32 shmfd 1 8K - 1 firewire 11 35K - 14 64,128,512,1024,2048,4096 pcb 30 13K - 66 16,32,64,1024,2048,4096 soname 37 4K - 774 16,32,64,128 vfscache 1 2048K - 1 export_host 1 1K - 5 256 vfs_hash 1 1024K - 1 vnodes 4 1K - 4 64,256 vnodemarker 0 0K - 51 512 mount 284 13K - 2703 16,32,64,128,256,512 iconv 2 1K - 2 128 BPF 17 1027K - 18 16,128,512 ether_multi 2 1K - 2 16,64 ifaddr 183 22K - 183 32,256,512,2048,4096 ifnet 14 27K - 14 128,2048 USBdev 37 10K - 37 64,128,1024 arpcom 3 1K - 3 16 lltable 13 7K - 13 512 USB 66 107K - 70 16,32,128,256,512,2048 acpidev 44 3K - 44 64 sbp 96 14K - 96 32,256 ddb_capture 1 48K - 1 acpiintr 1 1K - 1 64 acpica 4761 469K - 51712 16,32,64,128,256,512,1024,2048 routetbl 134 67K - 220 64,128,256,512 80211vap 1 4K - 1 4096 80211com 1 8K - 1 80211nodeie 3 1K - 6 32,128 80211node 1 12K - 2 80211ratectl 2 1K - 3 16,64 80211scan 4 7K - 4 512,2048,4096 netgraph_hci 1 1K - 1 256 netgraph_l2cap 1 1K - 1 128 netgraph_btsocks_hci_raw 1 8K - 23 128 netgraph_btsocks_l2cap 1 1K - 1 32 netgraph_btsocks_l2cap_raw 1 1K - 1 32 pci_link 16 2K - 16 16,32,128 acpi_perf 2 1K - 2 128 acpitask 1 2K - 1 2048 netgraph_node 7 2K - 16 256 netgraph_hook 10 2K - 10 128 netgraph_msg 0 0K - 40 64,128,256,1024 netgraph_sock 0 0K - 9 128 netgraph_path 0 0K - 11 16 igmp 13 4K - 13 256 DEVFS1 127 64K - 150 512 DEVFS3 292 73K - 304 256 DEVFS2 127 2K - 127 16 in_multi 1 1K - 1 256 IpFw/IpAcct 24 3K - 46 16,32,64,128,256 hostcache 1 28K - 1 syncache 1 96K - 1 NLM 0 0K - 1 16 crypto 1 1K - 1 512 rpc 67 19K - 188 16,32,64,128,256,512,1024,2048 audit_evclass 179 6K - 218 32 savedino 0 0K - 52 256 freework 1 1K - 18 16,128 newdirblk 0 0K - 6 64 dirrem 0 0K - 51 128 mkdir 0 0K - 12 128 diradd 4 1K - 61 128 freefile 24 2K - 48 64 freeblks 0 0K - 16 128 newblk 1 128K - 25 256 bmsafemap 1 8K - 15 256 inodedep 30 1039K - 113 512 pagedep 2 129K - 20 256 ufs_dirhash 81 17K - 81 16,32,64,128,256,512 ufs_mount 21 88K - 21 512,4096 vm_pgdata 2 129K - 2 128 UMAHash 1 1K - 1 512 DEVFS_RULE 60 28K - 116 64,512 DEVFS 40 1K - 41 16,128 DEVFSP 1 1K - 1 64 memdesc 1 4K - 1 4096 atkbddev 2 1K - 2 64 NFS fh 3 1K - 27 32 linux 15 1K - 15 64 NFSD srvcache 3 1K - 18 128 apmdev 1 1K - 1 128 newnfsmnt 3 3K - 3 1024 ntfs_nthash 1 1024K - 1 io_apic 1 2K - 1 2048 msi 4 1K - 4 128 nexusdev 4 1K - 4 16 nullfs_node 4 1K - 4 32 nullfs_hash 1 1K - 1 128 nullfs_mount 1 1K - 1 16 pfs_nodes 73 19K - 73 256 solaris 47153 17377K - 71415 16,32,64,128,256,512,1024,2048,4096 kstat_data 4 1K - 4 64 ------------------------------------------------------------------------ vmstat -z ITEM SIZE LIMIT USED FREE REQ FAIL SLEEP UMA Kegs: 208, 0, 194, 10, 194, 0, 0 UMA Zones: 640, 0, 194, 4, 194, 0, 0 UMA Slabs: 568, 0, 1644, 36, 2528, 0, 0 UMA RCntSlabs: 568, 0, 214, 3, 214, 0, 0 UMA Hash: 256, 0, 80, 10, 81, 0, 0 16 Bucket: 152, 0, 86, 14, 86, 0, 0 32 Bucket: 280, 0, 71, 13, 71, 0, 0 64 Bucket: 536, 0, 66, 4, 66, 57, 0 128 Bucket: 1048, 0, 54, 0, 54, 582, 0 VM OBJECT: 232, 0, 1482, 38, 18980, 0, 0 MAP: 232, 0, 7, 25, 7, 0, 0 KMAP ENTRY: 120, 150505, 52, 227, 14200, 0, 0 MAP ENTRY: 120, 0, 760, 77, 22150, 0, 0 fakepg: 120, 0, 0, 0, 0, 0, 0 mt_zone: 4112, 0, 311, 28, 311, 0, 0 16: 16, 0, 2850, 342, 30098, 0, 0 32: 32, 0, 2517, 412, 9739, 0, 0 64: 64, 0, 58354, 166, 99588, 0, 0 128: 128, 0, 8690, 126, 13233, 0, 0 256: 256, 0, 892, 113, 7226, 0, 0 512: 512, 0, 823, 108, 9178, 0, 0 1024: 1024, 0, 80, 632, 4213, 0, 0 2048: 2048, 0, 259, 35, 1310, 0, 0 4096: 4096, 0, 504, 50, 8530, 0, 0 Files: 80, 0, 208, 62, 6610, 0, 0 TURNSTILE: 136, 0, 307, 53, 307, 0, 0 umtx pi: 96, 0, 0, 0, 0, 0, 0 PROC: 1160, 0, 60, 12, 1760, 0, 0 THREAD: 1112, 0, 285, 21, 423, 0, 0 SLEEPQUEUE: 80, 0, 307, 41, 307, 0, 0 VMSPACE: 392, 0, 38, 32, 1739, 0, 0 cpuset: 72, 0, 2, 98, 2, 0, 0 audit_record: 960, 0, 0, 0, 0, 0, 0 mbuf_packet: 256, 0, 0, 261, 199, 0, 0 mbuf: 256, 0, 77, 312, 908, 0, 0 mbuf_cluster: 2048, 25600, 259, 15, 287, 0, 0 mbuf_jumbo_page: 4096, 12800, 64, 13, 93, 0, 0 mbuf_jumbo_9k: 9216, 19200, 0, 0, 0, 0, 0 mbuf_jumbo_16k: 16384, 12800, 0, 0, 0, 0, 0 mbuf_ext_refcnt: 4, 0, 0, 0, 0, 0, 0 NetGraph items: 72, 4118, 0, 87, 60, 0, 0 NetGraph data items: 72, 522, 0, 87, 55, 0, 0 g_bio: 232, 0, 0, 272, 13396, 0, 0 ttyinq: 160, 0, 720, 24, 795, 0, 0 ttyoutq: 256, 0, 372, 33, 412, 0, 0 ata_request: 320, 0, 0, 36, 53, 0, 0 ata_composite: 336, 0, 0, 0, 0, 0, 0 cryptop: 88, 0, 0, 0, 0, 0, 0 cryptodesc: 72, 0, 0, 0, 0, 0, 0 VNODE: 480, 0, 1124, 28, 1223, 0, 0 VNODEPOLL: 112, 0, 0, 0, 0, 0, 0 NAMEI: 1024, 0, 0, 28, 14308, 0, 0 S VFS Cache: 108, 0, 1125, 96, 3337, 0, 0 L VFS Cache: 328, 0, 0, 0, 0, 0, 0 DIRHASH: 1024, 0, 147, 13, 147, 0, 0 NCLNODE: 560, 0, 3, 18, 3, 0, 0 Mountpoints: 768, 0, 16, 9, 16, 0, 0 pipe: 728, 0, 34, 31, 989, 0, 0 ksiginfo: 112, 0, 59, 997, 65, 0, 0 itimer: 344, 0, 1, 21, 1, 0, 0 pfsrctrpl: 152, 0, 0, 0, 0, 0, 0 pfrulepl: 936, 0, 0, 0, 0, 0, 0 pfstatepl: 288, 10010, 0, 0, 0, 0, 0 pfstatekeypl: 288, 0, 0, 0, 0, 0, 0 pfstateitempl: 288, 0, 0, 0, 0, 0, 0 pfaltqpl: 240, 0, 0, 0, 0, 0, 0 pfpooladdrpl: 88, 0, 0, 0, 0, 0, 0 pfrktable: 1296, 0, 0, 0, 0, 0, 0 pfrkentry: 160, 0, 0, 0, 0, 0, 0 pffrent: 32, 5050, 0, 0, 0, 0, 0 pffrag: 80, 0, 0, 0, 0, 0, 0 pffrcache: 80, 10035, 0, 0, 0, 0, 0 pffrcent: 24, 50022, 0, 0, 0, 0, 0 pfstatescrub: 40, 0, 0, 0, 0, 0, 0 pfiaddrpl: 120, 0, 0, 0, 0, 0, 0 pfospfen: 112, 0, 0, 0, 0, 0, 0 pfosfp: 40, 0, 0, 0, 0, 0, 0 KNOTE: 128, 0, 62, 54, 100, 0, 0 socket: 680, 25602, 73, 11, 426, 0, 0 unpcb: 240, 25600, 47, 17, 141, 0, 0 ipq: 56, 819, 0, 0, 0, 0, 0 udp_inpcb: 392, 25600, 15, 15, 195, 0, 0 udpcb: 16, 25704, 15, 321, 195, 0, 0 tcp_inpcb: 392, 25600, 9, 21, 19, 0, 0 tcpcb: 976, 25600, 9, 11, 19, 0, 0 tcptw: 72, 5150, 0, 0, 0, 0, 0 syncache: 152, 15375, 0, 0, 0, 0, 0 hostcache: 136, 15372, 0, 0, 0, 0, 0 tcpreass: 40, 1680, 0, 0, 0, 0, 0 sackhole: 32, 0, 0, 0, 0, 0, 0 ripcb: 392, 25600, 0, 30, 23, 0, 0 rtentry: 200, 0, 1, 37, 1, 0, 0 IPFW dynamic rule: 120, 0, 0, 0, 0, 0, 0 selfd: 56, 0, 55, 134, 1846, 0, 0 SWAPMETA: 288, 116519, 0, 0, 0, 0, 0 FFS inode: 168, 0, 1033, 67, 1084, 0, 0 FFS1 dinode: 128, 0, 0, 0, 0, 0, 0 FFS2 dinode: 256, 0, 1033, 92, 1084, 0, 0 taskq_zone: 48, 0, 0, 216, 7, 0, 0 zio_cache: 880, 0, 1, 591, 4480, 0, 0 zio_link_cache: 48, 0, 0, 1080, 5018, 0, 0 zio_buf_512: 512, 0, 0, 0, 0, 0, 0 zio_data_buf_512: 512, 0, 0, 0, 0, 0, 0 zio_buf_1024: 1024, 0, 0, 0, 0, 0, 0 zio_data_buf_1024: 1024, 0, 0, 0, 0, 0, 0 zio_buf_1536: 1536, 0, 0, 0, 0, 0, 0 zio_data_buf_1536: 1536, 0, 0, 0, 0, 0, 0 zio_buf_2048: 2048, 0, 0, 0, 0, 0, 0 zio_data_buf_2048: 2048, 0, 0, 0, 0, 0, 0 zio_buf_2560: 2560, 0, 0, 0, 0, 0, 0 zio_data_buf_2560: 2560, 0, 0, 0, 0, 0, 0 zio_buf_3072: 3072, 0, 0, 0, 0, 0, 0 zio_data_buf_3072: 3072, 0, 0, 0, 0, 0, 0 zio_buf_3584: 3584, 0, 0, 0, 0, 0, 0 zio_data_buf_3584: 3584, 0, 0, 0, 0, 0, 0 zio_buf_4096: 4096, 0, 0, 0, 0, 0, 0 zio_data_buf_4096: 4096, 0, 0, 0, 0, 0, 0 zio_buf_5120: 5120, 0, 0, 0, 0, 0, 0 zio_data_buf_5120: 5120, 0, 0, 0, 0, 0, 0 zio_buf_6144: 6144, 0, 0, 0, 0, 0, 0 zio_data_buf_6144: 6144, 0, 0, 0, 0, 0, 0 zio_buf_7168: 7168, 0, 0, 0, 0, 0, 0 zio_data_buf_7168: 7168, 0, 0, 0, 0, 0, 0 zio_buf_8192: 8192, 0, 0, 0, 0, 0, 0 zio_data_buf_8192: 8192, 0, 0, 0, 0, 0, 0 zio_buf_10240: 10240, 0, 0, 0, 0, 0, 0 zio_data_buf_10240: 10240, 0, 0, 0, 0, 0, 0 zio_buf_12288: 12288, 0, 0, 0, 0, 0, 0 zio_data_buf_12288: 12288, 0, 0, 0, 0, 0, 0 zio_buf_14336: 14336, 0, 0, 0, 0, 0, 0 zio_data_buf_14336: 14336, 0, 0, 0, 0, 0, 0 zio_buf_16384: 16384, 0, 0, 0, 0, 0, 0 zio_data_buf_16384: 16384, 0, 0, 0, 0, 0, 0 zio_buf_20480: 20480, 0, 0, 0, 0, 0, 0 zio_data_buf_20480: 20480, 0, 0, 0, 0, 0, 0 zio_buf_24576: 24576, 0, 0, 0, 0, 0, 0 zio_data_buf_24576: 24576, 0, 0, 0, 0, 0, 0 zio_buf_28672: 28672, 0, 0, 0, 0, 0, 0 zio_data_buf_28672: 28672, 0, 0, 0, 0, 0, 0 zio_buf_32768: 32768, 0, 0, 0, 0, 0, 0 zio_data_buf_32768: 32768, 0, 0, 0, 0, 0, 0 zio_buf_36864: 36864, 0, 0, 0, 0, 0, 0 zio_data_buf_36864: 36864, 0, 0, 0, 0, 0, 0 zio_buf_40960: 40960, 0, 0, 0, 0, 0, 0 zio_data_buf_40960: 40960, 0, 0, 0, 0, 0, 0 zio_buf_45056: 45056, 0, 0, 0, 0, 0, 0 zio_data_buf_45056: 45056, 0, 0, 0, 0, 0, 0 zio_buf_49152: 49152, 0, 0, 0, 0, 0, 0 zio_data_buf_49152: 49152, 0, 0, 0, 0, 0, 0 zio_buf_53248: 53248, 0, 0, 0, 0, 0, 0 zio_data_buf_53248: 53248, 0, 0, 0, 0, 0, 0 zio_buf_57344: 57344, 0, 0, 0, 0, 0, 0 zio_data_buf_57344: 57344, 0, 0, 0, 0, 0, 0 zio_buf_61440: 61440, 0, 0, 0, 0, 0, 0 zio_data_buf_61440: 61440, 0, 0, 0, 0, 0, 0 zio_buf_65536: 65536, 0, 0, 0, 0, 0, 0 zio_data_buf_65536: 65536, 0, 0, 0, 0, 0, 0 zio_buf_69632: 69632, 0, 0, 0, 0, 0, 0 zio_data_buf_69632: 69632, 0, 0, 0, 0, 0, 0 zio_buf_73728: 73728, 0, 0, 0, 0, 0, 0 zio_data_buf_73728: 73728, 0, 0, 0, 0, 0, 0 zio_buf_77824: 77824, 0, 0, 0, 0, 0, 0 zio_data_buf_77824: 77824, 0, 0, 0, 0, 0, 0 zio_buf_81920: 81920, 0, 0, 0, 0, 0, 0 zio_data_buf_81920: 81920, 0, 0, 0, 0, 0, 0 zio_buf_86016: 86016, 0, 0, 0, 0, 0, 0 zio_data_buf_86016: 86016, 0, 0, 0, 0, 0, 0 zio_buf_90112: 90112, 0, 0, 0, 0, 0, 0 zio_data_buf_90112: 90112, 0, 0, 0, 0, 0, 0 zio_buf_94208: 94208, 0, 0, 0, 0, 0, 0 zio_data_buf_94208: 94208, 0, 0, 0, 0, 0, 0 zio_buf_98304: 98304, 0, 0, 0, 0, 0, 0 zio_data_buf_98304: 98304, 0, 0, 0, 0, 0, 0 zio_buf_102400: 102400, 0, 0, 0, 0, 0, 0 zio_data_buf_102400: 102400, 0, 0, 0, 0, 0, 0 zio_buf_106496: 106496, 0, 0, 0, 0, 0, 0 zio_data_buf_106496: 106496, 0, 0, 0, 0, 0, 0 zio_buf_110592: 110592, 0, 0, 0, 0, 0, 0 zio_data_buf_110592: 110592, 0, 0, 0, 0, 0, 0 zio_buf_114688: 114688, 0, 0, 0, 0, 0, 0 zio_data_buf_114688: 114688, 0, 0, 0, 0, 0, 0 zio_buf_118784: 118784, 0, 0, 0, 0, 0, 0 zio_data_buf_118784: 118784, 0, 0, 0, 0, 0, 0 zio_buf_122880: 122880, 0, 0, 0, 0, 0, 0 zio_data_buf_122880: 122880, 0, 0, 0, 0, 0, 0 zio_buf_126976: 126976, 0, 0, 0, 0, 0, 0 zio_data_buf_126976: 126976, 0, 0, 0, 0, 0, 0 zio_buf_131072: 131072, 0, 0, 0, 0, 0, 0 zio_data_buf_131072: 131072, 0, 0, 0, 0, 0, 0 sa_cache: 80, 0, 40, 95, 42, 0, 0 dnode_t: 856, 0, 199, 13, 317, 0, 0 dmu_buf_impl_t: 224, 0, 508, 70, 546, 0, 0 arc_buf_hdr_t: 216, 0, 485, 37, 512, 0, 0 arc_buf_t: 104, 0, 479, 61, 534, 0, 0 zil_lwb_cache: 192, 0, 0, 0, 0, 0, 0 zfs_znode_cache: 400, 0, 40, 23, 42, 0, 0