This panic comes from the system where NFS is served from NULLFS. It results from kernel compiled with DEBUG_VFS_LOCKS and vrele -> vput substitution inside null_nodeget that is suggested by kib@. ASSERT_VOP_LOCKED inside insmntque1 was removed to inhibit panic from insmntque1() and catch this one. See http://bugs.freebsd.org/164261 for the full description. Wed Jan 18 13:52:30 MSK 2012 FreeBSD XXX 10.0-CURRENT FreeBSD 10.0-CURRENT #13 r230080M: Wed Jan 18 13:46:29 MSK 2012 root@XXX:/sys/amd64/compile/SHADOW-CURRENT amd64 panic: from debugger GNU gdb 6.1.1 [FreeBSD] Copyright 2004 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "amd64-marcel-freebsd"... Unread portion of the kernel message buffer: nfsrvd_dorpc+0x3c0 nfssvc_program() at nfssvc_progrnafms(s)v ca_tp rogram+0x447 nfssvc_program+0x447 svc_run_internal() at svc_run_internal() at svc_run_internal+0x6f9 svc_run_internal+0x6f9 svc_thread_start() at svc_run() at svc_thread_start+0xb svc_run+0x8b fork_exit() at nfsrvd_nfsd() at fork_exit+0x114 nfsrvd_nfsd+0x177 fork_trampoline() at fork_trnafmspsovc_lnifnsed+(0)x e at- -- trap 0xc, rip = 0x8008ab6ac, rsp = 0x7fffffffd698, rbp = 0x5 --- vput: 0xfffffe01350513c0 is not locked but should be KDB: enter: lock violation Uptime: 1m25s Dumping 300 out of 4083 MB: Reading symbols from /boot/kernel/zfs.ko...Reading symbols from /boot/kernel/zfs.ko.symbols...done. done. Loaded symbols for /boot/kernel/zfs.ko Reading symbols from /boot/kernel/opensolaris.ko...Reading symbols from /boot/kernel/opensolaris.ko.symbols...done. done. Loaded symbols for /boot/kernel/opensolaris.ko Reading symbols from /boot/kernel/wpifw.ko...Reading symbols from /boot/kernel/wpifw.ko.symbols...done. done. Loaded symbols for /boot/kernel/wpifw.ko #0 doadump (textdump=1) at pcpu.h:224 224 pcpu.h: No such file or directory. in pcpu.h (kgdb) #0 doadump (textdump=1) at pcpu.h:224 #1 0xffffffff80532218 in kern_reboot (howto=260) at ../../../kern/kern_shutdown.c:455 #2 0xffffffff8053271d in panic (fmt=0x0) at ../../../kern/kern_shutdown.c:643 #3 0xffffffff802ece90 in db_panic (addr=Variable "addr" is not available. ) at ../../../ddb/db_command.c:482 #4 0xffffffff802ed281 in db_command (last_cmdp=0xffffffff80b3d900, cmd_table=Variable "cmd_table" is not available. ) at ../../../ddb/db_command.c:449 #5 0xffffffff802ed4c9 in db_command_loop () at ../../../ddb/db_command.c:502 #6 0xffffffff802ef4a7 in db_trap (type=Variable "type" is not available. ) at ../../../ddb/db_main.c:229 #7 0xffffffff80565d51 in kdb_trap (type=3, code=0, tf=0xffffff811df26170) at ../../../kern/subr_kdb.c:629 #8 0xffffffff80742172 in trap (frame=0xffffff811df26170) at ../../../amd64/amd64/trap.c:591 #9 0xffffffff8072d34f in calltrap () at ../../../amd64/amd64/exception.S:228 #10 0xffffffff80565aff in kdb_enter (why=0xffffffff8084bfaa "vfslock", msg=0x780
) at cpufunc.h:63 #11 0xffffffff805c4ff1 in assert_vop_locked (vp=0xfffffe01350513c0, str=0xffffffff8084c6b4 "vput") at ../../../kern/vfs_subr.c:3834 #12 0xffffffff805c5eab in vputx (vp=0xfffffe01350513c0, func=2) at ../../../kern/vfs_subr.c:2243 #13 0xffffffff804b9f87 in null_nodeget (mp=0xfffffe0006148c00, lowervp=Variable "lowervp" is not available. ) at ../../../fs/nullfs/null_subr.c:259 #14 0xffffffff804b138e in nfsrvd_readdirplus (nd=0xffffff811df26990, isdgram=1, vp=0xfffffe0006fe3d20, p=0xfffffe0006aee000, exp=0xffffff811df26830) at ../../../fs/nfsserver/nfs_nfsdport.c:2044 #15 0xffffffff804913b0 in nfsrvd_dorpc (nd=0xffffff811df26990, isdgram=1, p=0xfffffe0006aee000) at ../../../fs/nfsserver/nfs_nfsdsocket.c:440 #16 0xffffffff8049f9a0 in nfssvc_program (rqst=0xfffffe0006df2000, xprt=0xfffffe0006c86c00) at ../../../fs/nfsserver/nfs_nfsdkrpc.c:328 #17 0xffffffff806ac7e7 in svc_run_internal (pool=0xfffffe0001df6500, ismaster=0) at ../../../rpc/svc.c:895 #18 0xffffffff806acbb5 in svc_thread_start (arg=Variable "arg" is not available. ) at ../../../rpc/svc.c:1200 #19 0xffffffff80507d8f in fork_exit ( callout=0xffffffff806acbaa , arg=0xfffffe0001df6500, frame=0xffffff811df26c50) at ../../../kern/kern_fork.c:995 #20 0xffffffff8072d80e in fork_trampoline () at ../../../amd64/amd64/exception.S:602 #21 0x0000000000000080 in ?? () #22 0x00007fffffffd6d0 in ?? () #23 0x000000000000002e in ?? () #24 0x0000000000000000 in ?? () #25 0xfffffffffffffef4 in ?? () #26 0x0000000000000002 in ?? () #27 0x000000000000009b in ?? () #28 0x00007fffffffd6d0 in ?? () #29 0x0000000000000005 in ?? () #30 0x0000000000000003 in ?? () #31 0x00000000ffffffff in ?? () #32 0x00007fffffffd6f0 in ?? () #33 0x0000000000000000 in ?? () #34 0x0000000000000001 in ?? () #35 0x0000000000000001 in ?? () #36 0x001b00130000000c in ?? () #37 0x00007fffffffefe8 in ?? () #38 0x003b003b00000001 in ?? () #39 0x0000000000000002 in ?? () #40 0x00000008008ab6ac in ?? () #41 0x0000000000000043 in ?? () #42 0x0000000000000206 in ?? () #43 0x00007fffffffd698 in ?? () #44 0x000000000000003b in ?? () #45 0xffffffff80b7aa00 in affinity () #46 0xffffffff80b7aa00 in affinity () #47 0x0000000000000000 in ?? () #48 0xfffffe0006aee000 in ?? () #49 0xffffff811df25ce0 in ?? () #50 0xffffff811df25c88 in ?? () #51 0xffffffff80b7aa00 in affinity () #52 0xffffffff80559793 in sched_switch (td=0xffffffff806acbaa, newtd=Variable "newtd" is not available. ) at ../../../kern/sched_ule.c:1854 Previous frame inner to this frame (corrupt stack?) (kgdb) ------------------------------------------------------------------------ ps -axl UID PID PPID CPU PRI NI VSZ RSS MWCHAN STAT TT TIME COMMAND 0 0 0 0 -8 0 0 0 - DLs - 0:00.14 [kernel] 0 1 0 0 21 0 6280 0 wait DLs - 0:00.00 [init] 0 2 0 0 -16 0 0 0 crypto_w DL - 0:00.00 [crypto] 0 3 0 0 -16 0 0 0 crypto_r DL - 0:00.00 [crypto returns] 0 4 0 0 -16 0 0 0 - DL - 0:00.00 [fw0_probe] 0 5 0 0 -16 0 0 0 pftm DL - 0:00.00 [pfpurge] 0 6 0 0 -16 0 0 0 ccb_scan DL - 0:00.00 [xpt_thrd] 0 7 0 0 -16 0 0 0 psleep DL - 0:00.00 [pagedaemon] 0 8 0 0 -16 0 0 0 psleep DL - 0:00.00 [vmdaemon] 0 9 0 0 155 0 0 0 pgzero DL - 0:00.00 [pagezero] 0 10 0 0 -16 0 0 0 audit_wo DL - 0:00.00 [audit] 0 11 0 0 155 0 0 0 - RL - 0:14.64 [idle] 0 12 0 0 -84 0 0 0 - WL - 0:00.05 [intr] 0 13 0 0 -16 0 0 0 sleep DL - 0:00.00 [ng_queue] 0 14 0 0 -8 0 0 0 - DL - 0:00.04 [geom] 0 15 0 0 -16 0 0 0 - DL - 0:00.00 [yarrow] 0 16 0 0 -68 0 0 0 - DL - 0:00.01 [usb] 0 17 0 0 -16 0 0 0 tzpoll DL - 0:00.00 [acpi_thermal] 0 18 0 0 -16 0 0 0 cooling DL - 0:00.00 [acpi_cooling0] 0 19 0 0 -16 0 0 0 psleep DL - 0:00.00 [bufdaemon] 0 20 0 0 -16 0 0 0 vlruwt DL - 0:00.00 [vnlru] 0 21 0 0 16 0 0 0 syncer DL - 0:00.00 [syncer] 0 22 0 0 -16 0 0 0 sdflush DL - 0:00.00 [softdepflush] 0 49 0 0 -8 0 0 0 tx->tx_s DL - 0:00.00 [zfskern] 0 404 1 0 20 0 18376 0 select Ds - 0:00.00 [wpa_supplicant] 0 796 1 0 52 0 10372 0 select Ds - 0:00.00 [devd] 0 954 1 0 52 0 6280 0 select Ds - 0:00.00 [dhclient] 65 993 1 0 52 0 6280 0 select Ds - 0:00.00 [dhclient] 0 1169 1 0 20 0 12208 0 select Ds - 0:00.01 [syslogd] 53 1266 1 0 52 0 47224 0 kqread Ds - 0:00.07 [named] 0 1279 1 0 20 0 14288 0 select Ds - 0:00.01 [rpcbind] 0 1350 1 0 21 0 12204 0 select Ds - 0:00.00 [mountd] 0 1360 1 0 52 0 10072 0 accept Ds - 0:00.02 [nfsd] 0 1361 1360 0 52 0 10072 0 - R - 0:00.00 [nfsd] 0 1367 1 0 20 0 274312 0 select Ds - 0:00.00 [rpc.statd] 0 1373 1 0 52 0 14288 0 rpcsvc Ds - 0:00.00 [rpc.lockd] 0 1443 1 0 20 0 12204 0 select Ds - 0:00.00 [powerd] 0 1581 1 0 20 0 18532 0 kqread Ds - 0:00.01 [master] 125 1590 1581 0 20 0 18536 0 kqread D - 0:00.01 [pickup] 125 1591 1581 0 52 0 18532 0 kqread D - 0:00.01 [qmgr] 5999 1624 1 0 52 0 10060 0 select Ds - 0:00.01 [cvsupd] 0 1671 1 0 20 0 14288 0 nanslp Ds - 0:00.00 [cron] 0 1704 1 0 52 0 16368 0 select Ds - 0:00.00 [inetd] 0 1739 1 0 21 0 41320 0 wait Ds - 0:00.00 [login] 0 1740 1 0 52 0 12208 0 ttyin Ds+ - 0:00.00 [getty] 0 1741 1 0 52 0 12208 0 ttyin Ds+ - 0:00.00 [getty] 0 1742 1 0 52 0 12208 0 ttyin Ds+ - 0:00.00 [getty] 0 1743 1739 0 28 0 6276 0 pause D - 0:00.03 [csh] 0 1759 1743 0 40 0 41316 0 wait D - 0:00.00 [su] 1001 1760 1759 0 39 0 6276 0 wait D - 0:00.20 [bash] 1001 1786 1 0 52 0 36188 0 select Ds - 0:00.00 [ssh-agent] 1001 1801 1 0 25 0 24000 0 select Ds - 0:00.00 [gpg-agent] 1001 1807 1760 0 52 0 6276 0 wait D+ - 0:00.00 [sh] 1001 1809 1807 0 52 0 10076 0 wait D+ - 0:00.00 [find] 1001 1810 1807 0 52 0 10076 0 wait D+ - 0:00.00 [find] 1001 1811 1807 0 52 0 10076 0 wait D+ - 0:00.00 [find] 1001 1812 1810 0 52 0 6276 0 newnfsre D+ - 0:00.00 [ls] 1001 1813 1811 0 52 0 6276 0 newnfsre D+ - 0:00.00 [ls] 1001 1814 1809 0 52 0 6276 0 newnfsre D+ - 0:00.00 [ls] ------------------------------------------------------------------------ vmstat -s 94892 cpu context switches 251272 device interrupts 10133 software interrupts 176955 traps 240741 system calls 23 kernel threads created 1770 fork() calls 21 vfork() calls 0 rfork() calls 0 swap pager pageins 0 swap pager pages paged in 0 swap pager pageouts 0 swap pager pages paged out 767 vnode pager pageins 6816 vnode pager pages paged in 1 vnode pager pageouts 1 vnode pager pages paged out 0 page daemon wakeups 0 pages examined by the page daemon 197 pages reactivated 49176 copy-on-write faults 124 copy-on-write optimized faults 96395 zero fill pages zeroed 0 zero fill pages prezeroed 3 intransit blocking page faults 190581 total VM faults taken 0 pages affected by kernel thread creation 532452 pages affected by fork() 8874 pages affected by vfork() 0 pages affected by rfork() 0 pages cached 175287 pages freed 0 pages freed by daemon 0 pages freed by exiting processes 7622 pages active 6985 pages inactive 49 pages in VM cache 35013 pages wired down 956957 pages free 4096 bytes per page 37431 total name lookups cache hits (84% pos + 6% neg) system 0% per-directory deletions 0%, falsehits 0%, toolong 0% ------------------------------------------------------------------------ vmstat -m Type InUse MemUse HighUse Requests Size(s) kbdmux 6 18K - 6 16,512,1024,2048 LED 18 2K - 18 16,128 CAM queue 22 1K - 181 16,32,64,128,256 GEOM 159 41K - 1764 16,32,64,128,256,512,1024,2048 CAM SIM 6 2K - 6 256 acpisem 41 6K - 41 128 scsi_cd 0 0K - 6 16 CAM periph 6 2K - 27 16,32,64,128,256 CAM XPT 80 93K - 248 32,64,128,256,1024,2048 ata_generic 1 1K - 1 1024 ata_pci 1 1K - 1 64 isadev 7 1K - 7 128 acd_driver 1 2K - 1 2048 ata_cam 1 1K - 22 64,256,1024 entropy 1024 64K - 1024 64 cdev 9 3K - 9 256 CAM dev queue 6 1K - 6 128 sigio 1 1K - 1 64 filedesc 77 73K - 1924 16,32,64,128,512,1024,2048,4096 kenv 78 11K - 84 16,32,64,128 kqueue 8 15K - 33 256,2048 proc-args 35 3K - 788 16,32,64,128,256 hhook 2 1K - 2 128 hdac 10 32K - 10 64,128,256,512,2048,4096 ithread 82 13K - 82 32,128,256 KTRACE 100 13K - 100 128 linker 168 807K - 194 16,32,64,128,256,512,1024,2048,4096 lockf 32 4K - 72 64,128 loginclass 2 1K - 4 64 drm_ctxbitmap 1 4K - 1 4096 temp 58 52K - 8149 16,32,64,128,256,512,1024,2048,4096 devbuf 22698 47656K - 23447 16,32,64,128,256,512,1024,2048,4096 module 278 35K - 278 128 mtx_pool 2 16K - 2 osd 3 1K - 6 16,64,128 feeder 14 2K - 17 32,128 subproc 147 280K - 1903 512,4096 proc 2 16K - 2 session 23 3K - 25 128 pgrp 27 4K - 34 128 cred 176 28K - 7752 64,256 uidinfo 7 3K - 10 128,2048 plimit 15 4K - 134 256 sysctltmp 0 0K - 342 16,32,64,128 sysctloid 4747 234K - 4858 16,32,64,128 sysctl 0 0K - 1227 16,32,64 tidhash 1 16K - 1 callout 1 512K - 1 umtx 648 81K - 648 128 p1003.1b 1 1K - 1 16 SWAP 2 821K - 2 64 drm_maps 1 1K - 1 128 bus-sc 83 389K - 1410 16,32,64,128,256,512,1024,2048,4096 bus 841 75K - 3744 16,32,64,128,256,1024 devstat 20 41K - 20 32,4096 eventhandler 87 7K - 87 64,128 mixer 2 8K - 2 4096 kobj 178 712K - 205 4096 Per-cpu 1 1K - 1 32 drm_driver 4 3K - 8 16,32,64,128,512,2048 drm_sarea 1 1K - 1 16 rman 253 29K - 601 16,32,128 sbuf 0 0K - 1607 16,32,64,128,256,512,1024,2048,4096 fw_xfer 0 0K - 1 256 stack 0 0K - 2 256 taskqueue 57 6K - 87 16,32,64,128,1024 Unitno 19 2K - 609 32,64 iov 0 0K - 542 16,64,128,256,512 select 26 4K - 26 128 ioctlops 0 0K - 1601 16,32,64,128,256,512,1024,2048,4096 msg 4 30K - 4 2048,4096 sem 4 106K - 4 2048,4096 shm 1 20K - 1 tty 17 17K - 19 1024,2048 mbuf_tag 0 0K - 22 32,64 shmfd 1 8K - 1 firewire 11 35K - 14 64,128,512,1024,2048,4096 pcb 29 13K - 67 16,32,64,1024,2048,4096 soname 37 4K - 830 16,32,64,128 vfscache 1 2048K - 1 export_host 1 1K - 5 256 vfs_hash 1 1024K - 1 vnodes 4 1K - 4 64,256 vnodemarker 0 0K - 145 512 mount 284 13K - 2703 16,32,64,128,256,512 iconv 2 1K - 2 128 BPF 22 1035K - 23 16,128,512,4096 ether_multi 9 1K - 18 16,64 ifaddr 184 23K - 185 32,256,512,2048,4096 ifnet 14 27K - 14 128,2048 USBdev 37 10K - 37 64,128,1024 arpcom 3 1K - 3 16 lltable 14 7K - 14 256,512 USB 66 107K - 70 16,32,128,256,512,2048 acpidev 44 3K - 44 64 sbp 96 14K - 96 32,256 ddb_capture 1 48K - 1 acpiintr 1 1K - 1 64 acpica 4761 469K - 51777 16,32,64,128,256,512,1024,2048 routetbl 141 67K - 232 32,64,128,256,512 80211vap 1 4K - 1 4096 80211crypto 2 1K - 2 128,512 80211com 1 8K - 1 80211nodeie 3 1K - 4 32,128 80211node 1 12K - 3 80211ratectl 2 1K - 4 16,64 80211scan 3 7K - 4 512,2048,4096 netgraph_hci 1 1K - 1 256 netgraph_l2cap 1 1K - 1 128 netgraph_btsocks_hci_raw 1 8K - 23 128 netgraph_btsocks_l2cap 1 1K - 1 32 netgraph_btsocks_l2cap_raw 1 1K - 1 32 pci_link 16 2K - 16 16,32,128 acpi_perf 2 1K - 2 128 acpitask 1 2K - 1 2048 netgraph_node 7 2K - 16 256 netgraph_hook 10 2K - 10 128 netgraph_msg 0 0K - 40 64,128,256,1024 netgraph_sock 0 0K - 9 128 netgraph_path 0 0K - 11 16 igmp 13 4K - 13 256 DEVFS1 127 64K - 150 512 DEVFS3 292 73K - 304 256 DEVFS2 127 2K - 127 16 in_multi 2 1K - 3 256 IpFw/IpAcct 24 3K - 46 16,32,64,128,256 hostcache 1 28K - 1 syncache 1 96K - 1 NLM 0 0K - 1 16 crypto 1 1K - 1 512 rpc 67 19K - 183 16,32,64,128,256,512,1024,2048 audit_evclass 179 6K - 218 32 savedino 0 0K - 81 256 freework 1 1K - 22 16,128 newdirblk 0 0K - 6 64 dirrem 0 0K - 54 128 mkdir 0 0K - 12 128 diradd 0 0K - 63 128 freefile 0 0K - 51 64 freeblks 0 0K - 21 128 newblk 1 128K - 30 256 bmsafemap 1 8K - 17 256 inodedep 1 1024K - 121 512 pagedep 1 128K - 21 256 ufs_dirhash 81 17K - 81 16,32,64,128,256,512 ufs_mount 21 88K - 21 512,4096 vm_pgdata 2 129K - 2 128 UMAHash 1 1K - 1 512 DEVFS_RULE 60 28K - 116 64,512 DEVFS 40 1K - 41 16,128 DEVFSP 2 1K - 2 64 memdesc 1 4K - 1 4096 atkbddev 2 1K - 2 64 NFS fh 3 1K - 27 32 linux 15 1K - 15 64 NFSD srvcache 3 1K - 18 128 apmdev 1 1K - 1 128 newnfsmnt 3 3K - 3 1024 ntfs_nthash 1 1024K - 1 io_apic 1 2K - 1 2048 msi 4 1K - 4 128 nexusdev 4 1K - 4 16 nullfs_node 5 1K - 5 32 nullfs_hash 1 1K - 1 128 nullfs_mount 1 1K - 1 16 pfs_nodes 73 19K - 73 256 solaris 47156 17122K - 69925 16,32,64,128,256,512,1024,2048,4096 kstat_data 4 1K - 4 64 ------------------------------------------------------------------------ vmstat -z ITEM SIZE LIMIT USED FREE REQ FAIL SLEEP UMA Kegs: 208, 0, 194, 10, 194, 0, 0 UMA Zones: 640, 0, 194, 4, 194, 0, 0 UMA Slabs: 568, 0, 1588, 1, 2429, 0, 0 UMA RCntSlabs: 568, 0, 214, 3, 214, 0, 0 UMA Hash: 256, 0, 80, 10, 81, 0, 0 16 Bucket: 152, 0, 84, 16, 84, 0, 0 32 Bucket: 280, 0, 70, 0, 70, 0, 0 64 Bucket: 536, 0, 60, 3, 60, 57, 0 128 Bucket: 1048, 0, 51, 0, 51, 582, 0 VM OBJECT: 232, 0, 1483, 53, 19320, 0, 0 MAP: 232, 0, 7, 25, 7, 0, 0 KMAP ENTRY: 120, 150505, 51, 228, 14035, 0, 0 MAP ENTRY: 120, 0, 715, 122, 22672, 0, 0 fakepg: 120, 0, 0, 0, 0, 0, 0 mt_zone: 4112, 0, 311, 28, 311, 0, 0 16: 16, 0, 2838, 186, 30080, 0, 0 32: 32, 0, 2519, 410, 9566, 0, 0 64: 64, 0, 58381, 251, 98938, 0, 0 128: 128, 0, 8732, 142, 13202, 0, 0 256: 256, 0, 895, 95, 6935, 0, 0 512: 512, 0, 805, 84, 9316, 0, 0 1024: 1024, 0, 79, 565, 4194, 0, 0 2048: 2048, 0, 259, 35, 1195, 0, 0 4096: 4096, 0, 496, 31, 8561, 0, 0 Files: 80, 0, 208, 62, 6756, 0, 0 TURNSTILE: 136, 0, 325, 35, 325, 0, 0 umtx pi: 96, 0, 0, 0, 0, 0, 0 PROC: 1160, 0, 58, 29, 1814, 0, 0 THREAD: 1112, 0, 298, 26, 436, 0, 0 SLEEPQUEUE: 80, 0, 325, 81, 325, 0, 0 VMSPACE: 392, 0, 36, 14, 1793, 0, 0 cpuset: 72, 0, 2, 98, 2, 0, 0 audit_record: 960, 0, 0, 0, 0, 0, 0 mbuf_packet: 256, 0, 0, 261, 71, 0, 0 mbuf: 256, 0, 77, 312, 1731, 0, 0 mbuf_cluster: 2048, 25600, 259, 15, 290, 0, 0 mbuf_jumbo_page: 4096, 12800, 64, 13, 917, 0, 0 mbuf_jumbo_9k: 9216, 19200, 0, 0, 0, 0, 0 mbuf_jumbo_16k: 16384, 12800, 0, 0, 0, 0, 0 mbuf_ext_refcnt: 4, 0, 0, 0, 0, 0, 0 NetGraph items: 72, 4118, 0, 87, 60, 0, 0 NetGraph data items: 72, 522, 0, 87, 55, 0, 0 g_bio: 232, 0, 0, 224, 13404, 0, 0 ttyinq: 160, 0, 720, 72, 795, 0, 0 ttyoutq: 256, 0, 372, 33, 412, 0, 0 ata_request: 320, 0, 0, 36, 53, 0, 0 ata_composite: 336, 0, 0, 0, 0, 0, 0 cryptop: 88, 0, 0, 0, 0, 0, 0 cryptodesc: 72, 0, 0, 0, 0, 0, 0 VNODE: 480, 0, 1135, 17, 1235, 0, 0 VNODEPOLL: 112, 0, 0, 0, 0, 0, 0 NAMEI: 1024, 0, 0, 36, 14579, 0, 0 S VFS Cache: 108, 0, 1134, 87, 3166, 0, 0 L VFS Cache: 328, 0, 0, 0, 0, 0, 0 DIRHASH: 1024, 0, 147, 13, 147, 0, 0 NCLNODE: 560, 0, 3, 18, 3, 0, 0 Mountpoints: 768, 0, 16, 9, 16, 0, 0 pipe: 728, 0, 34, 21, 1020, 0, 0 ksiginfo: 112, 0, 72, 984, 79, 0, 0 itimer: 344, 0, 0, 0, 0, 0, 0 pfsrctrpl: 152, 0, 0, 0, 0, 0, 0 pfrulepl: 936, 0, 0, 0, 0, 0, 0 pfstatepl: 288, 10010, 0, 0, 0, 0, 0 pfstatekeypl: 288, 0, 0, 0, 0, 0, 0 pfstateitempl: 288, 0, 0, 0, 0, 0, 0 pfaltqpl: 240, 0, 0, 0, 0, 0, 0 pfpooladdrpl: 88, 0, 0, 0, 0, 0, 0 pfrktable: 1296, 0, 0, 0, 0, 0, 0 pfrkentry: 160, 0, 0, 0, 0, 0, 0 pffrent: 32, 5050, 0, 0, 0, 0, 0 pffrag: 80, 0, 0, 0, 0, 0, 0 pffrcache: 80, 10035, 0, 0, 0, 0, 0 pffrcent: 24, 50022, 0, 0, 0, 0, 0 pfstatescrub: 40, 0, 0, 0, 0, 0, 0 pfiaddrpl: 120, 0, 0, 0, 0, 0, 0 pfospfen: 112, 0, 0, 0, 0, 0, 0 pfosfp: 40, 0, 0, 0, 0, 0, 0 KNOTE: 128, 0, 62, 54, 104, 0, 0 socket: 680, 25602, 71, 13, 430, 0, 0 unpcb: 240, 25600, 46, 18, 153, 0, 0 ipq: 56, 819, 0, 0, 0, 0, 0 udp_inpcb: 392, 25600, 13, 17, 186, 0, 0 udpcb: 16, 25704, 13, 323, 186, 0, 0 tcp_inpcb: 392, 25600, 9, 21, 17, 0, 0 tcpcb: 976, 25600, 9, 11, 17, 0, 0 tcptw: 72, 5150, 0, 0, 0, 0, 0 syncache: 152, 15375, 0, 0, 0, 0, 0 hostcache: 136, 15372, 0, 0, 0, 0, 0 tcpreass: 40, 1680, 0, 0, 0, 0, 0 sackhole: 32, 0, 0, 0, 0, 0, 0 ripcb: 392, 25600, 1, 29, 24, 0, 0 rtentry: 200, 0, 4, 53, 5, 0, 0 IPFW dynamic rule: 120, 0, 0, 0, 0, 0, 0 selfd: 56, 0, 57, 132, 2184, 0, 0 SWAPMETA: 288, 116519, 0, 0, 0, 0, 0 FFS inode: 168, 0, 1045, 33, 1099, 0, 0 FFS1 dinode: 128, 0, 0, 0, 0, 0, 0 FFS2 dinode: 256, 0, 1045, 50, 1099, 0, 0 taskq_zone: 48, 0, 0, 216, 7, 0, 0 zio_cache: 880, 0, 1, 511, 3795, 0, 0 zio_link_cache: 48, 0, 0, 1008, 4209, 0, 0 zio_buf_512: 512, 0, 0, 0, 0, 0, 0 zio_data_buf_512: 512, 0, 0, 0, 0, 0, 0 zio_buf_1024: 1024, 0, 0, 0, 0, 0, 0 zio_data_buf_1024: 1024, 0, 0, 0, 0, 0, 0 zio_buf_1536: 1536, 0, 0, 0, 0, 0, 0 zio_data_buf_1536: 1536, 0, 0, 0, 0, 0, 0 zio_buf_2048: 2048, 0, 0, 0, 0, 0, 0 zio_data_buf_2048: 2048, 0, 0, 0, 0, 0, 0 zio_buf_2560: 2560, 0, 0, 0, 0, 0, 0 zio_data_buf_2560: 2560, 0, 0, 0, 0, 0, 0 zio_buf_3072: 3072, 0, 0, 0, 0, 0, 0 zio_data_buf_3072: 3072, 0, 0, 0, 0, 0, 0 zio_buf_3584: 3584, 0, 0, 0, 0, 0, 0 zio_data_buf_3584: 3584, 0, 0, 0, 0, 0, 0 zio_buf_4096: 4096, 0, 0, 0, 0, 0, 0 zio_data_buf_4096: 4096, 0, 0, 0, 0, 0, 0 zio_buf_5120: 5120, 0, 0, 0, 0, 0, 0 zio_data_buf_5120: 5120, 0, 0, 0, 0, 0, 0 zio_buf_6144: 6144, 0, 0, 0, 0, 0, 0 zio_data_buf_6144: 6144, 0, 0, 0, 0, 0, 0 zio_buf_7168: 7168, 0, 0, 0, 0, 0, 0 zio_data_buf_7168: 7168, 0, 0, 0, 0, 0, 0 zio_buf_8192: 8192, 0, 0, 0, 0, 0, 0 zio_data_buf_8192: 8192, 0, 0, 0, 0, 0, 0 zio_buf_10240: 10240, 0, 0, 0, 0, 0, 0 zio_data_buf_10240: 10240, 0, 0, 0, 0, 0, 0 zio_buf_12288: 12288, 0, 0, 0, 0, 0, 0 zio_data_buf_12288: 12288, 0, 0, 0, 0, 0, 0 zio_buf_14336: 14336, 0, 0, 0, 0, 0, 0 zio_data_buf_14336: 14336, 0, 0, 0, 0, 0, 0 zio_buf_16384: 16384, 0, 0, 0, 0, 0, 0 zio_data_buf_16384: 16384, 0, 0, 0, 0, 0, 0 zio_buf_20480: 20480, 0, 0, 0, 0, 0, 0 zio_data_buf_20480: 20480, 0, 0, 0, 0, 0, 0 zio_buf_24576: 24576, 0, 0, 0, 0, 0, 0 zio_data_buf_24576: 24576, 0, 0, 0, 0, 0, 0 zio_buf_28672: 28672, 0, 0, 0, 0, 0, 0 zio_data_buf_28672: 28672, 0, 0, 0, 0, 0, 0 zio_buf_32768: 32768, 0, 0, 0, 0, 0, 0 zio_data_buf_32768: 32768, 0, 0, 0, 0, 0, 0 zio_buf_36864: 36864, 0, 0, 0, 0, 0, 0 zio_data_buf_36864: 36864, 0, 0, 0, 0, 0, 0 zio_buf_40960: 40960, 0, 0, 0, 0, 0, 0 zio_data_buf_40960: 40960, 0, 0, 0, 0, 0, 0 zio_buf_45056: 45056, 0, 0, 0, 0, 0, 0 zio_data_buf_45056: 45056, 0, 0, 0, 0, 0, 0 zio_buf_49152: 49152, 0, 0, 0, 0, 0, 0 zio_data_buf_49152: 49152, 0, 0, 0, 0, 0, 0 zio_buf_53248: 53248, 0, 0, 0, 0, 0, 0 zio_data_buf_53248: 53248, 0, 0, 0, 0, 0, 0 zio_buf_57344: 57344, 0, 0, 0, 0, 0, 0 zio_data_buf_57344: 57344, 0, 0, 0, 0, 0, 0 zio_buf_61440: 61440, 0, 0, 0, 0, 0, 0 zio_data_buf_61440: 61440, 0, 0, 0, 0, 0, 0 zio_buf_65536: 65536, 0, 0, 0, 0, 0, 0 zio_data_buf_65536: 65536, 0, 0, 0, 0, 0, 0 zio_buf_69632: 69632, 0, 0, 0, 0, 0, 0 zio_data_buf_69632: 69632, 0, 0, 0, 0, 0, 0 zio_buf_73728: 73728, 0, 0, 0, 0, 0, 0 zio_data_buf_73728: 73728, 0, 0, 0, 0, 0, 0 zio_buf_77824: 77824, 0, 0, 0, 0, 0, 0 zio_data_buf_77824: 77824, 0, 0, 0, 0, 0, 0 zio_buf_81920: 81920, 0, 0, 0, 0, 0, 0 zio_data_buf_81920: 81920, 0, 0, 0, 0, 0, 0 zio_buf_86016: 86016, 0, 0, 0, 0, 0, 0 zio_data_buf_86016: 86016, 0, 0, 0, 0, 0, 0 zio_buf_90112: 90112, 0, 0, 0, 0, 0, 0 zio_data_buf_90112: 90112, 0, 0, 0, 0, 0, 0 zio_buf_94208: 94208, 0, 0, 0, 0, 0, 0 zio_data_buf_94208: 94208, 0, 0, 0, 0, 0, 0 zio_buf_98304: 98304, 0, 0, 0, 0, 0, 0 zio_data_buf_98304: 98304, 0, 0, 0, 0, 0, 0 zio_buf_102400: 102400, 0, 0, 0, 0, 0, 0 zio_data_buf_102400: 102400, 0, 0, 0, 0, 0, 0 zio_buf_106496: 106496, 0, 0, 0, 0, 0, 0 zio_data_buf_106496: 106496, 0, 0, 0, 0, 0, 0 zio_buf_110592: 110592, 0, 0, 0, 0, 0, 0 zio_data_buf_110592: 110592, 0, 0, 0, 0, 0, 0 zio_buf_114688: 114688, 0, 0, 0, 0, 0, 0 zio_data_buf_114688: 114688, 0, 0, 0, 0, 0, 0 zio_buf_118784: 118784, 0, 0, 0, 0, 0, 0 zio_data_buf_118784: 118784, 0, 0, 0, 0, 0, 0 zio_buf_122880: 122880, 0, 0, 0, 0, 0, 0 zio_data_buf_122880: 122880, 0, 0, 0, 0, 0, 0 zio_buf_126976: 126976, 0, 0, 0, 0, 0, 0 zio_data_buf_126976: 126976, 0, 0, 0, 0, 0, 0 zio_buf_131072: 131072, 0, 0, 0, 0, 0, 0 zio_data_buf_131072: 131072, 0, 0, 0, 0, 0, 0 sa_cache: 80, 0, 37, 98, 37, 0, 0 dnode_t: 856, 0, 193, 15, 311, 0, 0 dmu_buf_impl_t: 224, 0, 487, 23, 520, 0, 0 arc_buf_hdr_t: 216, 0, 462, 24, 487, 0, 0 arc_buf_t: 104, 0, 459, 81, 506, 0, 0 zil_lwb_cache: 192, 0, 0, 0, 0, 0, 0 zfs_znode_cache: 400, 0, 37, 26, 37, 0, 0