Project

General

Profile

Regression #12069 » 1624352039167-ddb.txt

Jim Pingle, 06/22/2021 07:21 AM

 
db:0:kdb.enter.default> run lockinfo
db:1:lockinfo> show locks
No such command; use "help" to list available commands
db:1:lockinfo> show alllocks
No such command; use "help" to list available commands
db:1:lockinfo> show lockedvnods
Locked vnodes
db:0:kdb.enter.default> show pcpu
cpuid = 0
dynamic pcpu = 0xd5b580
curthread = 0xfffff8001786c000: pid 53869 tid 100554 "pfctl"
curpcb = 0xfffff8001786c5a0
fpcurthread = 0xfffff8001786c000: pid 53869 "pfctl"
idlethread = 0xfffff8000514b000: tid 100003 "idle: cpu0"
curpmap = 0xfffff80039ced138
tssp = 0xffffffff837173a0
commontssp = 0xffffffff837173a0
rsp0 = 0xfffffe001905ecc0
kcr3 = 0x80000000150d4198
ucr3 = 0x8000000027dcf998
scr3 = 0x38a2c9a6
gs32p = 0xffffffff8371dbb8
ldt = 0xffffffff8371dbf8
tss = 0xffffffff8371dbe8
tlb gen = 0
curvnet = 0xfffff80005069b80
db:0:kdb.enter.default> bt
Tracing pid 53869 tid 100554 td 0xfffff8001786c000
kdb_enter() at kdb_enter+0x37/frame 0xfffffe001905dec0
vpanic() at vpanic+0x197/frame 0xfffffe001905df10
panic() at panic+0x43/frame 0xfffffe001905df70
trap_fatal() at trap_fatal+0x391/frame 0xfffffe001905dfd0
trap_pfault() at trap_pfault+0x4f/frame 0xfffffe001905e020
trap() at trap+0x286/frame 0xfffffe001905e130
calltrap() at calltrap+0x8/frame 0xfffffe001905e130
--- trap 0xc, rip = 0xffffffff811dc654, rsp = 0xfffffe001905e200, rbp = 0xfffffe001905e250 ---
uma_zfree_arg() at uma_zfree_arg+0x24/frame 0xfffffe001905e250
free() at free+0x5a/frame 0xfffffe001905e280
nvpair_free() at nvpair_free+0x99/frame 0xfffffe001905e2a0
nvlist_destroy() at nvlist_destroy+0x60/frame 0xfffffe001905e2d0
pf_state_to_nvstate() at pf_state_to_nvstate+0x94/frame 0xfffffe001905e310
pfioctl() at pfioctl+0x1b27/frame 0xfffffe001905e7e0
devfs_ioctl() at devfs_ioctl+0xb0/frame 0xfffffe001905e830
VOP_IOCTL_APV() at VOP_IOCTL_APV+0x7b/frame 0xfffffe001905e860
vn_ioctl() at vn_ioctl+0x16c/frame 0xfffffe001905e970
devfs_ioctl_f() at devfs_ioctl_f+0x1e/frame 0xfffffe001905e990
kern_ioctl() at kern_ioctl+0x2b7/frame 0xfffffe001905e9f0
sys_ioctl() at sys_ioctl+0x101/frame 0xfffffe001905eac0
amd64_syscall() at amd64_syscall+0x387/frame 0xfffffe001905ebf0
fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe001905ebf0
--- syscall (54, FreeBSD ELF64, sys_ioctl), rip = 0x800490b3a, rsp = 0x7fffffffe5c8, rbp = 0x7fffffffe640 ---
db:0:kdb.enter.default> ps
pid ppid pgrp uid state wmesg wchan cmd
39540 41042 41024 0 DW pfault 0xffffffff81e47f00 sh
53869 84578 347 0 R CPU 0 pfctl
23823 23707 22906 0 SW nanslp 0xffffffff836a0df0 sleep
23707 1 22906 0 SW wait 0xfffff80017891530 sh
86680 347 347 0 SW accept 0xfffff8000f8afba0 php-fpm
81288 347 347 0 SW accept 0xfffff8000f8afba0 php-fpm
28803 347 347 0 SW accept 0xfffff8000f8afba0 php-fpm
73848 72977 72702 122 D pfault 0xffffffff81e47f00 zabbix_agentd
73822 72977 72702 122 D pfault 0xffffffff81e47f00 zabbix_agentd
73710 72977 72702 122 D pfault 0xffffffff81e47f00 zabbix_agentd
73658 72977 72702 122 D pfault 0xffffffff81e47f00 zabbix_agentd
73543 72977 72702 122 D pfault 0xffffffff81e47f00 zabbix_agentd
72977 1 72702 122 SW wait 0xfffff80017890a60 zabbix_agentd
47362 1 47362 0 ?Ws vnstatd
84578 1 347 0 SW wait 0xfffff80038463530 sh
42187 1 42187 59 SWs kqread 0xfffff80028ecbd00 unbound
96197 1 96197 0 Ds (threaded) dpinger
100548 S uwait 0xfffff80028702800 dpinger
100576 S sbwait 0xfffff80039ac81d4 dpinger
100577 S nanslp 0xffffffff836a0df0 dpinger
100578 D pfault 0xffffffff81e47f00 dpinger
100579 S accept 0xfffff80038a774d0 dpinger
96061 1 96061 0 Ds (threaded) dpinger
100502 S uwait 0xfffff8002fe06980 dpinger
100572 S sbwait 0xfffff80039ac7c0c dpinger
100573 S nanslp 0xffffffff836a0df0 dpinger
100574 D pfault 0xffffffff81e47f00 dpinger
100575 S accept 0xfffff80028d2a4d0 dpinger
95746 1 95746 0 Ds (threaded) dpinger
100465 S uwait 0xfffff8000590b200 dpinger
100568 S sbwait 0xfffff80039ac6c0c dpinger
100569 S nanslp 0xffffffff836a0df0 dpinger
100570 D pfault 0xffffffff81e47f00 dpinger
100571 S accept 0xfffff80039ac64d0 dpinger
74733 1 74733 0 Ds pfault 0xffffffff81e47f00 openvpn
24374 347 347 0 SW accept 0xfffff8000f8afba0 php-fpm
60970 1 60970 0 Ss+ ttyin 0xfffff8000542f0b0 getty
60807 1 60807 0 Ss+ ttyin 0xfffff800054364b0 getty
60521 1 60521 0 Ss+ ttyin 0xfffff800054368b0 getty
60184 1 60184 0 Ss+ ttyin 0xfffff80005436cb0 getty
60178 1 60178 0 Ss+ ttyin 0xfffff800055c40b0 getty
59940 1 59940 0 Ss+ ttyin 0xfffff800055c44b0 getty
59747 1 59747 0 Ss+ ttyin 0xfffff800055c48b0 getty
59746 1 59746 0 Ss+ ttyin 0xfffff800055c4cb0 getty
59697 1 59697 0 Ss+ ttyin 0xfffff800055c50b0 getty
41042 1 41024 0 S piperd 0xfffff80005ce1000 php_wg
27058 347 347 0 SW accept 0xfffff8000f8afba0 php-fpm
12404 11991 11991 0 SW nanslp 0xffffffff836a0df0 minicron
11991 1 11991 0 SWs wait 0xfffff80005ce5000 minicron
11716 11363 11363 0 SW nanslp 0xffffffff836a0df0 minicron
11363 1 11363 0 SWs wait 0xfffff80038145530 minicron
11284 10804 10804 0 SW nanslp 0xffffffff836a0df0 minicron
10804 1 10804 0 SWs wait 0xfffff80038147000 minicron
38058 1 38058 0 Ss select 0xfffff8001f19f640 syslogd
3743 1 3743 0 Ds (threaded) ntpd
100500 D pfault 0xffffffff81e47f00 ntpd
3525 1 3525 0 SWs nanslp 0xffffffff836a0df0 cron
3146 2802 2802 0 SW kqread 0xfffff8002f9f7500 nginx
3079 2802 2802 0 SW kqread 0xfffff800387c8300 nginx
2802 1 2802 0 SWs pause 0xfffff800384650a8 nginx
57007 1 57007 0 Ss select 0xfffff8002fe051c0 sshd
40588 1 40588 0 Ss bpf 0xfffff8002ff93c00 filterlog
12311 1 12311 65 Ss select 0xfffff8002f056e40 dhclient
8007 1 8007 0 Ss select 0xfffff8002f056840 dhclient
6230 1 27 0 D+ (threaded) pcscd
100464 D pfault 0xffffffff81e47f00 pcscd
100474 S piperd 0xfffff80005ce1be0 pcscd
100475 D pfault 0xffffffff81e47f00 pcscd
477 1 477 0 Ss select 0xfffff80005fb8e40 devd
389 387 387 0 SW kqread 0xfffff800279a5400 check_reload_status
387 1 387 0 SWs kqread 0xfffff800279a5700 check_reload_status
347 1 347 0 Ds pfault 0xffffffff81e47f00 php-fpm
78 0 0 0 DL mdwait 0xfffff8000f432000 [md0]
26 0 0 0 DL (threaded) [zpool-pfSense]
100107 D spa->sp 0xfffffe0013bfacd8 [pfSense]
100108 D - 0xfffff80005652b00 [zio_null_issue]
100109 D - 0xfffff80005652900 [zio_null_intr]
100110 D - 0xfffff80005652700 [zio_read_issue_0]
100111 D - 0xfffff80005652700 [zio_read_issue_1]
100112 D - 0xfffff80005652700 [zio_read_issue_2]
100113 D - 0xfffff80005652700 [zio_read_issue_3]
100114 D - 0xfffff80005652700 [zio_read_issue_4]
100115 D - 0xfffff80005652700 [zio_read_issue_5]
100116 D - 0xfffff80005652700 [zio_read_issue_6]
100117 D - 0xfffff80005652700 [zio_read_issue_7]
100118 D - 0xfffff8000563ad00 [zio_read_intr_0_0]
100119 D - 0xfffff8000563ad00 [zio_read_intr_0_1]
100120 D - 0xfffff8000563ad00 [zio_read_intr_0_2]
100121 D - 0xfffff8000563ad00 [zio_read_intr_0_3]
100122 D - 0xfffff8000563ad00 [zio_read_intr_0_4]
100123 D - 0xfffff8000563ad00 [zio_read_intr_0_5]
100124 D - 0xfffff8000563ad00 [zio_read_intr_0_6]
100125 D - 0xfffff8000563ad00 [zio_read_intr_0_7]
100126 D - 0xfffff8000563ad00 [zio_read_intr_0_8]
100127 D - 0xfffff8000563ad00 [zio_read_intr_0_9]
100128 D - 0xfffff8000563ad00 [zio_read_intr_0_10]
100129 D - 0xfffff8000563ad00 [zio_read_intr_0_11]
100130 D - 0xfffff80005682500 [zio_read_intr_1_0]
100131 D - 0xfffff80005682500 [zio_read_intr_1_1]
100132 D - 0xfffff80005682500 [zio_read_intr_1_2]
100133 D - 0xfffff80005682500 [zio_read_intr_1_3]
100134 D - 0xfffff80005682500 [zio_read_intr_1_4]
100135 D - 0xfffff80005682500 [zio_read_intr_1_5]
100136 D - 0xfffff80005682500 [zio_read_intr_1_6]
100137 D - 0xfffff80005682500 [zio_read_intr_1_7]
100138 D - 0xfffff80005682500 [zio_read_intr_1_8]
100139 D - 0xfffff80005682500 [zio_read_intr_1_9]
100140 D - 0xfffff80005682500 [zio_read_intr_1_10]
100141 D - 0xfffff80005682500 [zio_read_intr_1_11]
100142 D - 0xfffff80005681700 [zio_read_intr_2_0]
100143 D - 0xfffff80005681700 [zio_read_intr_2_1]
100144 D - 0xfffff80005681700 [zio_read_intr_2_2]
100145 D - 0xfffff80005681700 [zio_read_intr_2_3]
100146 D - 0xfffff80005681700 [zio_read_intr_2_4]
100147 D - 0xfffff80005681700 [zio_read_intr_2_5]
100148 D - 0xfffff80005681700 [zio_read_intr_2_6]
100149 D - 0xfffff80005681700 [zio_read_intr_2_7]
100150 D - 0xfffff80005681700 [zio_read_intr_2_8]
100151 D - 0xfffff80005681700 [zio_read_intr_2_9]
100152 D - 0xfffff80005681700 [zio_read_intr_2_10]
100153 D - 0xfffff80005681700 [zio_read_intr_2_11]
100154 D - 0xfffff80005680900 [zio_read_intr_3_0]
100155 D - 0xfffff80005680900 [zio_read_intr_3_1]
100156 D - 0xfffff80005680900 [zio_read_intr_3_2]
100157 D - 0xfffff80005680900 [zio_read_intr_3_3]
100158 D - 0xfffff80005680900 [zio_read_intr_3_4]
100159 D - 0xfffff80005680900 [zio_read_intr_3_5]
100160 D - 0xfffff80005680900 [zio_read_intr_3_6]
100161 D - 0xfffff80005680900 [zio_read_intr_3_7]
100162 D - 0xfffff80005680900 [zio_read_intr_3_8]
100163 D - 0xfffff80005680900 [zio_read_intr_3_9]
100164 D - 0xfffff80005680900 [zio_read_intr_3_10]
100165 D - 0xfffff80005680900 [zio_read_intr_3_11]
100166 D - 0xfffff8000567fb00 [zio_read_intr_4_0]
100167 D - 0xfffff8000567fb00 [zio_read_intr_4_1]
100168 D - 0xfffff8000567fb00 [zio_read_intr_4_2]
100169 D - 0xfffff8000567fb00 [zio_read_intr_4_3]
100170 D - 0xfffff8000567fb00 [zio_read_intr_4_4]
100171 D - 0xfffff8000567fb00 [zio_read_intr_4_5]
100172 D - 0xfffff8000567fb00 [zio_read_intr_4_6]
100173 D - 0xfffff8000567fb00 [zio_read_intr_4_7]
100174 D - 0xfffff8000567fb00 [zio_read_intr_4_8]
100175 D - 0xfffff8000567fb00 [zio_read_intr_4_9]
100176 D - 0xfffff8000567fb00 [zio_read_intr_4_10]
100177 D - 0xfffff8000567fb00 [zio_read_intr_4_11]
100178 D - 0xfffff800056ba000 [zio_read_intr_5_0]
100179 D - 0xfffff800056ba000 [zio_read_intr_5_1]
100180 D - 0xfffff800056ba000 [zio_read_intr_5_2]
100181 D - 0xfffff800056ba000 [zio_read_intr_5_3]
100182 D - 0xfffff800056ba000 [zio_read_intr_5_4]
100183 D - 0xfffff800056ba000 [zio_read_intr_5_5]
100184 D - 0xfffff800056ba000 [zio_read_intr_5_6]
100185 D - 0xfffff800056ba000 [zio_read_intr_5_7]
100186 D - 0xfffff800056ba000 [zio_read_intr_5_8]
100187 D - 0xfffff800056ba000 [zio_read_intr_5_9]
100188 D - 0xfffff800056ba000 [zio_read_intr_5_10]
100189 D - 0xfffff800056ba000 [zio_read_intr_5_11]
100190 D - 0xfffff800056b9200 [zio_read_intr_6_0]
100191 D - 0xfffff800056b9200 [zio_read_intr_6_1]
100192 D - 0xfffff800056b9200 [zio_read_intr_6_2]
100193 D - 0xfffff800056b9200 [zio_read_intr_6_3]
100194 D - 0xfffff800056b9200 [zio_read_intr_6_4]
100195 D - 0xfffff800056b9200 [zio_read_intr_6_5]
100196 D - 0xfffff800056b9200 [zio_read_intr_6_6]
100197 D - 0xfffff800056b9200 [zio_read_intr_6_7]
100198 D - 0xfffff800056b9200 [zio_read_intr_6_8]
100199 D - 0xfffff800056b9200 [zio_read_intr_6_9]
100200 D - 0xfffff800056b9200 [zio_read_intr_6_10]
100201 D - 0xfffff800056b9200 [zio_read_intr_6_11]
100202 D - 0xfffff800056b8400 [zio_read_intr_7_0]
100203 D - 0xfffff800056b8400 [zio_read_intr_7_1]
100204 D - 0xfffff800056b8400 [zio_read_intr_7_2]
100205 D - 0xfffff800056b8400 [zio_read_intr_7_3]
100206 D - 0xfffff800056b8400 [zio_read_intr_7_4]
100207 D - 0xfffff800056b8400 [zio_read_intr_7_5]
100208 D - 0xfffff800056b8400 [zio_read_intr_7_6]
100209 D - 0xfffff800056b8400 [zio_read_intr_7_7]
100210 D - 0xfffff800056b8400 [zio_read_intr_7_8]
100211 D - 0xfffff800056b8400 [zio_read_intr_7_9]
100212 D - 0xfffff800056b8400 [zio_read_intr_7_10]
100213 D - 0xfffff800056b8400 [zio_read_intr_7_11]
100214 D - 0xfffff800056b7600 [zio_write_issue]
100215 D - 0xfffff800056b7400 [zio_write_issue_hig]
100216 D - 0xfffff800056b7400 [zio_write_issue_hig]
100217 D - 0xfffff800056b7400 [zio_write_issue_hig]
100218 D - 0xfffff800056b7400 [zio_write_issue_hig]
100219 D - 0xfffff800056b7400 [zio_write_issue_hig]
100220 D - 0xfffff80005683d00 [zio_write_intr_0_0]
100221 D - 0xfffff80005683d00 [zio_write_intr_0_1]
100222 D - 0xfffff80005683d00 [zio_write_intr_0_2]
100223 D - 0xfffff80005683d00 [zio_write_intr_0_3]
100224 D - 0xfffff80005683d00 [zio_write_intr_0_4]
100225 D - 0xfffff80005683d00 [zio_write_intr_0_5]
100226 D - 0xfffff80005683d00 [zio_write_intr_0_6]
100227 D - 0xfffff80005683d00 [zio_write_intr_0_7]
100228 D - 0xfffff80005683d00 [zio_write_intr_0_8]
100229 D - 0xfffff80005683d00 [zio_write_intr_0_9]
100230 D - 0xfffff80005683d00 [zio_write_intr_0_10]
100231 D - 0xfffff80005683d00 [zio_write_intr_0_11]
100232 D - 0xfffff800056e8200 [zio_write_intr_1_0]
100233 D - 0xfffff800056e8200 [zio_write_intr_1_1]
100234 D - 0xfffff800056e8200 [zio_write_intr_1_2]
100235 D - 0xfffff800056e8200 [zio_write_intr_1_3]
100236 D - 0xfffff800056e8200 [zio_write_intr_1_4]
100237 D - 0xfffff800056e8200 [zio_write_intr_1_5]
100238 D - 0xfffff800056e8200 [zio_write_intr_1_6]
100239 D - 0xfffff800056e8200 [zio_write_intr_1_7]
100240 D - 0xfffff800056e8200 [zio_write_intr_1_8]
100241 D - 0xfffff800056e8200 [zio_write_intr_1_9]
100242 D - 0xfffff800056e8200 [zio_write_intr_1_10]
100243 D - 0xfffff800056e8200 [zio_write_intr_1_11]
100244 D - 0xfffff800056e7400 [zio_write_intr_2_0]
100245 D - 0xfffff800056e7400 [zio_write_intr_2_1]
100246 D - 0xfffff800056e7400 [zio_write_intr_2_2]
100247 D - 0xfffff800056e7400 [zio_write_intr_2_3]
100248 D - 0xfffff800056e7400 [zio_write_intr_2_4]
100249 D - 0xfffff800056e7400 [zio_write_intr_2_5]
100250 D - 0xfffff800056e7400 [zio_write_intr_2_6]
100251 D - 0xfffff800056e7400 [zio_write_intr_2_7]
100252 D - 0xfffff800056e7400 [zio_write_intr_2_8]
100253 D - 0xfffff800056e7400 [zio_write_intr_2_9]
100254 D - 0xfffff800056e7400 [zio_write_intr_2_10]
100255 D - 0xfffff800056e7400 [zio_write_intr_2_11]
100256 D - 0xfffff800056e6600 [zio_write_intr_3_0]
100257 D - 0xfffff800056e6600 [zio_write_intr_3_1]
100258 D - 0xfffff800056e6600 [zio_write_intr_3_2]
100259 D - 0xfffff800056e6600 [zio_write_intr_3_3]
100260 D - 0xfffff800056e6600 [zio_write_intr_3_4]
100261 D - 0xfffff800056e6600 [zio_write_intr_3_5]
100262 D - 0xfffff800056e6600 [zio_write_intr_3_6]
100263 D - 0xfffff800056e6600 [zio_write_intr_3_7]
100264 D - 0xfffff800056e6600 [zio_write_intr_3_8]
100265 D - 0xfffff800056e6600 [zio_write_intr_3_9]
100266 D - 0xfffff800056e6600 [zio_write_intr_3_10]
100267 D - 0xfffff800056e6600 [zio_write_intr_3_11]
100268 D - 0xfffff800056e5800 [zio_write_intr_4_0]
100269 D - 0xfffff800056e5800 [zio_write_intr_4_1]
100270 D - 0xfffff800056e5800 [zio_write_intr_4_2]
100271 D - 0xfffff800056e5800 [zio_write_intr_4_3]
100272 D - 0xfffff800056e5800 [zio_write_intr_4_4]
100273 D - 0xfffff800056e5800 [zio_write_intr_4_5]
100274 D - 0xfffff800056e5800 [zio_write_intr_4_6]
100275 D - 0xfffff800056e5800 [zio_write_intr_4_7]
100276 D - 0xfffff800056e5800 [zio_write_intr_4_8]
100277 D - 0xfffff800056e5800 [zio_write_intr_4_9]
100278 D - 0xfffff800056e5800 [zio_write_intr_4_10]
100279 D - 0xfffff800056e5800 [zio_write_intr_4_11]
100280 D - 0xfffff800056baa00 [zio_write_intr_5_0]
100281 D - 0xfffff800056baa00 [zio_write_intr_5_1]
100282 D - 0xfffff800056baa00 [zio_write_intr_5_2]
100283 D - 0xfffff800056baa00 [zio_write_intr_5_3]
100284 D - 0xfffff800056baa00 [zio_write_intr_5_4]
100285 D - 0xfffff800056baa00 [zio_write_intr_5_5]
100286 D - 0xfffff800056baa00 [zio_write_intr_5_6]
100287 D - 0xfffff800056baa00 [zio_write_intr_5_7]
100288 D - 0xfffff800056baa00 [zio_write_intr_5_8]
100289 D - 0xfffff800056baa00 [zio_write_intr_5_9]
100290 D - 0xfffff800056baa00 [zio_write_intr_5_10]
100291 D - 0xfffff800056baa00 [zio_write_intr_5_11]
100292 D - 0xfffff80005717e00 [zio_write_intr_6_0]
100293 D - 0xfffff80005717e00 [zio_write_intr_6_1]
100294 D - 0xfffff80005717e00 [zio_write_intr_6_2]
100295 D - 0xfffff80005717e00 [zio_write_intr_6_3]
100296 D - 0xfffff80005717e00 [zio_write_intr_6_4]
100297 D - 0xfffff80005717e00 [zio_write_intr_6_5]
100298 D - 0xfffff80005717e00 [zio_write_intr_6_6]
100299 D - 0xfffff80005717e00 [zio_write_intr_6_7]
100300 D - 0xfffff80005717e00 [zio_write_intr_6_8]
100301 D - 0xfffff80005717e00 [zio_write_intr_6_9]
100302 D - 0xfffff80005717e00 [zio_write_intr_6_10]
100303 D - 0xfffff80005717e00 [zio_write_intr_6_11]
100304 D - 0xfffff80005717100 [zio_write_intr_7_0]
100305 D - 0xfffff80005717100 [zio_write_intr_7_1]
100306 D - 0xfffff80005717100 [zio_write_intr_7_2]
100307 D - 0xfffff80005717100 [zio_write_intr_7_3]
100308 D - 0xfffff80005717100 [zio_write_intr_7_4]
100309 D - 0xfffff80005717100 [zio_write_intr_7_5]
100310 D - 0xfffff80005717100 [zio_write_intr_7_6]
100311 D - 0xfffff80005717100 [zio_write_intr_7_7]
100312 D - 0xfffff80005717100 [zio_write_intr_7_8]
100313 D - 0xfffff80005717100 [zio_write_intr_7_9]
100314 D - 0xfffff80005717100 [zio_write_intr_7_10]
100315 D - 0xfffff80005717100 [zio_write_intr_7_11]
100316 D - 0xfffff80005716300 [zio_write_intr_high]
100317 D - 0xfffff80005716300 [zio_write_intr_high]
100318 D - 0xfffff80005716300 [zio_write_intr_high]
100319 D - 0xfffff80005716300 [zio_write_intr_high]
100320 D - 0xfffff80005716300 [zio_write_intr_high]
100321 D - 0xfffff80005715c00 [zio_free_issue_0_0]
100322 D - 0xfffff80005715c00 [zio_free_issue_0_1]
100323 D - 0xfffff80005715c00 [zio_free_issue_0_2]
100324 D - 0xfffff80005715c00 [zio_free_issue_0_3]
100325 D - 0xfffff80005715c00 [zio_free_issue_0_4]
100326 D - 0xfffff80005715c00 [zio_free_issue_0_5]
100327 D - 0xfffff80005715c00 [zio_free_issue_0_6]
100328 D - 0xfffff80005715c00 [zio_free_issue_0_7]
100329 D - 0xfffff80005715c00 [zio_free_issue_0_8]
100330 D - 0xfffff80005715c00 [zio_free_issue_0_9]
100331 D - 0xfffff80005715c00 [zio_free_issue_0_10]
100332 D - 0xfffff80005715c00 [zio_free_issue_0_11]
100333 D - 0xfffff800056e8e00 [zio_free_issue_1_0]
100334 D - 0xfffff800056e8e00 [zio_free_issue_1_1]
100335 D - 0xfffff800056e8e00 [zio_free_issue_1_2]
100336 D - 0xfffff800056e8e00 [zio_free_issue_1_3]
100337 D - 0xfffff800056e8e00 [zio_free_issue_1_4]
100338 D - 0xfffff800056e8e00 [zio_free_issue_1_5]
100339 D - 0xfffff800056e8e00 [zio_free_issue_1_6]
100340 D - 0xfffff800056e8e00 [zio_free_issue_1_7]
100341 D - 0xfffff800056e8e00 [zio_free_issue_1_8]
100342 D - 0xfffff800056e8e00 [zio_free_issue_1_9]
100343 D - 0xfffff800056e8e00 [zio_free_issue_1_10]
100344 D - 0xfffff800056e8e00 [zio_free_issue_1_11]
100345 D - 0xfffff80005745300 [zio_free_issue_2_0]
100346 D - 0xfffff80005745300 [zio_free_issue_2_1]
100347 D - 0xfffff80005745300 [zio_free_issue_2_2]
100348 D - 0xfffff80005745300 [zio_free_issue_2_3]
100349 D - 0xfffff80005745300 [zio_free_issue_2_4]
100350 D - 0xfffff80005745300 [zio_free_issue_2_5]
100351 D - 0xfffff80005745300 [zio_free_issue_2_6]
100352 D - 0xfffff80005745300 [zio_free_issue_2_7]
100353 D - 0xfffff80005745300 [zio_free_issue_2_8]
100354 D - 0xfffff80005745300 [zio_free_issue_2_9]
100355 D - 0xfffff80005745300 [zio_free_issue_2_10]
100356 D - 0xfffff80005745300 [zio_free_issue_2_11]
100357 D - 0xfffff80005744500 [zio_free_issue_3_0]
100358 D - 0xfffff80005744500 [zio_free_issue_3_1]
100359 D - 0xfffff80005744500 [zio_free_issue_3_2]
100360 D - 0xfffff80005744500 [zio_free_issue_3_3]
100361 D - 0xfffff80005744500 [zio_free_issue_3_4]
100362 D - 0xfffff80005744500 [zio_free_issue_3_5]
100363 D - 0xfffff80005744500 [zio_free_issue_3_6]
100364 D - 0xfffff80005744500 [zio_free_issue_3_7]
100365 D - 0xfffff80005744500 [zio_free_issue_3_8]
100366 D - 0xfffff80005744500 [zio_free_issue_3_9]
100367 D - 0xfffff80005744500 [zio_free_issue_3_10]
100368 D - 0xfffff80005744500 [zio_free_issue_3_11]
100369 D - 0xfffff80005743700 [zio_free_issue_4_0]
100370 D - 0xfffff80005743700 [zio_free_issue_4_1]
100371 D - 0xfffff80005743700 [zio_free_issue_4_2]
100372 D - 0xfffff80005743700 [zio_free_issue_4_3]
100373 D - 0xfffff80005743700 [zio_free_issue_4_4]
100374 D - 0xfffff80005743700 [zio_free_issue_4_5]
100375 D - 0xfffff80005743700 [zio_free_issue_4_6]
100376 D - 0xfffff80005743700 [zio_free_issue_4_7]
100377 D - 0xfffff80005743700 [zio_free_issue_4_8]
100378 D - 0xfffff80005743700 [zio_free_issue_4_9]
100379 D - 0xfffff80005743700 [zio_free_issue_4_10]
100380 D - 0xfffff80005743700 [zio_free_issue_4_11]
100381 D - 0xfffff80005742900 [zio_free_issue_5_0]
100382 D - 0xfffff80005742900 [zio_free_issue_5_1]
100383 D - 0xfffff80005742900 [zio_free_issue_5_2]
100384 D - 0xfffff80005742900 [zio_free_issue_5_3]
100385 D - 0xfffff80005742900 [zio_free_issue_5_4]
100386 D - 0xfffff80005742900 [zio_free_issue_5_5]
100387 D - 0xfffff80005742900 [zio_free_issue_5_6]
100388 D - 0xfffff80005742900 [zio_free_issue_5_7]
100389 D - 0xfffff80005742900 [zio_free_issue_5_8]
100390 D - 0xfffff80005742900 [zio_free_issue_5_9]
100391 D - 0xfffff80005742900 [zio_free_issue_5_10]
100392 D - 0xfffff80005742900 [zio_free_issue_5_11]
100393 D - 0xfffff80005718b00 [zio_free_issue_6_0]
100394 D - 0xfffff80005718b00 [zio_free_issue_6_1]
100395 D - 0xfffff80005718b00 [zio_free_issue_6_2]
100396 D - 0xfffff80005718b00 [zio_free_issue_6_3]
100397 D - 0xfffff80005718b00 [zio_free_issue_6_4]
100398 D - 0xfffff80005718b00 [zio_free_issue_6_5]
100399 D - 0xfffff80005718b00 [zio_free_issue_6_6]
100400 D - 0xfffff80005718b00 [zio_free_issue_6_7]
100401 D - 0xfffff80005718b00 [zio_free_issue_6_8]
100402 D - 0xfffff80005718b00 [zio_free_issue_6_9]
100403 D - 0xfffff80005718b00 [zio_free_issue_6_10]
100404 D - 0xfffff80005718b00 [zio_free_issue_6_11]
100405 D - 0xfffff80005773000 [zio_free_issue_7_0]
100406 D - 0xfffff80005773000 [zio_free_issue_7_1]
100407 D - 0xfffff80005773000 [zio_free_issue_7_2]
100408 D - 0xfffff80005773000 [zio_free_issue_7_3]
100409 D - 0xfffff80005773000 [zio_free_issue_7_4]
100410 D - 0xfffff80005773000 [zio_free_issue_7_5]
100411 D - 0xfffff80005773000 [zio_free_issue_7_6]
100412 D - 0xfffff80005773000 [zio_free_issue_7_7]
100413 D - 0xfffff80005773000 [zio_free_issue_7_8]
100414 D - 0xfffff80005773000 [zio_free_issue_7_9]
100415 D - 0xfffff80005773000 [zio_free_issue_7_10]
100416 D - 0xfffff80005773000 [zio_free_issue_7_11]
100417 D - 0xfffff80005772200 [zio_free_intr]
100418 D - 0xfffff80005772000 [zio_claim_issue]
100419 D - 0xfffff80005771d00 [zio_claim_intr]
100420 D - 0xfffff80005771b00 [zio_ioctl_issue]
100421 D - 0xfffff80005771900 [zio_ioctl_intr]
100440 D tx->tx_ 0xfffff8000563e2a8 [txg_thread_enter]
100441 D tx->tx_ 0xfffff8000563e288 [txg_thread_enter]
25 0 0 0 DL - 0xffffffff836ebf9c [soaiod4]
24 0 0 0 DL - 0xffffffff836ebf9c [soaiod3]
23 0 0 0 DL - 0xffffffff836ebf9c [soaiod2]
22 0 0 0 DL - 0xffffffff836ebf9c [soaiod1]
21 0 0 0 DL aldslp 0xffffffff8368dd48 [ALQ Daemon]
20 0 0 0 DL vlruwt 0xfffff8000561f530 [vnlru]
19 0 0 0 DL syncer 0xffffffff836ed288 [syncer]
18 0 0 0 DL (threaded) [bufdaemon]
100096 D qsleep 0xffffffff836ec590 [bufdaemon]
100102 D - 0xffffffff81e0a900 [bufspacedaemon-0]
17 0 0 0 DL psleep 0xffffffff83714d88 [vmdaemon]
9 0 0 0 DL (threaded) [pagedaemon]
100094 D pwait 0xffffffff8369fa90 [dom0]
100100 D laundp 0xffffffff8369fa90 [laundry: dom0]
100101 D umarcl 0xffffffff811ddd90 [uma]
8 0 0 0 DL - 0xffffffff834e5af8 [rand_harvestq]
7 0 0 0 DL pftm 0xffffffff81086290 [pf purge]
6 0 0 0 DL waiting 0xffffffff836f25f8 [sctp_iterator]
5 0 0 0 DL (threaded) [zfskern]
100060 D t->zthr 0xfffff8000537d6c8 [solthread 0xfffffff]
100061 D soldela 0xffffffff8369fa90 [solthread 0xfffffff]
100062 D arc_dnl 0xffffffff83eb0c10 [arc_dnlc_evicts_thr]
100063 D - 0xfffff80005571600 [dbu_evict]
100064 D dbuf_ev 0xffffffff83eb3838 [dbuf_evict_thread]
100065 D - 0xfffff80005571100 [z_vdev_file_0]
100066 D - 0xfffff80005571100 [z_vdev_file_1]
100067 D - 0xfffff80005571100 [z_vdev_file_2]
100068 D - 0xfffff80005571100 [z_vdev_file_3]
100069 D - 0xfffff80005571100 [z_vdev_file_4]
100070 D - 0xfffff80005571100 [z_vdev_file_5]
100071 D - 0xfffff80005571100 [z_vdev_file_6]
100072 D - 0xfffff80005571100 [z_vdev_file_7]
100073 D - 0xfffff80005571100 [z_vdev_file_8]
100074 D - 0xfffff80005571100 [z_vdev_file_9]
100075 D - 0xfffff80005571100 [z_vdev_file_10]
100076 D - 0xfffff80005571100 [z_vdev_file_11]
100077 D - 0xfffff80005571100 [z_vdev_file_12]
100078 D - 0xfffff80005571100 [z_vdev_file_13]
100079 D - 0xfffff80005571100 [z_vdev_file_14]
100080 D - 0xfffff80005571100 [z_vdev_file_15]
100081 D l2arc_f 0xffffffff83eb0d60 [l2arc_feed_thread]
100082 D - 0xfffff8000556fd00 [zfsvfs]
100422 D spa->sp 0xfffffe0013bfad28 [trim pfSense]
100423 D - 0xfffff80005771600 [z_zvol]
100424 D - 0xfffff80005635c00 [dp_sync_taskq]
100425 D - 0xfffff80005771400 [dp_zil_clean_taskq]
100426 D - 0xfffff80005635d00 [zfs_vn_rele_taskq]
100427 D - 0xfffff80005770600 [metaslab_group_task]
100442 S mmp->mm 0xfffffe0013bfaf08 [mmp_thread_enter]
100443 D t->zthr 0xfffff8000590c6c8 [solthread 0xfffffff]
100444 D t->zthr 0xfffff8000590c5c8 [solthread 0xfffffff]
16 0 0 0 DL (threaded) [usb]
100040 D - 0xfffffe0000665dc8 [usbus0]
100041 D - 0xfffffe0000665e20 [usbus0]
100042 D - 0xfffffe0000665e78 [usbus0]
100043 D - 0xfffffe0000665ed0 [usbus0]
100044 D - 0xfffffe0000665f28 [usbus0]
4 0 0 0 DL (threaded) [cam]
100036 D - 0xffffffff8339c4c0 [doneq0]
100092 D - 0xffffffff8339c390 [scanner]
3 0 0 0 DL crypto_ 0xfffff80005202bb0 [crypto returns 0]
2 0 0 0 DL crypto_ 0xffffffff837072b8 [crypto]
15 0 0 0 DL seqstat 0xfffff80005122888 [sequencer 00]
14 0 0 0 DL (threaded) [geom]
100029 D - 0xffffffff8368c3c8 [g_event]
100030 D - 0xffffffff8368c3d0 [g_up]
100031 D - 0xffffffff8368c3d8 [g_down]
13 0 0 0 DL sleep 0xffffffff8330ec10 [ng_queue0]
12 0 0 0 WL (threaded) [intr]
100005 I [swi5: fast taskq]
100010 I [swi6: task queue]
100012 I [swi6: Giant taskq]
100024 I [swi1: netisr 0]
100025 I [swi3: vm]
100026 I [swi4: clock (0)]
100037 I [irq14: ata0]
100038 I [irq15: ata1]
100039 I [irq11: uhci0]
100045 I [irq256: virtio_pci0]
100046 I [irq257: virtio_pci0]
100047 I [irq258: virtio_pci0]
100050 I [irq259: virtio_pci1]
100051 I [irq260: virtio_pci1]
100052 I [irq10: virtio_pci2]
100054 I [irq1: atkbd0]
100055 I [irq12: psm0]
100056 I [swi0: uart uart]
100085 I [swi1: pf send]
100086 I [swi1: pfsync]
100471 I [irq9: acpi0 intsmb0]
11 0 0 0 RL [idle: cpu0]
1 0 1 0 SLs wait 0xfffff80005149530 [init]
10 0 0 0 DL audit_w 0xffffffff83707898 [audit]
0 0 0 0 DLs (threaded) [kernel]
100000 D swapin 0xffffffff8368c568 [swapper]
100004 D - 0xfffff8000505eb00 [thread taskq]
100006 D - 0xfffff8000505e600 [in6m_free taskq]
100007 D - 0xfffff8000505e400 [kqueue_ctx taskq]
100008 D - 0xfffff8000505e200 [pci_hp taskq]
100009 D - 0xfffff8000505e000 [aiod_kick taskq]
100011 D - 0xfffff8000505da00 [inm_free taskq]
100013 D - 0xfffff8000505d500 [linuxkpi_short_wq_0]
100014 D - 0xfffff8000505d500 [linuxkpi_short_wq_1]
100015 D - 0xfffff8000505d500 [linuxkpi_short_wq_2]
100016 D - 0xfffff8000505d500 [linuxkpi_short_wq_3]
100017 D - 0xfffff8000505d000 [linuxkpi_long_wq_0]
100018 D - 0xfffff8000505d000 [linuxkpi_long_wq_1]
100019 D - 0xfffff8000505d000 [linuxkpi_long_wq_2]
100020 D - 0xfffff8000505d000 [linuxkpi_long_wq_3]
100021 D - 0xfffff80005170c00 [if_config_tqg_0]
100022 D - 0xfffff80005170a00 [if_io_tqg_0]
100023 D - 0xfffff80005170800 [softirq_0]
100028 D - 0xfffff8000516fd00 [firmware taskq]
100033 D - 0xfffff8000516e400 [crypto]
100048 D - 0xfffff800053b1100 [vtnet0 rxq 0]
100049 D - 0xfffff800053b1000 [vtnet0 txq 0]
100053 D vtbslp 0xfffff800053a8800 [virtio_balloon]
100057 D - 0xfffff800053ae900 [mca taskq]
100058 D - 0xfffff800053ae700 [system_taskq]
100059 D - 0xfffff8000534f700 [arc_prune]
100087 D - 0xfffff800055eec00 [acpi_task_0]
100088 D - 0xfffff800055eec00 [acpi_task_1]
100089 D - 0xfffff800055eec00 [acpi_task_2]
100091 D - 0xfffff80005061700 [CAM taskq]
100093 D - 0xfffff800055ed900 [mlx4]
100472 D - 0xfffff8002f985100 [wg_tqg_0]
db:0:kdb.enter.default> alltrace

Tracing command sh pid 39540 tid 100526 td 0xfffff8001c021000
sched_switch() at sched_switch+0x630/frame 0xfffffe001900e7e0
db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe001905d610
kdb_reenter() at kdb_reenter+0x2c/frame 0xfffffe001905d620
trap() at trap+0x56/frame 0xfffffe001905d730
calltrap() at calltrap+0x8/frame 0xfffffe001905d730
--- trap 0xc, rip = 0xffffffff81357560, rsp = 0xfffffe001905d800, rbp = 0xfffffe001905d890 ---
db_read_bytes() at db_read_bytes+0x90/frame 0xfffffe001905d890
db_get_value() at db_get_value+0x32/frame 0xfffffe001905d8d0
db_backtrace() at db_backtrace+0x23e/frame 0xfffffe001905d960
db_trace_thread() at db_trace_thread+0x3b/frame 0xfffffe001905d980
_db_stack_trace_all() at _db_stack_trace_all+0xb6/frame 0xfffffe001905da20
db_command() at db_command+0x256/frame 0xfffffe001905daf0
db_script_exec() at db_script_exec+0x1a8/frame 0xfffffe001905db90
db_script_kdbenter() at db_script_kdbenter+0x55/frame 0xfffffe001905dbd0
db_trap() at db_trap+0xe3/frame 0xfffffe001905dc60
kdb_trap() at kdb_trap+0x142/frame 0xfffffe001905dcd0
trap() at trap+0x75f/frame 0xfffffe001905dde0
calltrap() at calltrap+0x8/frame 0xfffffe001905dde0
--- trap 0x3, rip = 0xffffffff80dddc57, rsp = 0xfffffe001905deb0, rbp = 0xfffffe001905dec0 ---
kdb_enter() at kdb_enter+0x37/frame 0xfffffe001905dec0
vpanic() at vpanic+0x197/frame 0xfffffe001905df10
panic() at panic+0x43/frame 0xfffffe001905df70
trap_fatal() at trap_fatal+0x391/frame 0xfffffe001905dfd0
trap_pfault() at trap_pfault+0x4f/frame 0xfffffe001905e020
trap() at trap+0x286/frame 0xfffffe001905e130
calltrap() at calltrap+0x8/frame 0xfffffe001905e130
--- trap 0xc, rip = 0xffffffff811dc654, rsp = 0xfffffe001905e200, rbp = 0xfffffe001905e250 ---
uma_zfree_arg() at uma_zfree_arg+0x24/frame 0xfffffe001905e250
free() at free+0x5a/frame 0xfffffe001905e280
nvpair_free() at nvpair_free+0x99/frame 0xfffffe001905e2a0
nvlist_destroy() at nvlist_destroy+0x60/frame 0xfffffe001905e2d0
pf_state_to_nvstate() at pf_state_to_nvstate+0x94/frame 0xfffffe001905e310
pfioctl() at pfioctl+0x1b27/frame 0xfffffe001905e7e0
devfs_ioctl() at devfs_ioctl+0xb0/frame 0xfffffe001905e830
VOP_IOCTL_APV() at VOP_IOCTL_APV+0x7b/frame 0xfffffe001905e860
vn_ioctl() at vn_ioctl+0x16c/frame 0xfffffe001905e970
devfs_ioctl_f() at devfs_ioctl_f+0x1e/frame 0xfffffe001905e990
kern_ioctl() at kern_ioctl+0x2b7/frame 0xfffffe001905e9f0
sys_ioctl() at sys_ioctl+0x101/frame 0xfffffe001905eac0
amd64_syscall() at amd64_syscall+0x387/frame 0xfffffe001905ebf0
fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe001905ebf0
--- syscall (54, FreeBSD ELF64, sys_ioctl), rip = 0x800490b3a, rsp = 0x7fffffffe5c8, rbp = 0x7fffffffe640 ---
*** error reading from address fffffe001900e7e8 ***
db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe001905d880
kdb_reenter() at kdb_reenter+0x2c/frame 0xfffffe001905d890
db_get_value() at db_get_value+0x4c/frame 0xfffffe001905d8d0
db_backtrace() at db_backtrace+0x23e/frame 0xfffffe001905d960
db_trace_thread() at db_trace_thread+0x3b/frame 0xfffffe001905d980
_db_stack_trace_all() at _db_stack_trace_all+0xb6/frame 0xfffffe001905da20
db_command() at db_command+0x256/frame 0xfffffe001905daf0
db_script_exec() at db_script_exec+0x1a8/frame 0xfffffe001905db90
db_script_kdbenter() at db_script_kdbenter+0x55/frame 0xfffffe001905dbd0
db_trap() at db_trap+0xe3/frame 0xfffffe001905dc60
kdb_trap() at kdb_trap+0x142/frame 0xfffffe001905dcd0
trap() at trap+0x75f/frame 0xfffffe001905dde0
calltrap() at calltrap+0x8/frame 0xfffffe001905dde0
--- trap 0x3, rip = 0xffffffff80dddc57, rsp = 0xfffffe001905deb0, rbp = 0xfffffe001905dec0 ---
kdb_enter() at kdb_enter+0x37/frame 0xfffffe001905dec0
vpanic() at vpanic+0x197/frame 0xfffffe001905df10
panic() at panic+0x43/frame 0xfffffe001905df70
trap_fatal() at trap_fatal+0x391/frame 0xfffffe001905dfd0
trap_pfault() at trap_pfault+0x4f/frame 0xfffffe001905e020
trap() at trap+0x286/frame 0xfffffe001905e130
calltrap() at calltrap+0x8/frame 0xfffffe001905e130
--- trap 0xc, rip = 0xffffffff811dc654, rsp = 0xfffffe001905e200, rbp = 0xfffffe001905e250 ---
uma_zfree_arg() at uma_zfree_arg+0x24/frame 0xfffffe001905e250
free() at free+0x5a/frame 0xfffffe001905e280
nvpair_free() at nvpair_free+0x99/frame 0xfffffe001905e2a0
nvlist_destroy() at nvlist_destroy+0x60/frame 0xfffffe001905e2d0
pf_state_to_nvstate() at pf_state_to_nvstate+0x94/frame 0xfffffe001905e310
pfioctl() at pfioctl+0x1b27/frame 0xfffffe001905e7e0
devfs_ioctl() at devfs_ioctl+0xb0/frame 0xfffffe001905e830
VOP_IOCTL_APV() at VOP_IOCTL_APV+0x7b/frame 0xfffffe001905e860
vn_ioctl() at vn_ioctl+0x16c/frame 0xfffffe001905e970
devfs_ioctl_f() at devfs_ioctl_f+0x1e/frame 0xfffffe001905e990
kern_ioctl() at kern_ioctl+0x2b7/frame 0xfffffe001905e9f0
sys_ioctl() at sys_ioctl+0x101/frame 0xfffffe001905eac0
amd64_syscall() at amd64_syscall+0x387/frame 0xfffffe001905ebf0
fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe001905ebf0
--- syscall (54, FreeBSD ELF64, sys_ioctl), rip = 0x800490b3a, rsp = 0x7fffffffe5c8, rbp = 0x7fffffffe640 ---

Tracing command pfctl pid 53869 tid 100554 td 0xfffff8001786c000 (CPU 0)
kdb_enter() at kdb_enter+0x37/frame 0xfffffe001905dec0
vpanic() at vpanic+0x197/frame 0xfffffe001905df10
panic() at panic+0x43/frame 0xfffffe001905df70
trap_fatal() at trap_fatal+0x391/frame 0xfffffe001905dfd0
trap_pfault() at trap_pfault+0x4f/frame 0xfffffe001905e020
trap() at trap+0x286/frame 0xfffffe001905e130
calltrap() at calltrap+0x8/frame 0xfffffe001905e130
--- trap 0xc, rip = 0xffffffff811dc654, rsp = 0xfffffe001905e200, rbp = 0xfffffe001905e250 ---
uma_zfree_arg() at uma_zfree_arg+0x24/frame 0xfffffe001905e250
free() at free+0x5a/frame 0xfffffe001905e280
nvpair_free() at nvpair_free+0x99/frame 0xfffffe001905e2a0
nvlist_destroy() at nvlist_destroy+0x60/frame 0xfffffe001905e2d0
pf_state_to_nvstate() at pf_state_to_nvstate+0x94/frame 0xfffffe001905e310
pfioctl() at pfioctl+0x1b27/frame 0xfffffe001905e7e0
devfs_ioctl() at devfs_ioctl+0xb0/frame 0xfffffe001905e830
VOP_
(4-4/7)