Actions
Bug #2793
closed
Snapshot built from today's (Feb 4th) code is unstable, goes into panic loop.
Start date:
02/04/2013
Due date:
% Done:
0%
Estimated time:
Plus Target Version:
Release Notes:
Affected Version:
2.1
Affected Architecture:
amd64
Description
I can't get the VM to stay up long enough to capture or submit a full dump, but here is what I was able to get:
Many of these printed over and over to the console and system log:
Feb 5 00:02:46 pfsense-amd64 kernel: pfi_kif_unref: state refcount <= 0 Feb 5 00:02:47 pfsense-amd64 kernel: pfi_kif_unref: state refcount <= 0 Feb 5 00:02:47 pfsense-amd64 kernel: pfi_kif_unref: state refcount <= 0
And then within a few moments, it will panic. It seems like the more network traffic you send, the faster it panics.
Messages continue even with pfctl disabled (pfctl -d) and the FTP proxy off (debug.pfftpproxy=1)
Here is what I was able to get of one crash dump:
db:0:kdb.enter.default> run lockinfo
db:1:lockinfo> show locks
No such command
db:1:locks> show alllocks
No such command
db:1:alllocks> show lockedvnods
Locked vnodes
db:0:kdb.enter.default> show pcpu
cpuid = 0
dynamic pcpu = 0x32f000
curthread = 0xffffff000251f000: pid 8 "pfpurge"
curpcb = 0xffffff8000104d10
fpcurthread = none
idlethread = 0xffffff00022be460: tid 100003 "idle: cpu0"
curpmap = 0xffffffff81386090
tssp = 0xffffffff8140ab80
commontssp = 0xffffffff8140ab80
rsp0 = 0xffffff8000104d10
gs32p = 0xffffffff814099b8
ldt = 0xffffffff814099f8
tss = 0xffffffff814099e8
db:0:kdb.enter.default> bt
Tracing pid 8 tid 100033 td 0xffffff000251f000
pf_state_tree_lan_ext_RB_REMOVE_COLOR() at pf_state_tree_lan_ext_RB_REMOVE_COLOR+0xd4
pf_state_tree_lan_ext_RB_REMOVE() at pf_state_tree_lan_ext_RB_REMOVE+0xc6
pf_unlink_state() at pf_unlink_state+0x4f
pf_purge_expired_states() at pf_purge_expired_states+0xa3
pf_purge_thread() at pf_purge_thread+0xcb
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000104d00, rbp = 0 ---
db:0:kdb.enter.default> ps
pid ppid pgrp uid state wmesg wchan cmd
74552 20564 23 0 S+ nanslp 0xffffffff81389c28 sleep
64598 1 64598 0 Ss select 0xffffff0002c8a4c0 racoon
30060 26036 26036 0 S piperd 0xffffff0002602000 rrdtool
26036 1 26036 0 Ss select 0xffffff0002a50d40 apinger
22847 1 22847 0 Ss select 0xffffff0002dd71c0 miniupnpd
20564 1 23 0 S+ wait 0xffffff0002db5470 sh
13463 1 13463 0 Ss select 0xffffff0002a98d40 bsnmpd
12994 273 273 0 S nanslp 0xffffffff81389c28 initial thread
97327 1 97327 0 Ss (threaded) mpd4
100102 S select 0xffffff0002e68e40 mpd4
89605 1 89605 0 Ss select 0xffffff0002f9a4c0 radvd
87743 1 87743 0 Ss kqread 0xffffff0002feb800 dhcpleases6
87528 1 87528 1002 Ss select 0xffffff0002a97540 dhcpd
86238 1 86238 1002 Ss select 0xffffff0002f9b3c0 dhcpd
84915 83727 83727 0 S accept 0xffffff0002de3dae php
83727 75232 83727 0 Ss wait 0xffffff0002fd3470 initial thread
81222 80912 80912 913 S kqread 0xffffff0002e05500 relayd
81179 80912 80912 913 S kqread 0xffffff0002e93400 relayd
80912 1 80912 0 Ss kqread 0xffffff0002e93800 relayd
78174 1 77885 65534 S select 0xffffff0002e68040 dnsmasq
77034 1 77034 0 Ss kqread 0xffffff0002c77700 dhcpleases
76064 75609 75609 0 S accept 0xffffff0002e2b85e php
75609 75232 75609 0 Ss wait 0xffffff0002b9b000 initial thread
75232 1 75181 0 S kqread 0xffffff0002e05300 lighttpd
52491 1 52491 0 Ss (threaded) filterdns
100072 S ucond 0xffffff0002a97a00 signal-thread
100071 S ucond 0xffffff0002a97700 www.google.com
51346 1 51346 0 Ss select 0xffffff0002a500c0 openvpn
51185 1 51185 0 Ss select 0xffffff0002a50040 inetd
47042 1 23 0 S+ piperd 0xffffff0002603000 logger
46923 1 23 0 S+ bpf 0xffffff0002a40c00 tcpdump
46752 1 46752 0 Ss select 0xffffff0002796a40 openvpn
45819 1 45819 0 Ss select 0xffffff0002796c40 openvpn
35580 1 35580 0 Ss select 0xffffff0002796240 openvpn
24150 1 24150 0 Ss select 0xffffff00028e4d40 openvpn
13479 1 13479 0 Ss kqread 0xffffff0002a12000 choparp
12635 8359 12635 0 Ss (threaded) sshlockout_pf
100060 S nanslp 0xffffffff81389c28 sshlockout_pf
100054 S piperd 0xffffff00026025b0 initial thread
12135 1 12135 0 Ss select 0xffffff0002a04940 sshd
8359 1 8359 0 Ss select 0xffffff0002a043c0 syslogd
286 273 273 0 S kqread 0xffffff00025cfa00 check_reload_status
285 1 285 0 Ss select 0xffffff000269c740 devd
277 23 23 0 S+ select 0xffffff0002a972c0 initial thread
273 1 273 0 Ss kqread 0xffffff0002604100 check_reload_status
73 0 0 0 SL mdwait 0xffffff00025c6000 [md0]
23 1 23 0 Ss+ wait 0xffffff00025e48e0 sh
22 0 0 0 RL [softdepflush]
21 0 0 0 RL [vnlru]
20 0 0 0 RL [syncer]
19 0 0 0 RL [bufdaemon]
18 0 0 0 SL pollid 0xffffffff81388a48 [idlepoll]
17 0 0 0 SL pgzero 0xffffffff813d248c [pagezero]
16 0 0 0 SL psleep 0xffffffff813d1828 [vmdaemon]
15 0 0 0 SL psleep 0xffffffff813d17ec [pagedaemon]
9 0 0 0 SL ccb_scan 0xffffffff8134ca60 [xpt_thrd]
8 0 0 0 RL CPU 0 [pfpurge]
7 0 0 0 SL waiting_ 0xffffffff813ba280 [sctp_iterator]
14 0 0 0 RL [yarrow]
6 0 0 0 SL crypto_r 0xffffffff813cf820 [crypto returns]
5 0 0 0 SL crypto_w 0xffffffff813cf7e0 [crypto]
4 0 0 0 SL - 0xffffffff81385548 [g_down]
3 0 0 0 SL - 0xffffffff81385540 [g_up]
2 0 0 0 SL - 0xffffffff81385530 [g_event]
13 0 0 0 SL sleep 0xffffffff812f6550 [ng_queue0]
12 0 0 0 WL (threaded) intr
100031 I [irq12: psm0]
100030 I [irq1: atkbd0]
100027 I [irq15: ata1]
100026 I [irq14: ata0]
100025 I [irq9: acpi0]
100023 I [swi5: +]
100017 I [swi2: cambio]
100016 I [swi6: task queue]
100015 I [swi6: Giant taskq]
100006 I [swi3: vm]
100005 I [swi4: clock]
100004 I [swi1: netisr 0]
11 0 0 0 RL [idle: cpu0]
1 0 1 0 SLs wait 0xffffff00022bb8e0 [init]
10 0 0 0 SL audit_wo 0xffffffff813cfe10 [audit]
0 0 0 0 SLs (threaded) kernel
100029 D - 0xffffff000250a200 [em1 taskq]
100028 D - 0xffffff00024d1880 [em0 taskq]
100024 D - 0xffffff00023f5780 [thread taskq]
100022 D - 0xffffff00023f5c00 [ffs_trim taskq]
100021 D - 0xffffff00023f5c80 [kqueue taskq]
100020 D - 0xffffff00023f5d00 [acpi_task_2]
100019 D - 0xffffff00023f5d00 [acpi_task_1]
100018 D - 0xffffff00023f5d00 [acpi_task_0]
100011 D - 0xffffff0002256580 [firmware taskq]
100000 D sched 0xffffffff813856c0 [swapper]
db:0:kdb.enter.default> alltrace
Tracing command sleep pid 74552 tid 100101 td 0xffffff0002f5c460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_sleep() at _sleep+0x1b1
kern_nanosleep() at kern_nanosleep+0x118
nanosleep() at nanosleep+0x6e
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (240, FreeBSD ELF64, nanosleep), rip = 0x80071fc7c, rsp = 0x7fffffffecf8, rbp = 0x7fffffffed70 ---
Tracing command racoon pid 64598 tid 100081 td 0xffffff0002dc8460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_cv_timedwait_sig() at _cv_timedwait_sig+0x134
seltdwait() at seltdwait+0x98
kern_select() at kern_select+0x6ef
select() at select+0x5d
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (93, FreeBSD ELF64, select), rip = 0x800ef0a4c, rsp = 0x7fffffffe848, rbp = 0x1 ---
Tracing command rrdtool pid 30060 tid 100069 td 0xffffff0002a46000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
pipe_read() at pipe_read+0x3f4
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (3, FreeBSD ELF64, read), rip = 0x800f9bacc, rsp = 0x7fffffffeb88, rbp = 0x801204000 ---
Tracing command apinger pid 26036 tid 100044 td 0xffffff000259e8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_cv_timedwait_sig() at _cv_timedwait_sig+0x134
seltdwait() at seltdwait+0x98
poll() at poll+0x478
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (209, FreeBSD ELF64, poll), rip = 0x8006e9d6c, rsp = 0x7fffffffcb08, rbp = 0x2 ---
Tracing command miniupnpd pid 22847 tid 100082 td 0xffffff0002dc8000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_cv_timedwait_sig() at _cv_timedwait_sig+0x134
seltdwait() at seltdwait+0x98
kern_select() at kern_select+0x6ef
select() at select+0x5d
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (93, FreeBSD ELF64, select), rip = 0x800856a4c, rsp = 0x7fffffffea18, rbp = 0xb ---
Tracing command sh pid 20564 tid 100073 td 0xffffff0002dcb000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
kern_wait() at kern_wait+0x6fd
wait4() at wait4+0x35
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (7, FreeBSD ELF64, wait4), rip = 0x800942dec, rsp = 0x7fffffffe898, rbp = 0x7fffffffea58 ---
Tracing command bsnmpd pid 13463 tid 100052 td 0xffffff000262b000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_cv_timedwait_sig() at _cv_timedwait_sig+0x134
seltdwait() at seltdwait+0x98
kern_select() at kern_select+0x6ef
select() at select+0x5d
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (93, FreeBSD ELF64, select), rip = 0x800a6ca4c, rsp = 0x7fffffffc368, rbp = 0x7fffffffc55c ---
Tracing command php pid 12994 tid 100098 td 0xffffff0002dcb8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_sleep() at _sleep+0x1b1
kern_nanosleep() at kern_nanosleep+0x118
nanosleep() at nanosleep+0x6e
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (240, FreeBSD ELF64, nanosleep), rip = 0x8013e0c7c, rsp = 0x7fffffffaf18, rbp = 0x8038e9060 ---
Tracing command mpd4 pid 97327 tid 100102 td 0xffffff0002f5c000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x128
seltdwait() at seltdwait+0x110
poll() at poll+0x478
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (209, FreeBSD ELF64, poll), rip = 0x80178ed6c, rsp = 0x7fffffbfeed8, rbp = 0xffffffff ---
Tracing command radvd pid 89605 tid 100074 td 0xffffff0002dca8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_cv_timedwait_sig() at _cv_timedwait_sig+0x134
seltdwait() at seltdwait+0x98
poll() at poll+0x478
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (209, FreeBSD ELF64, poll), rip = 0x8006edd6c, rsp = 0x7fffffffe5c8, rbp = 0x7fffffffec10 ---
Tracing command dhcpleases6 pid 87743 tid 100053 td 0xffffff000262a8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
kern_kevent() at kern_kevent+0x38b
kevent() at kevent+0x90
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (363, FreeBSD ELF64, kevent), rip = 0x800722c7c, rsp = 0x7fffffffeb18, rbp = 0x3 ---
Tracing command dhcpd pid 87528 tid 100048 td 0xffffff000259d460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x128
seltdwait() at seltdwait+0x110
kern_select() at kern_select+0x6ef
select() at select+0x5d
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (93, FreeBSD ELF64, select), rip = 0x8009e7a4c, rsp = 0x7fffffffea68, rbp = 0x7fffffffeab8 ---
Tracing command dhcpd pid 86238 tid 100099 td 0xffffff0002dcb460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x128
seltdwait() at seltdwait+0x110
kern_select() at kern_select+0x6ef
select() at select+0x5d
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (93, FreeBSD ELF64, select), rip = 0x8009e7a4c, rsp = 0x7fffffffea78, rbp = 0x7fffffffeac8 ---
Tracing command php pid 84915 tid 100077 td 0xffffff0002dc98c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
kern_accept() at kern_accept+0x1c7
accept() at accept+0x75
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (30, FreeBSD ELF64, accept), rip = 0x80139526c, rsp = 0x7ffffffec7a8, rbp = 0x7fffffffec9c ---
Tracing command php pid 83727 tid 100100 td 0xffffff0002f5c8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
kern_wait() at kern_wait+0x6fd
wait4() at wait4+0x35
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (7, FreeBSD ELF64, wait4), rip = 0x801372dec, rsp = 0x7fffffffc8d8, rbp = 0x7fffffffec9c ---
Tracing command relayd pid 81222 tid 100076 td 0xffffff0002dca000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_sleep() at _sleep+0x1b1
kern_kevent() at kern_kevent+0x38b
kevent() at kevent+0x90
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (363, FreeBSD ELF64, kevent), rip = 0x800c7fc7c, rsp = 0x7fffffffe778, rbp = 0x80106f000 ---
Tracing command relayd pid 81179 tid 100080 td 0xffffff0002dc88c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_sleep() at _sleep+0x1b1
kern_kevent() at kern_kevent+0x38b
kevent() at kevent+0x90
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (363, FreeBSD ELF64, kevent), rip = 0x800c7fc7c, rsp = 0x7fffffffe778, rbp = 0x80106f000 ---
Tracing command relayd pid 80912 tid 100059 td 0xffffff00025a18c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
kern_kevent() at kern_kevent+0x38b
kevent() at kevent+0x90
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (363, FreeBSD ELF64, kevent), rip = 0x800c7fc7c, rsp = 0x7fffffffe7b8, rbp = 0x80106f000 ---
Tracing command dnsmasq pid 78174 tid 100092 td 0xffffff0002e658c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x128
seltdwait() at seltdwait+0x110
kern_select() at kern_select+0x6ef
select() at select+0x5d
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (93, FreeBSD ELF64, select), rip = 0x80089fa4c, rsp = 0x7fffffffe9c8, rbp = 0x7fffffffeb10 ---
Tracing command dhcpleases pid 77034 tid 100085 td 0xffffff0002dc7000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
kern_kevent() at kern_kevent+0x38b
kevent() at kevent+0x90
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (363, FreeBSD ELF64, kevent), rip = 0x800723c7c, rsp = 0x7fffffffeb88, rbp = 0x3 ---
Tracing command php pid 76064 tid 100094 td 0xffffff0002e65000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
kern_accept() at kern_accept+0x1c7
accept() at accept+0x75
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (30, FreeBSD ELF64, accept), rip = 0x80139526c, rsp = 0x7ffffffec7a8, rbp = 0x7fffffffec9c ---
Tracing command php pid 75609 tid 100091 td 0xffffff0002e67000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
kern_wait() at kern_wait+0x6fd
wait4() at wait4+0x35
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (7, FreeBSD ELF64, wait4), rip = 0x801372dec, rsp = 0x7fffffffc8d8, rbp = 0x7fffffffec9c ---
Tracing command lighttpd pid 75232 tid 100089 td 0xffffff0002e678c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_sleep() at _sleep+0x1b1
kern_kevent() at kern_kevent+0x38b
kevent() at kevent+0x90
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (363, FreeBSD ELF64, kevent), rip = 0x800de5c7c, rsp = 0x7fffffffe998, rbp = 0x801088800 ---
Tracing command filterdns pid 52491 tid 100072 td 0xffffff0002a44000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
do_cv_wait() at do_cv_wait+0x861
__umtx_op_cv_wait() at __umtx_op_cv_wait+0x66
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (454, FreeBSD ELF64, _umtx_op), rip = 0x80065d39c, rsp = 0x7fffff9fde98, rbp = 0x800a0ae40 ---
Tracing command filterdns pid 52491 tid 100071 td 0xffffff0002a44460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_sleep() at _sleep+0x1b1
do_cv_wait() at do_cv_wait+0x630
__umtx_op_cv_wait() at __umtx_op_cv_wait+0x66
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (454, FreeBSD ELF64, _umtx_op), rip = 0x80065d39c, rsp = 0x7fffffbfe9c8, rbp = 0x800a2b1c0 ---
Tracing command openvpn pid 51346 tid 100065 td 0xffffff00027d8460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_cv_timedwait_sig() at _cv_timedwait_sig+0x134
seltdwait() at seltdwait+0x98
poll() at poll+0x478
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (209, FreeBSD ELF64, poll), rip = 0x800ca0d6c, rsp = 0x7fffffffdf38, rbp = 0x7fffffffe030 ---
Tracing command inetd pid 51185 tid 100062 td 0xffffff00027d9460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x128
seltdwait() at seltdwait+0x110
kern_select() at kern_select+0x6ef
select() at select+0x5d
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (93, FreeBSD ELF64, select), rip = 0x800a64a4c, rsp = 0x7fffffffdd58, rbp = 0 ---
Tracing command logger pid 47042 tid 100058 td 0xffffff0002629000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
pipe_read() at pipe_read+0x3f4
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (3, FreeBSD ELF64, read), rip = 0x80073cacc, rsp = 0x7fffffffe838, rbp = 0x7fffffffed60 ---
Tracing command tcpdump pid 46923 tid 100061 td 0xffffff00027d98c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_sleep() at _sleep+0x1b1
bpfread() at bpfread+0x27b
devfs_read_f() at devfs_read_f+0x88
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (3, FreeBSD ELF64, read), rip = 0x800ba4acc, rsp = 0x7fffffffe938, rbp = 0x800e05400 ---
Tracing command openvpn pid 46752 tid 100067 td 0xffffff0002a468c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_cv_timedwait_sig() at _cv_timedwait_sig+0x134
seltdwait() at seltdwait+0x98
poll() at poll+0x478
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (209, FreeBSD ELF64, poll), rip = 0x800ca0d6c, rsp = 0x7fffffffdf38, rbp = 0x7fffffffe030 ---
Tracing command openvpn pid 45819 tid 100063 td 0xffffff00027d9000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_cv_timedwait_sig() at _cv_timedwait_sig+0x134
seltdwait() at seltdwait+0x98
poll() at poll+0x478
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (209, FreeBSD ELF64, poll), rip = 0x800ca0d6c, rsp = 0x7fffffffd1a8, rbp = 0x7fffffffd350 ---
Tracing command openvpn pid 35580 tid 100066 td 0xffffff00027d8000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_cv_timedwait_sig() at _cv_timedwait_sig+0x134
seltdwait() at seltdwait+0x98
poll() at poll+0x478
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (209, FreeBSD ELF64, poll), rip = 0x800ca0d6c, rsp = 0x7fffffffd1a8, rbp = 0x7fffffffd350 ---
Tracing command openvpn pid 24150 tid 100068 td 0xffffff0002a46460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_cv_timedwait_sig() at _cv_timedwait_sig+0x134
seltdwait() at seltdwait+0x98
poll() at poll+0x478
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (209, FreeBSD ELF64, poll), rip = 0x800ca0d6c, rsp = 0x7fffffffdf38, rbp = 0x7fffffffe030 ---
Tracing command choparp pid 13479 tid 100051 td 0xffffff000262b460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
kern_kevent() at kern_kevent+0x38b
kevent() at kevent+0x90
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (363, FreeBSD ELF64, kevent), rip = 0x800722c7c, rsp = 0x7fffffffeb28, rbp = 0x8 ---
Tracing command sshlockout_pf pid 12635 tid 100060 td 0xffffff00025a1460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_sleep() at _sleep+0x1b1
kern_nanosleep() at kern_nanosleep+0x118
nanosleep() at nanosleep+0x6e
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (240, FreeBSD ELF64, nanosleep), rip = 0x80083ac7c, rsp = 0x7fffffbfef58, rbp = 0x800a0ae48 ---
Tracing command sshlockout_pf pid 12635 tid 100054 td 0xffffff000262a460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
pipe_read() at pipe_read+0x3f4
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (3, FreeBSD ELF64, read), rip = 0x800855acc, rsp = 0x7fffffffe808, rbp = 0x7fffffffecd0 ---
Tracing command sshd pid 12135 tid 100056 td 0xffffff00026298c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x128
seltdwait() at seltdwait+0x110
kern_select() at kern_select+0x6ef
select() at select+0x5d
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (93, FreeBSD ELF64, select), rip = 0x8010b1a4c, rsp = 0x7fffffffdda8, rbp = 0x2 ---
Tracing command syslogd pid 8359 tid 100055 td 0xffffff000262a000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x128
seltdwait() at seltdwait+0x110
kern_select() at kern_select+0x6ef
select() at select+0x5d
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (93, FreeBSD ELF64, select), rip = 0x800dcca4c, rsp = 0x7fffffffe068, rbp = 0xf ---
Tracing command check_reload_status pid 286 tid 100047 td 0xffffff000259d8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
kern_kevent() at kern_kevent+0x38b
kevent() at kevent+0x90
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (363, FreeBSD ELF64, kevent), rip = 0x80083dc7c, rsp = 0x7fffffffeb08, rbp = 0x7fffffffec00 ---
Tracing command devd pid 285 tid 100049 td 0xffffff000259d000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x128
seltdwait() at seltdwait+0x110
kern_select() at kern_select+0x6ef
select() at select+0x5d
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (93, FreeBSD ELF64, select), rip = 0x440fcc, rsp = 0x7fffffffe878, rbp = 0x7fffffffe890 ---
Tracing command php pid 277 tid 100050 td 0xffffff000262b8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_cv_timedwait_sig() at _cv_timedwait_sig+0x134
seltdwait() at seltdwait+0x98
poll() at poll+0x478
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (209, FreeBSD ELF64, poll), rip = 0x80139cd6c, rsp = 0x7fffffffa3d8, rbp = 0x3e8 ---
Tracing command check_reload_status pid 273 tid 100045 td 0xffffff000259e460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
kern_kevent() at kern_kevent+0x38b
kevent() at kevent+0x90
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (363, FreeBSD ELF64, kevent), rip = 0x80083dc7c, rsp = 0x7fffffffea18, rbp = 0x800a1a000 ---
Tracing command md0 pid 73 tid 100046 td 0xffffff000259e000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
_sleep() at _sleep+0x317
md_kthread() at md_kthread+0x215
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000145d00, rbp = 0 ---
Tracing command sh pid 23 tid 100043 td 0xffffff00025a0000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
kern_wait() at kern_wait+0x6fd
wait4() at wait4+0x35
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (7, FreeBSD ELF64, wait4), rip = 0x800942dec, rsp = 0x7fffffffea58, rbp = 0x7fffffffec18 ---
Tracing command softdepflush pid 22 tid 100042 td 0xffffff00025a0460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_timedwait() at sleepq_timedwait+0x42
_sleep() at _sleep+0x301
softdep_flush() at softdep_flush+0x334
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000131d00, rbp = 0 ---
Tracing command vnlru pid 21 tid 100041 td 0xffffff00025a08c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_timedwait() at sleepq_timedwait+0x42
_sleep() at _sleep+0x301
vnlru_proc() at vnlru_proc+0x626
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800012cd00, rbp = 0 ---
Tracing command syncer pid 20 tid 100040 td 0xffffff00025a1000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_timedwait() at sleepq_timedwait+0x42
_cv_timedwait() at _cv_timedwait+0x134
sched_sync() at sched_sync+0x520
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000127d00, rbp = 0 ---
Tracing command bufdaemon pid 19 tid 100039 td 0xffffff000251d000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_timedwait() at sleepq_timedwait+0x42
_sleep() at _sleep+0x301
buf_daemon() at buf_daemon+0x182
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000122d00, rbp = 0 ---
Tracing command idlepoll pid 18 tid 100038 td 0xffffff000251d460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_timedwait() at sleepq_timedwait+0x42
_sleep() at _sleep+0x301
poll_idle() at poll_idle+0x1bb
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800011dd00, rbp = 0 ---
Tracing command pagezero pid 17 tid 100037 td 0xffffff000251d8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_timedwait() at sleepq_timedwait+0x42
_sleep() at _sleep+0x301
vm_pagezero() at vm_pagezero+0x83
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000118d00, rbp = 0 ---
Tracing command vmdaemon pid 16 tid 100036 td 0xffffff000251e000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
_sleep() at _sleep+0x317
vm_daemon() at vm_daemon+0x81
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000113d00, rbp = 0 ---
Tracing command pagedaemon pid 15 tid 100035 td 0xffffff000251e460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_timedwait() at sleepq_timedwait+0x42
_sleep() at _sleep+0x301
vm_pageout() at vm_pageout+0x90e
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800010ed00, rbp = 0 ---
Tracing command xpt_thrd pid 9 tid 100034 td 0xffffff000251e8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
_sleep() at _sleep+0x317
xpt_scanner_thread() at xpt_scanner_thread+0xfa
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000109d00, rbp = 0 ---
Tracing command pfpurge pid 8 tid 100033 td 0xffffff000251f000
pf_state_tree_lan_ext_RB_REMOVE_COLOR() at pf_state_tree_lan_ext_RB_REMOVE_COLOR+0xd4
pf_state_tree_lan_ext_RB_REMOVE() at pf_state_tree_lan_ext_RB_REMOVE+0xc6
pf_unlink_state() at pf_unlink_state+0x4f
pf_purge_expired_states() at pf_purge_expired_states+0xa3
pf_purge_thread() at pf_purge_thread+0xcb
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000104d00, rbp = 0 ---
Tracing command sctp_iterator pid 7 tid 100032 td 0xffffff000251f460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
_sleep() at _sleep+0x317
sctp_iterator_thread() at sctp_iterator_thread+0x41
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff80000ffd00, rbp = 0 ---
Tracing command yarrow pid 14 tid 100014 td 0xffffff00022cf460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_timedwait() at sleepq_timedwait+0x42
_sleep() at _sleep+0x301
random_kthread() at random_kthread+0x1e2
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800005bd00, rbp = 0 ---
Tracing command crypto returns pid 6 tid 100013 td 0xffffff00022cf8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
_sleep() at _sleep+0x317
crypto_ret_proc() at crypto_ret_proc+0x192
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000056d00, rbp = 0 ---
Tracing command crypto pid 5 tid 100012 td 0xffffff00022d0000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
_sleep() at _sleep+0x317
crypto_proc() at crypto_proc+0x197
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000051d00, rbp = 0 ---
Tracing command g_down pid 4 tid 100010 td 0xffffff00022d08c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
_sleep() at _sleep+0x317
g_io_schedule_down() at g_io_schedule_down+0x20e
g_down_procbody() at g_down_procbody+0x58
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000043d00, rbp = 0 ---
Tracing command g_up pid 3 tid 100009 td 0xffffff00022bf460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
_sleep() at _sleep+0x317
g_io_schedule_up() at g_io_schedule_up+0xd8
g_up_procbody() at g_up_procbody+0x58
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800003ed00, rbp = 0 ---
Tracing command g_event pid 2 tid 100008 td 0xffffff00022bf8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
_sleep() at _sleep+0x317
g_run_events() at g_run_events+0x430
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000039d00, rbp = 0 ---
Tracing command ng_queue pid 13 tid 100007 td 0xffffff00022cc000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
_sleep() at _sleep+0x317
ngthread() at ngthread+0x72
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000034d00, rbp = 0 ---
Tracing command intr pid 12 tid 100031 td 0xffffff000251f8c0
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100030 td 0xffffff000245b460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
ithread_loop() at ithread_loop+0x1d1
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff80000f1d00, rbp = 0 ---
Tracing command intr pid 12 tid 100027 td 0xffffff000245d460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
ithread_loop() at ithread_loop+0x1d1
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff80000e2d00, rbp = 0 ---
Tracing command intr pid 12 tid 100026 td 0xffffff000245d8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
ithread_loop() at ithread_loop+0x1d1
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff80000bad00, rbp = 0 ---
Tracing command intr pid 12 tid 100025 td 0xffffff000245e000
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100023 td 0xffffff000245e8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
ithread_loop() at ithread_loop+0x1d1
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000088d00, rbp = 0 ---
Tracing command intr pid 12 tid 100017 td 0xffffff00024598c0
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100016 td 0xffffff000245b000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
ithread_loop() at ithread_loop+0x1d1
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000065d00, rbp = 0 ---
Tracing command intr pid 12 tid 100015 td 0xffffff00022cf000
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100006 td 0xffffff00022cc460
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100005 td 0xffffff00022cc8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
ithread_loop() at ithread_loop+0x1d1
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800002ad00, rbp = 0 ---
Tracing command intr pid 12 tid 100004 td 0xffffff00022be000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
ithread_loop() at ithread_loop+0x1d1
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000025d00, rbp = 0 ---
Tracing command idle pid 11 tid 100003 td 0xffffff00022be460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
critical_exit() at critical_exit+0xa5
lapic_handle_timer() at lapic_handle_timer+0xe0
Xtimerint() at Xtimerint+0x8d
--- interrupt, rip = 0xffffffff80aae986, rsp = 0xffffff8000020b20, rbp = 0xffffff8000020b30 ---
acpi_cpu_c1() at acpi_cpu_c1+0x6
acpi_cpu_idle() at acpi_cpu_idle+0x20a
sched_idletd() at sched_idletd+0x11f
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000020d00, rbp = 0 ---
Tracing command init pid 1 tid 100002 td 0xffffff00022be8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_catch_signals() at sleepq_catch_signals+0x31c
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x269
kern_wait() at kern_wait+0x6fd
wait4() at wait4+0x35
amd64_syscall() at amd64_syscall+0x1f4
Xfast_syscall() at Xfast_syscall+0xfc
--- syscall (7, FreeBSD ELF64, wait4), rip = 0x412c6c, rsp = 0x7fffffffe758, rbp = 0x7fffffffe7bc ---
Tracing command audit pid 10 tid 100001 td 0xffffff00022bf000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
_cv_wait() at _cv_wait+0x129
audit_worker() at audit_worker+0x384
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000016d00, rbp = 0 ---
Tracing command kernel pid 0 tid 100029 td 0xffffff000245b8c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
msleep_spin() at msleep_spin+0x1a2
taskqueue_thread_loop() at taskqueue_thread_loop+0x71
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff80000ecd00, rbp = 0 ---
Tracing command kernel pid 0 tid 100028 td 0xffffff000245d000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
msleep_spin() at msleep_spin+0x1a2
taskqueue_thread_loop() at taskqueue_thread_loop+0x71
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff80000e7d00, rbp = 0 ---
Tracing command kernel pid 0 tid 100024 td 0xffffff000245e460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
_sleep() at _sleep+0x317
taskqueue_thread_loop() at taskqueue_thread_loop+0xbe
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800008dd00, rbp = 0 ---
Tracing command kernel pid 0 tid 100022 td 0xffffff0002458000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
_sleep() at _sleep+0x317
taskqueue_thread_loop() at taskqueue_thread_loop+0xbe
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000083d00, rbp = 0 ---
Tracing command kernel pid 0 tid 100021 td 0xffffff0002458460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
_sleep() at _sleep+0x317
taskqueue_thread_loop() at taskqueue_thread_loop+0xbe
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800007ed00, rbp = 0 ---
Tracing command kernel pid 0 tid 100020 td 0xffffff00024588c0
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
msleep_spin() at msleep_spin+0x1a2
taskqueue_thread_loop() at taskqueue_thread_loop+0x71
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000079d00, rbp = 0 ---
Tracing command kernel pid 0 tid 100019 td 0xffffff0002459000
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
msleep_spin() at msleep_spin+0x1a2
taskqueue_thread_loop() at taskqueue_thread_loop+0x71
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000074d00, rbp = 0 ---
Tracing command kernel pid 0 tid 100018 td 0xffffff0002459460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
msleep_spin() at msleep_spin+0x1a2
taskqueue_thread_loop() at taskqueue_thread_loop+0x71
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff800006fd00, rbp = 0 ---
Tracing command kernel pid 0 tid 100011 td 0xffffff00022d0460
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_wait() at sleepq_wait+0x42
_sleep() at _sleep+0x317
taskqueue_thread_loop() at taskqueue_thread_loop+0xbe
fork_exit() at fork_exit+0x11f
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffff8000048d00, rbp = 0 ---
Tracing command kernel pid 0 tid 100000 td 0xffffffff81385b30
sched_switch() at sched_switch+0x102
mi_switch() at mi_switch+0x176
sleepq_timedwait() at sleepq_timedwait+0x42
_sleep() at _sleep+0x301
scheduler() at scheduler+0x357
mi_startup() at mi_startup+0x77
btext() at btext+0x2c
db:0:kdb.enter.default> capture off
[...]
Fatal trap 12: page fault while in kernel mode
cpuid = 0; apic id = 00
fault virtual address = 0x68
fault code = supervisor read data, page not present
instruction pointer = 0x20:0xffffffff80210224
stack pointer = 0x28:0xffffff8000104ac0
frame pointer = 0x28:0xffffff8000104ad8
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 8 (pfpurge)
Updated by 
Updated by Ermal Luçi
Actions