pfSense

Seeing this panic on a main snapshot that has PPP WANs:

panic: malloc(M_WAITOK) with sleeping prohibited
cpuid = 1
time = 1662055300
KDB: enter: panic
[ thread pid 13 tid 100034 ]
Stopped at      kdb_enter+0x32: movq    $0,0x27d1f73(%rip)
db:0:kdb.enter.default> textdump set
textdump set
db:0:kdb.enter.default>  capture on
db:0:kdb.enter.default>  show registers
cs                        0x20
ds                        0x3b
es                        0x3b
fs                        0x13
gs                        0x1b
ss                        0x28
rax                       0x12
rcx                        0x1
rdx         0xffffffff8154c026
rbx                      0x100
rsp         0xfffffe0046ffec40
rbp         0xfffffe0046ffec40
rsi                          0
rdi         0xffffffff8354cac0  cnputs_mtx
r8                           0
r9                      0x304f  _binary_elf_vdso_so_1_size+0x2a3f
r10                          0
r11                          0
r12                     0x8800  _binary_elf_vdso_so_1_size+0x81f0
r13         0xfffff8000587f800
r14         0xfffffe0046ffecd0
r15         0xfffffe0001d1a3a0
rip         0xffffffff80dc5d12  kdb_enter+0x32
rflags                    0x86
kdb_enter+0x32: movq    $0,0x27d1f73(%rip)
db:0:kdb.enter.default>  run lockinfo
db:1:lockinfo> show locks
db:1:lockinfo>  show alllocks
Process 78776 (sh) thread 0xfffffe0089fa21e0 (100247)
exclusive rw vm object (vm object) r = 0 (0xfffff80029b82210) locked @ /var/jenkins/workspace/pfSense-CE-snapshots-master-main/sources/FreeBSD-src-devel-main/sys/vm/vm_object.c:1458
exclusive sx vm map (user) (vm map (user)) r = 0 (0xfffffe0001c91460) locked @ /var/jenkins/workspace/pfSense-CE-snapshots-master-main/sources/FreeBSD-src-devel-main/sys/vm/vm_map.c:4930
Process 78619 (sh) thread 0xfffffe0089fa2900 (100246)
exclusive rw vm object (vm object) r = 0 (0xfffff8002945e528) locked @ /var/jenkins/workspace/pfSense-CE-snapshots-master-main/sources/FreeBSD-src-devel-main/sys/vm/vm_fault.c:1081
shared sx vm map (user) (vm map (user)) r = 0 (0xfffffe008c1c8460) locked @ /var/jenkins/workspace/pfSense-CE-snapshots-master-main/sources/FreeBSD-src-devel-main/sys/vm/vm_map.c:4930
Process 78030 (mpd5) thread 0xfffffe008aba3720 (100270)
exclusive sx so_snd_sx (so_snd_sx) r = 0 (0xfffff8000d449140) locked @ /var/jenkins/workspace/pfSense-CE-snapshots-master-main/sources/FreeBSD-src-devel-main/sys/kern/uipc_socket.c:3958
Process 55235 (dpinger) thread 0xfffffe008a8be720 (100262)
exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xfffff8000d2bc900) locked @ /var/jenkins/workspace/pfSense-CE-snapshots-master-main/sources/FreeBSD-src-devel-main/sys/kern/uipc_socket.c:3958
Process 54824 (dpinger) thread 0xfffffe008a8c03a0 (100258)
exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xfffff8000d2bb540) locked @ /var/jenkins/workspace/pfSense-CE-snapshots-master-main/sources/FreeBSD-src-devel-main/sys/kern/uipc_socket.c:3958
Process 54510 (dpinger) thread 0xfffffe0089fa3020 (100254)
exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xfffff8000d444900) locked @ /var/jenkins/workspace/pfSense-CE-snapshots-master-main/sources/FreeBSD-src-devel-main/sys/kern/uipc_socket.c:3958
Process 7148 (syslogd) thread 0xfffffe0088f81ac0 (100210)
shared lockmgr zfs (zfs) r = 0 (0xfffff8000d0b95b0) locked @ /var/jenkins/workspace/pfSense-CE-snapshots-master-main/sources/FreeBSD-src-devel-main/sys/kern/vfs_vnops.c:1166
db:1:lockinfo>  show lockedvnods
Locked vnodes
vnode 0xfffff8000d0b9540: type VREG
    usecount 1, writecount 1, refcount 1 seqc users 0
    hold count flags ()
    flags ()
    v_object 0xfffff800276e5840 ref 0 pages 0 cleanbuf 0 dirtybuf 0
    lock type zfs: SHARED (count 1)
db:0:kdb.enter.default>  show pcpu
cpuid        = 1
dynamic pcpu = 0xfffffe007e965d40
curthread    = 0xfffffe0001d1a3a0: pid 13 tid 100034 critnest 1 "ng_queue1" 
curpcb       = 0xfffffe0001d1a8c0
fpcurthread  = none
idlethread   = 0xfffffe0001c8ac80: tid 100004 "idle: cpu1" 
self         = 0xffffffff84811000
curpmap      = 0xffffffff8354bd70
tssp         = 0xffffffff84811384
rsp0         = 0xfffffe0046fff000
kcr3         = 0x46d7000
ucr3         = 0xffffffffffffffff
scr3         = 0x29ab1000
gs32p        = 0xffffffff84811404
ldt          = 0xffffffff84811444
tss          = 0xffffffff84811434
curvnet      = 0xfffff80005184e80
spin locks held:
db:0:kdb.enter.default>  bt
Tracing pid 13 tid 100034 td 0xfffffe0001d1a3a0
kdb_enter() at kdb_enter+0x32/frame 0xfffffe0046ffec40
vpanic() at vpanic+0x182/frame 0xfffffe0046ffec90
panic() at panic+0x43/frame 0xfffffe0046ffecf0
malloc_dbg() at malloc_dbg+0xd4/frame 0xfffffe0046ffed10
malloc() at malloc+0x2d/frame 0xfffffe0046ffed50
in_getsockaddr() at in_getsockaddr+0x6a/frame 0xfffffe0046ffed80
ng_ksocket_rcvmsg() at ng_ksocket_rcvmsg+0x345/frame 0xfffffe0046ffedf0
ng_apply_item() at ng_apply_item+0x3be/frame 0xfffffe0046ffee80
ngthread() at ngthread+0x200/frame 0xfffffe0046ffeef0
fork_exit() at fork_exit+0x80/frame 0xfffffe0046ffef30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0046ffef30
--- trap 0xdeadc0de, rip = 0xdeadc0dedeadc0de, rsp = 0xdeadc0dedeadc0de, rbp = 0xdeadc0dedeadc0de ---

The system panics and reboots before it can complete the boot process, resulting in a loop.

Textdumps are at https://netgate.slack.com/archives/C4GUL8CKF/p1662056026435759