#!/bin/sh
#
# dpinger_watchdog.sh - Detects hung/missing/zombie dpinger processes and restarts them
#
# Detection covers three failure modes:
#   1. Missing: dpinger process gone entirely (no socket, no proc)
#   2. Hung: socket exists but unresponsive (usocket_thread dead)
#   3. Zombie: socket responds but reports 100% loss while manual ping
#      from bind_addr to monitor_addr succeeds (send/recv threads orphaned
#      after ipsec interface destroy/recreate)
#
# Usage:
#   dpinger_watchdog.sh           # quiet (cron mode) - logs to syslog only
#   dpinger_watchdog.sh -v        # verbose - prints progress to stdout
#   When run on a TTY, verbose mode is enabled automatically.

SOCK_DIR="/var/run"
TIMEOUT_SEC=5
PING_COUNT=4
PING_WAIT=1
LOGCMD="logger -t dpinger_watchdog"

# Verbose if -v passed or stdout is a TTY
VERBOSE=0
if [ "$1" = "-v" ] || [ -t 1 ]; then
    VERBOSE=1
fi

say() {
    $LOGCMD "$1"
    [ $VERBOSE -eq 1 ] && echo "[$(date '+%H:%M:%S')] $1"
}

vsay() {
    [ $VERBOSE -eq 1 ] && echo "[$(date '+%H:%M:%S')] $1"
}

restart_needed=0
problem_gateways=""

vsay "=== dpinger watchdog start ==="

# --- Check 1: Missing dpinger processes ---
vsay "Check 1: querying pfSense for expected gateways..."
expected_gateways=$(/usr/local/bin/php -r '
require_once("config.inc");
require_once("gwlb.inc");
$gateways = get_gateways();
foreach ($gateways as $gw) {
    if (isset($gw["disabled"])) continue;
    if (isset($gw["monitor_disable"])) continue;
    echo $gw["name"] . "\n";
}
' 2>/dev/null)

vsay "Expected gateways: $(echo $expected_gateways | tr '\n' ' ')"

for gwname in $expected_gateways; do
    sock_match=$(ls ${SOCK_DIR}/dpinger_${gwname}~*.sock 2>/dev/null)
    if [ -z "$sock_match" ]; then
        if ! pgrep -f "dpinger.*-i ${gwname}" > /dev/null 2>&1; then
            restart_needed=1
            problem_gateways="${problem_gateways} ${gwname}(missing)"
            say "MISSING: dpinger process gone for gateway ${gwname}"
        else
            vsay "  ${gwname}: proc exists but socket missing (transient?)"
        fi
    else
        vsay "  ${gwname}: socket present"
    fi
done

# --- Checks 2 & 3: hung sockets and zombie processes ---
vsay "Checks 2/3: probing each socket and checking for zombies..."
for sock in ${SOCK_DIR}/dpinger_*.sock; do
    [ -e "$sock" ] || continue

    base=$(basename "$sock" .sock)
    gwname=$(echo "$base" | sed 's|^dpinger_||; s|~.*||')
    bind_addr=$(echo "$base" | awk -F~ '{print $2}')
    monitor_addr=$(echo "$base" | awk -F~ '{print $3}')

    result=$(timeout ${TIMEOUT_SEC} nc -U "$sock" < /dev/null 2>/dev/null)
    rc=$?

    # Check 2: hung socket
    if [ $rc -ne 0 ] || [ -z "$result" ]; then
        restart_needed=1
        problem_gateways="${problem_gateways} ${gwname}(hung)"
        say "HUNG: socket unresponsive for ${gwname} (rc=${rc})"
        continue
    fi

    latency=$(echo "$result" | awk '{print $2}')
    stddev=$(echo "$result" | awk '{print $3}')
    loss=$(echo "$result" | awk '{print $4}')
    vsay "  ${gwname} (bind=${bind_addr} mon=${monitor_addr}): latency=${latency}us stddev=${stddev}us loss=${loss}%"

    # Check 3: zombie state
    if [ "$loss" = "100" ]; then
        case "$bind_addr" in
            *:*) PING=ping6 ;;
            *)   PING=ping ;;
        esac
        vsay "    100% loss reported - probing manually with $PING -c $PING_COUNT -S $bind_addr $monitor_addr"
        ping_out=$(timeout 8 $PING -c $PING_COUNT -W $((PING_WAIT * 1000)) -S "$bind_addr" "$monitor_addr" 2>/dev/null)
        replies=$(echo "$ping_out" | awk -F, '/packets received/ {gsub(/ /, "", $2); print $2}' | grep -oE '^[0-9]+')
        replies=${replies:-0}
        vsay "    probe result: ${replies}/${PING_COUNT} replies"
        if [ "$replies" -gt 0 ]; then
            restart_needed=1
            problem_gateways="${problem_gateways} ${gwname}(zombie)"
            say "ZOMBIE: ${gwname} reports 100% loss but ${replies}/${PING_COUNT} pings succeeded from ${bind_addr} to ${monitor_addr}"
        else
            vsay "    real outage (probe also fails) - skipping"
        fi
    fi
done

# --- Restart if needed ---
if [ $restart_needed -eq 1 ]; then
    say "Restarting gateway monitoring for:${problem_gateways}"
    /usr/local/bin/php -r '
        require_once("config.inc");
        require_once("gwlb.inc");
        setup_gateways_monitor();
    ' 2>&1
    rc=$?
    if [ $rc -eq 0 ]; then
        say "Gateway monitoring restarted successfully"
    else
        say "ERROR: failed to restart gateway monitoring (rc=${rc})"
    fi
else
    vsay "All gateways healthy - no action"
fi

vsay "=== done ==="