#!/usr/bin/env bash
# wml-wan-monitor — WAN health + cable-swap auto re-detect.
#
# - Pings 1.1.1.1 + 8.8.8.8 via each WAN every 2s.
# - Maintains ECMP default route across healthy WANs.
# - GRACE PERIOD: no failure verdict in first 60s after start (services warming up).
# - SWAP DETECT: every 30s, also re-probe each WAN's DHCP lease. If the lease lands
#   inside our LAN subnet, the user swapped cables — trigger a full re-detect.
# - If WAN has been FAILED for >120s consecutively, also trigger full re-detect
#   (cable physically disconnected, plugged into a non-upstream port, etc.)

set -u

LOG=/var/log/wml/wan-monitor.log
mkdir -p /var/log/wml
PROBE_HOSTS=("1.1.1.1" "8.8.8.8")
INTERVAL=2
FAIL_THRESHOLD=3
OK_THRESHOLD=2
GRACE_SECONDS=60
SWAP_CHECK_EVERY=5           # seconds between active swap probes
FAIL_REDETECT_AFTER=120       # seconds of continuous fail before triggering re-detect

log() { echo "$(date -Is) $*" >> "$LOG"; }
log "wml-wan-monitor started (grace=${GRACE_SECONDS}s, swap-check=${SWAP_CHECK_EVERY}s)"

declare -A FAIL_COUNT
declare -A OK_COUNT
declare -A HEALTHY
declare -A FAIL_SINCE      # epoch when iface first went bad (0 if healthy)
START_EPOCH=$(date +%s)
LAST_SWAP_CHECK=$START_EPOCH

probe_wan() {
    local iface="$1"
    for host in "${PROBE_HOSTS[@]}"; do
        if ping -c 1 -W 2 -I "$iface" "$host" >/dev/null 2>&1; then
            return 0
        fi
    done
    return 1
}

read_wans() {
    [[ -f /etc/wml/interfaces.conf ]] || { echo ""; return; }
    awk -F= '/^WAN_IFS=/{gsub(/"/,"",$2); print $2}' /etc/wml/interfaces.conf
}

read_lan_family() {
    [[ -f /etc/wml/interfaces.conf ]] || { echo ""; return; }
    awk -F= '/^LAN_NETWORK=/{gsub(/"/,"",$2); print $2}' /etc/wml/interfaces.conf | cut -d. -f1-2
}

get_gateway() {
    local iface="$1"
    ip -4 route show dev "$iface" default 2>/dev/null | awk '{print $3; exit}'
}

current_iface_ip() {
    local iface="$1"
    ip -o -4 addr show "$iface" 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -1
}

rebuild_default_route() {
    local healthy_wans=()
    for w in "${WANS[@]}"; do
        [[ "${HEALTHY[$w]:-0}" == "1" ]] && healthy_wans+=("$w")
    done

    if [[ ${#healthy_wans[@]} -eq 0 ]]; then
        log "All WANs down - leaving existing default route alone"
        return
    fi

    local nh_args=()
    for w in "${healthy_wans[@]}"; do
        gw=$(get_gateway "$w")
        if [[ -n "$gw" ]]; then
            nh_args+=("nexthop" "via" "$gw" "dev" "$w" "weight" "1")
        fi
    done

    if [[ ${#nh_args[@]} -eq 0 ]]; then
        log "No gateways resolvable for healthy WANs - skipping route update"
        return
    fi

    if [[ ${#healthy_wans[@]} -eq 1 ]]; then
        local w="${healthy_wans[0]}"
        local gw
        gw=$(get_gateway "$w")
        ip route replace default via "$gw" dev "$w" 2>/dev/null \
            && log "Default route -> single WAN $w via $gw"
    else
        ip route replace default "${nh_args[@]}" 2>/dev/null \
            && log "Default route -> ECMP across ${healthy_wans[*]}"
    fi
}

# Trigger a full WAN/LAN re-detect by re-running wml-firstboot logic.
# We do this by clearing the firstboot marker + re-launching the firstboot
# script synchronously so it rewrites interfaces.conf, networkd, kea, NAT.
trigger_full_redetect() {
    local reason="$1"
    log "TRIGGER FULL REDETECT: $reason"
    # Bug 32: release ALL dhcp leases and flush ALL IPs on every NIC first.
    # Otherwise networkd keeps the stale lease while wml-firstboot re-probes,
    # leading to "both NICs show 172.30.30.x" flicker for 30-90s.
    for d in /sys/class/net/*; do
        n=$(basename "$d")
        case "$n" in lo|docker*|virbr*|veth*|tap*|wg*|br-*|tun*|tailscale*) continue ;; esac
        [[ -e "$d/device" ]] || continue
        dhclient -r "$n" 2>/dev/null &
        ip -4 addr flush dev "$n" 2>/dev/null
    done
    wait
    # Remove the marker so wml-firstboot can run again
    rm -f /var/lib/wml/firstboot-done 2>/dev/null || true
    if [[ -x /usr/local/sbin/wml-firstboot ]]; then
        /usr/local/sbin/wml-firstboot >> "$LOG" 2>&1 || log "wml-firstboot returned non-zero"
        log "Full re-detect finished, re-reading WANs"
        # Re-initialize our state from the new interfaces.conf
        WANS=()
        read -ra WANS < <(read_wans)
        for w in "${WANS[@]}"; do
            HEALTHY[$w]=1
            FAIL_COUNT[$w]=0
            OK_COUNT[$w]=0
            FAIL_SINCE[$w]=0
        done
        log "After re-detect: monitoring WANs=[${WANS[*]:-(none)}]"
    else
        log "ERROR: /usr/local/sbin/wml-firstboot not executable - cannot re-detect"
    fi
}

# Active swap-check: probe every non-WAN NIC for an UPSTREAM DHCP lease.
# If a LAN-side NIC gets a lease OUTSIDE our LAN family, the cable was
# physically swapped onto an upstream link -> trigger full re-detect.
# Also check if any WAN currently shows a LAN-family IP (rare; networkd
# usually holds old lease until expiry, but defensive).
check_for_swap() {
    local lan_family
    lan_family=$(read_lan_family)
    [[ -z "$lan_family" ]] && return

    # Passive check on current WANs
    for w in "${WANS[@]}"; do
        local cur_ip
        cur_ip=$(current_iface_ip "$w")
        if [[ -n "$cur_ip" && "$cur_ip" == ${lan_family}.* ]]; then
            log "SWAP DETECTED: WAN $w now has LAN-family IP $cur_ip"
            trigger_full_redetect "WAN $w IP $cur_ip is in LAN family $lan_family"
            return
        fi
    done

    # NOTE: We intentionally do NOT actively probe LAN-side NICs with dhclient
    # here — that would disrupt Kea's DHCP server. Cable-swap detection happens
    # via:
    #   1. The passive WAN-IP check above (rare but covered)
    #   2. The FAIL_REDETECT_AFTER trigger in the main loop (if all WANs go
    #      persistently dead, we re-run wml-firstboot which probes everything)
    #   3. The wml-carrier-watcher service (handles link-down -> link-up events)
    #   4. Manual "Re-detect interfaces" option in the console menu (option 14)
}

# Initial population
WANS=()
read -ra WANS < <(read_wans)
for w in "${WANS[@]}"; do
    HEALTHY[$w]=1
    FAIL_COUNT[$w]=0
    OK_COUNT[$w]=0
    FAIL_SINCE[$w]=0
done
log "Monitoring WANs: ${WANS[*]:-(none)}"

while true; do
    now=$(date +%s)
    in_grace=0
    if (( now - START_EPOCH < GRACE_SECONDS )); then
        in_grace=1
    fi

    # Re-read WAN list each cycle
    NEW_WANS=()
    read -ra NEW_WANS < <(read_wans)
    if [[ "${NEW_WANS[*]}" != "${WANS[*]}" ]]; then
        log "WAN list changed: [${WANS[*]}] -> [${NEW_WANS[*]}]"
        WANS=("${NEW_WANS[@]}")
        for w in "${WANS[@]}"; do
            [[ -z "${HEALTHY[$w]:-}" ]] && HEALTHY[$w]=1
            [[ -z "${FAIL_COUNT[$w]:-}" ]] && FAIL_COUNT[$w]=0
            [[ -z "${OK_COUNT[$w]:-}" ]] && OK_COUNT[$w]=0
            [[ -z "${FAIL_SINCE[$w]:-}" ]] && FAIL_SINCE[$w]=0
        done
    fi

    for w in "${WANS[@]}"; do
        if probe_wan "$w"; then
            FAIL_COUNT[$w]=0
            FAIL_SINCE[$w]=0
            OK_COUNT[$w]=$((OK_COUNT[$w]+1))
            if [[ "${HEALTHY[$w]}" == "0" && "${OK_COUNT[$w]}" -ge $OK_THRESHOLD ]]; then
                HEALTHY[$w]=1
                log "WAN $w RECOVERED"
                rebuild_default_route
            fi
        else
            OK_COUNT[$w]=0
            FAIL_COUNT[$w]=$((FAIL_COUNT[$w]+1))
            [[ "${FAIL_SINCE[$w]}" == "0" ]] && FAIL_SINCE[$w]=$now

            # In grace period, don't declare failure yet
            if (( in_grace )); then
                continue
            fi

            if [[ "${HEALTHY[$w]}" == "1" && "${FAIL_COUNT[$w]}" -ge $FAIL_THRESHOLD ]]; then
                HEALTHY[$w]=0
                log "WAN $w FAILED (no response from ${PROBE_HOSTS[*]})"
                rebuild_default_route
            fi

            # If WAN has been FAILED continuously > FAIL_REDETECT_AFTER, run re-detect
            if [[ "${HEALTHY[$w]}" == "0" ]]; then
                fail_age=$(( now - ${FAIL_SINCE[$w]} ))
                if (( fail_age >= FAIL_REDETECT_AFTER )); then
                    log "WAN $w failed for ${fail_age}s - triggering re-detect"
                    trigger_full_redetect "WAN $w persistently failed"
                    LAST_SWAP_CHECK=$(date +%s)
                fi
            fi
        fi
    done

    # Periodic active swap check (independent of ping success)
    if (( now - LAST_SWAP_CHECK >= SWAP_CHECK_EVERY )); then
        LAST_SWAP_CHECK=$now
        check_for_swap
    fi

    sleep $INTERVAL
done
