wan-watchdog: self-heal router WAN on ISP lease flap / IP change

Flushes stale NAT conntrack on WAN IP change; renews lease then restarts
networkd if internet stays down. ntfy on action. Fixes needing a manual
reboot after the ISP re-IP'd us.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

common.nix

@@ -45,6 +45,7 @@
     ./services/code-server.nix
     ./services/memos.nix
     ./services/hardware-health.nix
+    ./services/wan-watchdog.nix
   ];
 
   ### Make build time quicker

services/wan-watchdog.nix

@@ -0,0 +1,97 @@
+# services/wan-watchdog.nix — self-heal the router's WAN when the ISP flaps.
+#
+# Jul 2026: the ISP migrated us to a different gateway/subnet; eno1's DHCP
+# lease flapped and the LAN lost internet until a manual reboot. Root cause of
+# the "needs a reboot" part: after the WAN IP changed, the kernel NAT
+# conntrack table still mapped LAN flows to the OLD WAN IP, so masquerade kept
+# using stale info until entries aged out. A reboot flushed it.
+#
+# This watchdog (30s timer) fixes that without a reboot:
+#   1. WAN IP changed since last run  -> flush conntrack (clears stale NAT).
+#   2. Internet unreachable via eno1:
+#        ~90s down  -> `networkctl renew eno1` + flush conntrack
+#        ~4min down -> restart systemd-networkd + flush conntrack
+#      Escalates once at each step, then stops (a real ISP outage can't be
+#      fixed by thrashing the link) until connectivity returns.
+#   3. ntfy alert (reuses /var/secrets/ntfy-url) on each action + on recovery.
+
+{ config, lib, pkgs, ... }:
+let
+  # Ping out the WAN specifically (source-bound) so a healthy LAN doesn't read
+  # as "up". Two anycast targets so one being down isn't a false alarm.
+  watchdog = pkgs.writeShellScript "wan-watchdog" ''
+    set -uo pipefail
+    wan=eno1
+    state=/var/lib/wan-watchdog
+    ipfile=$state/wan-ip
+    failfile=$state/failcount
+    secret=/var/secrets/ntfy-url
+
+    ping=${pkgs.iputils}/bin/ping
+    ip=${pkgs.iproute2}/bin/ip
+    conntrack=${pkgs.conntrack-tools}/bin/conntrack
+    networkctl=${pkgs.systemd}/bin/networkctl
+    systemctl=${pkgs.systemd}/bin/systemctl
+
+    notify() { # body
+      [ -f "$secret" ] || return 0
+      url=$(${pkgs.coreutils}/bin/tr -d '\n' < "$secret")
+      ${pkgs.curl}/bin/curl -fsS --max-time 10 \
+        -H "Title: WAN watchdog" -H "Priority: high" -H "Tags: satellite_antenna" \
+        -d "$1" "$url" >/dev/null 2>&1 || true
+    }
+
+    # --- 1. Flush stale NAT when the WAN IP changes ---
+    cur_ip=$($ip -4 -o addr show "$wan" 2>/dev/null | ${pkgs.gawk}/bin/awk '{print $4}' | ${pkgs.coreutils}/bin/head -1)
+    old_ip=$(${pkgs.coreutils}/bin/cat "$ipfile" 2>/dev/null || true)
+    if [ -n "$cur_ip" ] && [ "$cur_ip" != "$old_ip" ]; then
+      $conntrack -F >/dev/null 2>&1 || true
+      ${pkgs.coreutils}/bin/printf '%s' "$cur_ip" > "$ipfile"
+      # Only page on a real change, not the first-ever run.
+      [ -n "$old_ip" ] && notify "WAN IP changed $old_ip -> $cur_ip; flushed NAT conntrack."
+    fi
+
+    # --- 2. Connectivity check via the WAN ---
+    if $ping -c1 -W3 -I "$wan" 1.1.1.1 >/dev/null 2>&1 \
+       || $ping -c1 -W3 -I "$wan" 8.8.8.8 >/dev/null 2>&1; then
+      prev=$(${pkgs.coreutils}/bin/cat "$failfile" 2>/dev/null || echo 0)
+      [ "$prev" -ge 3 ] && notify "WAN back up (was down ~$(( prev * 30 ))s)."
+      echo 0 > "$failfile"
+      exit 0
+    fi
+
+    # --- Down: escalate on consecutive failures (timer fires every 30s) ---
+    fails=$(( $(${pkgs.coreutils}/bin/cat "$failfile" 2>/dev/null || echo 0) + 1 ))
+    echo "$fails" > "$failfile"
+    case "$fails" in
+      3)  # ~90s down: renew the lease
+        $networkctl renew "$wan" >/dev/null 2>&1 || true
+        $conntrack -F >/dev/null 2>&1 || true
+        notify "WAN unreachable ~90s: renewed $wan DHCP lease + flushed conntrack." ;;
+      8)  # ~4min down: full network stack restart
+        $systemctl restart systemd-networkd >/dev/null 2>&1 || true
+        $conntrack -F >/dev/null 2>&1 || true
+        notify "WAN still down ~4min: restarted systemd-networkd." ;;
+    esac
+  '';
+in
+{
+  config = lib.mkIf (config.networking.hostName == "FredOS-Mediaserver") {
+
+    systemd.services.wan-watchdog = {
+      description = "Self-heal WAN on ISP lease flap / IP change";
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = watchdog;
+        StateDirectory = "wan-watchdog";
+      };
+    };
+    systemd.timers.wan-watchdog = {
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnBootSec = "1min";        # let the network settle after boot first
+        OnUnitActiveSec = "30s";
+      };
+    };
+  };
+}