rope/nixos
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

wan-watchdog: self-heal router WAN on ISP lease flap / IP change

Browse source 
Flushes stale NAT conntrack on WAN IP change; renews lease then restarts
networkd if internet stays down. ntfy on action. Fixes needing a manual
reboot after the ISP re-IP'd us.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
rope 2026-07-31 13:58:07 +01:00
parent dd5040ca79
commit 3fbc9f884b
2 changed files with 98 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

97
services/wan-watchdog.nix Normal file
View file

@ -0,0 +1,97 @@
# services/wan-watchdog.nix — self-heal the router's WAN when the ISP flaps.
#
# Jul 2026: the ISP migrated us to a different gateway/subnet; eno1's DHCP
# lease flapped and the LAN lost internet until a manual reboot. Root cause of
# the "needs a reboot" part: after the WAN IP changed, the kernel NAT
# conntrack table still mapped LAN flows to the OLD WAN IP, so masquerade kept
# using stale info until entries aged out. A reboot flushed it.
#
# This watchdog (30s timer) fixes that without a reboot:
# 1. WAN IP changed since last run -> flush conntrack (clears stale NAT).
# 2. Internet unreachable via eno1:
# ~90s down -> `networkctl renew eno1` + flush conntrack
# ~4min down -> restart systemd-networkd + flush conntrack
# Escalates once at each step, then stops (a real ISP outage can't be
# fixed by thrashing the link) until connectivity returns.
# 3. ntfy alert (reuses /var/secrets/ntfy-url) on each action + on recovery.
{ config, lib, pkgs, ... }:
let
# Ping out the WAN specifically (source-bound) so a healthy LAN doesn't read
# as "up". Two anycast targets so one being down isn't a false alarm.
watchdog = pkgs.writeShellScript "wan-watchdog" ''
set -uo pipefail
wan=eno1
state=/var/lib/wan-watchdog
ipfile=$state/wan-ip
failfile=$state/failcount
secret=/var/secrets/ntfy-url
ping=${pkgs.iputils}/bin/ping
ip=${pkgs.iproute2}/bin/ip
conntrack=${pkgs.conntrack-tools}/bin/conntrack
networkctl=${pkgs.systemd}/bin/networkctl
systemctl=${pkgs.systemd}/bin/systemctl
notify() { # body
[ -f "$secret" ] || return 0
url=$(${pkgs.coreutils}/bin/tr -d '\n' < "$secret")
${pkgs.curl}/bin/curl -fsS --max-time 10 \
-H "Title: WAN watchdog" -H "Priority: high" -H "Tags: satellite_antenna" \
-d "$1" "$url" >/dev/null 2>&1 || true
}
# --- 1. Flush stale NAT when the WAN IP changes ---
cur_ip=$($ip -4 -o addr show "$wan" 2>/dev/null | ${pkgs.gawk}/bin/awk '{print $4}' | ${pkgs.coreutils}/bin/head -1)
old_ip=$(${pkgs.coreutils}/bin/cat "$ipfile" 2>/dev/null || true)
if [ -n "$cur_ip" ] && [ "$cur_ip" != "$old_ip" ]; then
$conntrack -F >/dev/null 2>&1 || true
${pkgs.coreutils}/bin/printf '%s' "$cur_ip" > "$ipfile"
# Only page on a real change, not the first-ever run.
[ -n "$old_ip" ] && notify "WAN IP changed $old_ip -> $cur_ip; flushed NAT conntrack."
fi
# --- 2. Connectivity check via the WAN ---
if $ping -c1 -W3 -I "$wan" 1.1.1.1 >/dev/null 2>&1 \
|| $ping -c1 -W3 -I "$wan" 8.8.8.8 >/dev/null 2>&1; then
prev=$(${pkgs.coreutils}/bin/cat "$failfile" 2>/dev/null || echo 0)
[ "$prev" -ge 3 ] && notify "WAN back up (was down ~$(( prev * 30 ))s)."
echo 0 > "$failfile"
exit 0
fi
# --- Down: escalate on consecutive failures (timer fires every 30s) ---
fails=$(( $(${pkgs.coreutils}/bin/cat "$failfile" 2>/dev/null || echo 0) + 1 ))
echo "$fails" > "$failfile"
case "$fails" in
3) # ~90s down: renew the lease
$networkctl renew "$wan" >/dev/null 2>&1 || true
$conntrack -F >/dev/null 2>&1 || true
notify "WAN unreachable ~90s: renewed $wan DHCP lease + flushed conntrack." ;;
8) # ~4min down: full network stack restart
$systemctl restart systemd-networkd >/dev/null 2>&1 || true
$conntrack -F >/dev/null 2>&1 || true
notify "WAN still down ~4min: restarted systemd-networkd." ;;
esac
'';
in
{
config = lib.mkIf (config.networking.hostName == "FredOS-Mediaserver") {
systemd.services.wan-watchdog = {
description = "Self-heal WAN on ISP lease flap / IP change";
serviceConfig = {
Type = "oneshot";
ExecStart = watchdog;
StateDirectory = "wan-watchdog";
};
};
systemd.timers.wan-watchdog = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "1min"; # let the network settle after boot first
OnUnitActiveSec = "30s";
};
};
};
}