rope/nixos
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Harden deployment, authentication, firewall and media storage failure paths

Browse source
This commit is contained in:
rope 2026-09-14 11:50:22 +01:00
parent 64ede7bd05
commit 7145a35927
35 changed files with 2529 additions and 499 deletions
Download patch file Download diff file Expand all files Collapse all files

64
services/router-ui.nix
View file

@ -6,15 +6,8 @@
# deploy that exact commit. services/router.nix reads those same files, so the
# UI is a constrained editor for config that is still fully reviewable in git.
#
# Apply is deliberately paranoid, because a bad generation here takes the
# household's internet with it:
#
# nixos-rebuild test -> eval/build failure aborts before anything activates
# wait, then health-check WAN ping + dnsmasq + LAN address + NAT table
# healthy -> nixos-rebuild switch (only now does it become the boot default)
# unhealthy -> switch-to-configuration test on the old profile, + ntfy alert
#
# `test` never touches the boot default, so a reboot is always an escape hatch.
# Apply delegates to the shared deployment transaction in deployment.nix.
# Builds precede activation; an independent timer recovers partial failures.
#
# Requires a Forgejo token with write:repository on rope/nixos:
# printf '%s' '<token>' | sudo tee /var/secrets/forgejo-router-token
@ -34,7 +27,7 @@ let
sudo = "/run/wrappers/bin/sudo";
applyScript = pkgs.writeShellScript "router-apply" ''
set -uo pipefail
set -euo pipefail
rev="$1"
flake="${flakeUrl}?rev=$rev"
secret=/var/secrets/ntfy-url
@ -47,45 +40,16 @@ let
-d "$1" "$url" >/dev/null 2>&1 || true
}
old=$(${pkgs.coreutils}/bin/readlink /run/current-system)
echo "==> building + activating $rev (test: boot default untouched)"
if ! nixos-rebuild test --refresh --flake "$flake" -L; then
echo "!! eval or build failed — nothing was activated, old config still running"
notify "Router apply ''${rev:0:8}: build failed. Nothing changed."
if [[ ! "$rev" =~ ^[0-9a-f]{40}$ ]]; then
echo "Invalid commit revision" >&2
exit 1
fi
echo "==> activated, waiting 45s for the network to settle"
sleep 45
healthy=1
${pkgs.iputils}/bin/ping -c1 -W3 -I eno1 1.1.1.1 >/dev/null 2>&1 \
|| ${pkgs.iputils}/bin/ping -c1 -W3 -I eno1 8.8.8.8 >/dev/null 2>&1 \
|| { echo "!! no WAN"; healthy=0; }
${systemctl} is-active --quiet dnsmasq || { echo "!! dnsmasq down"; healthy=0; }
${pkgs.iproute2}/bin/ip -4 addr show eth0 \
| ${pkgs.gnugrep}/bin/grep -q '10\.0\.0\.1' || { echo "!! LAN address gone"; healthy=0; }
${pkgs.nftables}/bin/nft list table ip router-nat >/dev/null 2>&1 \
|| { echo "!! NAT table missing"; healthy=0; }
if [ "$healthy" = 1 ]; then
echo "==> healthy — making $rev the boot default"
if nixos-rebuild switch --refresh --flake "$flake" -L; then
${pkgs.coreutils}/bin/printf '%s' "$rev" > ${stateDir}/last-rev
${pkgs.coreutils}/bin/chown router-ui ${stateDir}/last-rev || true
record-update "$old" /run/current-system || true
notify "Router applied ''${rev:0:8}."
echo "==> done"
else
echo "!! switch failed after a healthy test — running config is fine, boot default is not"
notify "Router apply ''${rev:0:8}: switch failed after a healthy test. Check the box."
exit 1
fi
if ${config.system.build.deploy}/bin/fred-deploy --flake "$flake"; then
${pkgs.coreutils}/bin/printf '%s' "$rev" > ${stateDir}/last-rev
${pkgs.coreutils}/bin/chown router-ui ${stateDir}/last-rev
notify "Router applied ''${rev:0:8}."
else
echo "!! health check failed — reverting to the previous generation"
/nix/var/nix/profiles/system/bin/switch-to-configuration test
notify "Router apply ''${rev:0:8} FAILED health check. Rolled back."
notify "Router apply ''${rev:0:8} failed. Check the apply journal and /var/lib/fred-deploy/last-result.json for recovery status."
exit 1
fi
'';
@ -160,13 +124,15 @@ in
# Deploy one specific commit. Instance name is the git rev.
systemd.services."router-apply@" = {
description = "Deploy router config %i (test, health check, then switch)";
path = [ pkgs.nixos-rebuild pkgs.nix pkgs.systemd "/run/current-system/sw" ];
description = "Deploy router config %i through the shared transaction";
restartIfChanged = false;
unitConfig.X-StopOnRemoval = false;
path = [ pkgs.systemd ];
environment.HOME = "/root";
serviceConfig = {
Type = "oneshot";
ExecStart = "${applyScript} %i";
TimeoutStartSec = "90min";
TimeoutStartSec = "105min";
};
};