Harden deployment, authentication, firewall and media storage failure paths

scripts/deploy-check.py

@@ -0,0 +1,486 @@
+"""Fault-injection checks: python3 scripts/deploy-check.py.
+
+All system/profile/bootloader operations are replaced with filesystem fixtures.
+No Nix build, systemd unit, network request or production activation is performed.
+"""
+
+import contextlib
+import importlib.util
+import io
+import json
+import os
+import signal
+import subprocess
+import sys
+from pathlib import Path
+import tempfile
+import time
+import unittest
+from unittest.mock import patch, MagicMock
+
+spec = importlib.util.spec_from_file_location(
+    "deploy", Path(__file__).with_name("deploy.py")
+)
+m = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(m)
+
+
+class Fixture(m.Deployment):
+    def __init__(self, root):
+        self.calls = []
+        self.fail = set()
+        self.health_calls = 0
+        self.root = root
+        self.candidate = root / "candidate"
+        self.old_running = root / "running"
+        self.old_boot = root / "boot"
+        recovery = self.candidate / "etc/systemd/system/fred-deploy-recover.service"
+        recovery.parent.mkdir(parents=True, exist_ok=True)
+        recovery.write_text("fixture")
+        for system in (self.candidate, self.old_running, self.old_boot):
+            (system / "bin").mkdir(parents=True, exist_ok=True)
+            (system / "bin/switch-to-configuration").write_text("fixture")
+        runner = root / "tools/bin/worker"
+        runner.parent.mkdir(parents=True, exist_ok=True)
+        runner.write_text("fixture")
+        super().__init__(
+            runner, root / "state", root / "roots", root / "lock", self.fake
+        )
+        self.current = root / "current"
+        self.profile = root / "profile"
+        if not self.current.is_symlink():
+            self.current.symlink_to(self.old_running)
+        if not self.profile.is_symlink():
+            self.profile.symlink_to(self.old_boot)
+        self.boot_default = self.old_boot
+
+    def fake(self, args, timeout=120):
+        args = list(map(str, args))
+        self.calls.append(args)
+        step = args[0]
+        if args[:2] == ["nix", "build"]:
+            step = "build"
+            self.pin("candidate", self.candidate)
+        elif args[0] == "systemd-run":
+            step = "arm"
+        elif args[0] == "nix-env":
+            target = Path(args[-1])
+            self.profile.unlink()
+            self.profile.symlink_to(target)
+            step = "commit-profile" if target == self.candidate else "restore-profile"
+        elif args[0].endswith("switch-to-configuration"):
+            target = Path(args[0]).parents[1]
+            if args[1] == "test":
+                self.current.unlink()
+                self.current.symlink_to(target)
+            else:
+                self.boot_default = target
+            step = ("candidate-" if target == self.candidate else "restore-") + args[1]
+        elif args[:2] == ["systemctl", "stop"]:
+            step = "disarm" if args[-1].endswith(".timer") else "stop-worker"
+        elif args[:2] == ["systemctl", "show"]:
+            return "not-found"
+        if step in self.fail:
+            raise RuntimeError("Injected failure after side effect: " + step)
+        return ""
+
+    def healthy(self):
+        self.health_calls += 1
+        if self.current.resolve() == self.candidate:
+            if "health" in self.fail or (
+                "final-health" in self.fail and self.health_calls == 2
+            ):
+                raise RuntimeError("Injected unhealthy router")
+        elif "restore-health" in self.fail:
+            raise RuntimeError("Restored configuration is unhealthy")
+
+    def wait_healthy(self, duration=90):
+        self.healthy()
+
+    def deploy(self, ident="a" * 32, flake="fixture", reboot=False):
+        with patch.object(
+            Path,
+            "exists",
+            autospec=True,
+            side_effect=lambda p: (
+                False
+                if str(p) == "/run/current-system/sw/bin/record-update"
+                else original_exists(p)
+            ),
+        ):
+            super().deploy(ident, flake, reboot)
+
+
+original_exists = Path.exists
+
+
+class Transactions(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory(prefix="fred-deploy-check-")
+        self.addCleanup(self.temp.cleanup)
+        self.f = Fixture(Path(self.temp.name))
+        self.output = contextlib.redirect_stdout(io.StringIO())
+        self.output.__enter__()
+        self.addCleanup(self.output.__exit__, None, None, None)
+
+    def assert_restored(self):
+        self.assertEqual(self.f.current.resolve(), self.f.old_running)
+        self.assertEqual(self.f.profile.resolve(), self.f.old_boot)
+        self.assertEqual(self.f.boot_default, self.f.old_boot)
+
+    def pending(self, phase="activating", boot_id=None):
+        self.f.pin("old-running", self.f.old_running)
+        self.f.pin("old-boot", self.f.old_boot)
+        record = {
+            "id": "a" * 32,
+            "unit": "fred-deploy-" + "a" * 32 + ".service",
+            "old_running": str(self.f.old_running),
+            "old_boot": str(self.f.old_boot),
+            "candidate": str(self.f.candidate),
+            "boot_id": boot_id or self.f.boot_id(),
+        }
+        self.f.save(record, phase)
+        self.f.current.unlink()
+        self.f.current.symlink_to(self.f.candidate)
+        return record
+
+    def test_success_orders_test_health_and_boot_commit(self):
+        self.f.deploy()
+        self.assertEqual(self.f.read()["phase"], "committed")
+        self.assertEqual(self.f.profile.resolve(), self.f.candidate)
+        commands = self.f.calls
+        arm = next(i for i, c in enumerate(commands) if c[0] == "systemd-run")
+        test = commands.index(
+            [str(self.f.candidate / "bin/switch-to-configuration"), "test"]
+        )
+        boot = commands.index(
+            [str(self.f.candidate / "bin/switch-to-configuration"), "boot"]
+        )
+        self.assertLess(arm, test)
+        self.assertLess(test, boot)
+        self.assertEqual(self.f.health_calls, 2)
+        self.assertFalse(list(self.f.roots.iterdir()))
+
+    def test_build_failure_never_activates_or_arms(self):
+        self.f.fail.add("build")
+        with self.assertRaises(RuntimeError):
+            self.f.deploy()
+        self.assertEqual(self.f.read()["phase"], "build-failed")
+        self.assert_restored()
+        self.assertFalse(
+            any(
+                c[0] == "systemd-run" or c[0].endswith("switch-to-configuration")
+                for c in self.f.calls
+            )
+        )
+
+    def test_candidate_must_retain_boot_recovery(self):
+        (self.f.candidate / "etc/systemd/system/fred-deploy-recover.service").unlink()
+        with self.assertRaisesRegex(RuntimeError, "missing the boot recovery"):
+            self.f.deploy()
+        self.assert_restored()
+        self.assertEqual(self.f.read()["phase"], "build-failed")
+
+    def test_activation_and_commit_failure_paths_restore_both_systems(self):
+        for failure in [
+            "arm",
+            "candidate-test",
+            "health",
+            "commit-profile",
+            "candidate-boot",
+            "final-health",
+        ]:
+            with self.subTest(failure=failure), tempfile.TemporaryDirectory() as temp:
+                f = Fixture(Path(temp))
+                f.fail.add(failure)
+                with self.assertRaises(RuntimeError):
+                    f.deploy()
+                self.assertEqual(f.read()["phase"], "rolled-back")
+                self.assertEqual(f.current.resolve(), f.old_running)
+                self.assertEqual(f.profile.resolve(), f.old_boot)
+                self.assertEqual(f.boot_default, f.old_boot)
+
+    def test_failed_boot_restore_still_restores_runtime_and_retains_roots(self):
+        self.f.fail.update({"candidate-test", "restore-boot"})
+        with self.assertRaisesRegex(RuntimeError, "needs attention"):
+            self.f.deploy()
+        self.assertEqual(self.f.current.resolve(), self.f.old_running)
+        self.assertEqual(self.f.read()["phase"], "rollback-failed")
+        self.assertTrue((self.f.roots / "old-boot").exists())
+        with self.assertRaisesRegex(RuntimeError, "needs recovery"):
+            self.f.deploy("b" * 32)
+        self.f.fail.clear()
+        self.f.recover()
+        self.assertEqual(self.f.read()["phase"], "rolled-back")
+
+    def test_unhealthy_restored_router_is_not_reported_as_success(self):
+        self.f.fail.update({"candidate-test", "restore-health"})
+        with self.assertRaisesRegex(RuntimeError, "needs attention"):
+            self.f.deploy()
+        self.assertEqual(self.f.read()["phase"], "rollback-failed")
+
+    def test_late_timer_cannot_undo_commit(self):
+        self.f.fail.add("disarm")
+        self.f.deploy()
+        self.assertEqual(self.f.read()["phase"], "committed")
+        before = len(self.f.calls)
+        self.f.recover("a" * 32)
+        self.assertEqual(len(self.f.calls), before)
+
+    def test_stale_timer_cannot_touch_new_transaction(self):
+        self.pending()
+        self.f.recover("b" * 32)
+        self.assertFalse(self.f.calls)
+        self.assertEqual(self.f.read()["phase"], "activating")
+
+    def test_boot_service_does_not_cancel_its_own_installation(self):
+        self.pending()
+        self.f.recover(boot=True)
+        self.assertFalse(self.f.calls)
+        self.assertEqual(self.f.read()["phase"], "activating")
+
+    def test_next_boot_recovers_even_if_worker_unit_was_collected(self):
+        self.pending(boot_id="previous-boot")
+        self.f.fail.add("stop-worker")
+        self.f.recover(boot=True)
+        self.assert_restored()
+        self.assertEqual(self.f.read()["phase"], "rolled-back")
+
+    def test_interrupted_build_can_be_retried_without_activation(self):
+        self.pending(phase="building")
+        # A building record never altered the running system.
+        self.f.current.unlink()
+        self.f.current.symlink_to(self.f.old_running)
+        self.f.deploy("b" * 32)
+        self.assertEqual(self.f.read()["phase"], "committed")
+
+    def test_boot_time_deployment_recovers_previous_boot_first(self):
+        self.pending(boot_id="previous-boot")
+        self.f.deploy("b" * 32)
+        restore = self.f.calls.index(
+            [str(self.f.old_running / "bin/switch-to-configuration"), "test"]
+        )
+        build = next(
+            i for i, args in enumerate(self.f.calls) if args[:2] == ["nix", "build"]
+        )
+        self.assertLess(restore, build)
+        self.assertEqual(self.f.read()["phase"], "committed")
+
+    def test_failed_state_write_does_not_prevent_restoring_system(self):
+        record = self.pending()
+        save = self.f.save
+
+        def fail_start(record, phase, **fields):
+            if phase == "rolling-back":
+                raise OSError("disk full")
+            return save(record, phase, **fields)
+
+        self.f.save = fail_start
+        self.f.rollback_locked(record)
+        self.assert_restored()
+
+    def test_recovery_rereads_state_after_stopping_worker(self):
+        record = self.pending()
+        command = self.f.command
+
+        def race(args, **kw):
+            result = command(args, **kw)
+            if args[:2] == ["systemctl", "stop"]:
+                self.f.save({**record, "id": "b" * 32}, "armed")
+            return result
+
+        self.f.command = race
+        self.f.recover("a" * 32)
+        self.assertEqual(self.f.read()["id"], "b" * 32)
+        self.assertFalse(
+            any(c[0].endswith("switch-to-configuration") for c in self.f.calls)
+        )
+
+    def test_transaction_lock_excludes_another_process(self):
+        marker = self.f.root / "entered"
+        code = "import fcntl,pathlib,sys; f=open(sys.argv[1],'a'); fcntl.flock(f,fcntl.LOCK_EX); pathlib.Path(sys.argv[2]).touch()"
+        with self.f.exclusive():
+            child = subprocess.Popen(
+                [sys.executable, "-c", code, str(self.f.lock), str(marker)]
+            )
+            time.sleep(0.2)
+            self.assertFalse(marker.exists())
+        child.wait(timeout=3)
+        self.assertEqual(child.returncode, 0)
+        self.assertTrue(marker.exists())
+
+    def test_independent_recovery_stops_hung_worker_before_taking_lock(self):
+        self.pending()
+        marker = self.f.root / "holding-lock"
+        code = "import fcntl,pathlib,sys,time; f=open(sys.argv[1],'a'); fcntl.flock(f,fcntl.LOCK_EX); pathlib.Path(sys.argv[2]).touch(); time.sleep(30)"
+        worker = subprocess.Popen(
+            [sys.executable, "-c", code, str(self.f.lock), str(marker)]
+        )
+        try:
+            deadline = time.monotonic() + 3
+            while not marker.exists() and time.monotonic() < deadline:
+                time.sleep(0.01)
+            self.assertTrue(marker.exists())
+            recovery = subprocess.run(
+                [
+                    sys.executable,
+                    "-B",
+                    __file__,
+                    "recover-fixture",
+                    str(self.f.root),
+                    str(worker.pid),
+                ],
+                timeout=5,
+                capture_output=True,
+                text=True,
+            )
+            self.assertEqual(recovery.returncode, 0, recovery.stdout + recovery.stderr)
+            worker.wait(timeout=2)
+            self.assertEqual(worker.returncode, -signal.SIGKILL)
+            self.assert_restored()
+            self.assertEqual(self.f.read()["phase"], "rolled-back")
+        finally:
+            if worker.poll() is None:
+                worker.kill()
+                worker.wait()
+
+    def test_command_timeout_kills_descendants(self):
+        marker = self.f.root / "unexpected-child-write"
+        child = (
+            "import pathlib,time,sys; time.sleep(.5); pathlib.Path(sys.argv[1]).touch()"
+        )
+        parent = "import subprocess,sys,time; subprocess.Popen([sys.executable,'-c',sys.argv[1],sys.argv[2]]); time.sleep(30)"
+        with self.assertRaisesRegex(RuntimeError, "timed out"):
+            m.run([sys.executable, "-c", parent, child, str(marker)], timeout=0.1)
+        time.sleep(0.6)
+        self.assertFalse(marker.exists())
+
+    def test_reboot_is_requested_only_after_commit(self):
+        self.f.deploy(reboot=True)
+        self.assertEqual(self.f.calls[-1], ["systemctl", "reboot", "--no-block"])
+        self.assertEqual(self.f.read()["phase"], "committed")
+
+    def test_router_health_rejects_missing_essential_functions(self):
+        for failure in [
+            None,
+            "service",
+            "address",
+            "route",
+            "firewall",
+            "nat",
+            "wan",
+            "dns",
+            "ssh",
+            "proxy",
+        ]:
+            with self.subTest(failure=failure):
+
+                def health_command(args, timeout=120):
+                    if args[0] == "systemctl":
+                        self.assertNotIn("jellyfin", args)
+                        if failure == "service":
+                            raise RuntimeError("inactive")
+                        return ""
+                    if args[:4] == ["ip", "-j", "-4", "address"]:
+                        return json.dumps(
+                            [
+                                {
+                                    "addr_info": []
+                                    if failure == "address"
+                                    else [{"local": "10.0.0.1", "prefixlen": 24}]
+                                }
+                            ]
+                        )
+                    if args[0] == "ip":
+                        return json.dumps(
+                            []
+                            if failure == "route"
+                            else [{"dst": "default", "dev": "eno1"}]
+                        )
+                    if args[0] == "nft":
+                        if args[-1] == "filter":
+                            return json.dumps(
+                                {
+                                    "nftables": [
+                                        {
+                                            "chain": {
+                                                "hook": hook,
+                                                "policy": "accept"
+                                                if failure == "firewall"
+                                                else "drop",
+                                            }
+                                        }
+                                        for hook in ["input", "forward"]
+                                    ]
+                                }
+                            )
+                        return json.dumps(
+                            {
+                                "nftables": [
+                                    {"chain": {"hook": hook}}
+                                    for hook in ["prerouting", "postrouting"]
+                                ]
+                                + (
+                                    []
+                                    if failure == "nat"
+                                    else [
+                                        {
+                                            "rule": {
+                                                "expr": [
+                                                    {"dnat": {}},
+                                                    {"masquerade": None},
+                                                ]
+                                            }
+                                        }
+                                    ]
+                                )
+                            }
+                        )
+                    if args[0] == "ping":
+                        if failure == "wan":
+                            raise RuntimeError("unreachable")
+                        return ""
+                    if args[0] == "dig":
+                        return "" if failure == "dns" else "93.184.215.14"
+                    if args[0] == "curl":
+                        return "503" if failure == "proxy" else "302"
+                    raise AssertionError(args)
+
+                self.f.command = health_command
+                ssh = MagicMock()
+                ssh.__enter__.return_value.recv.return_value = (
+                    b"HTTP/1.1" if failure == "ssh" else b"SSH-2.0-fixture"
+                )
+                read_text = Path.read_text
+
+                def read(path, *args, **kw):
+                    if str(path) == "/proc/sys/net/ipv4/ip_forward":
+                        return "1"
+                    return read_text(path, *args, **kw)
+
+                with (
+                    patch.object(m.socket, "create_connection", return_value=ssh),
+                    patch.object(Path, "read_text", read),
+                ):
+                    if failure:
+                        with self.assertRaises(RuntimeError):
+                            m.Deployment.healthy(self.f)
+                    else:
+                        m.Deployment.healthy(self.f)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1 and sys.argv[1] == "recover-fixture":
+        fixture = Fixture(Path(sys.argv[2]))
+        worker_pid = int(sys.argv[3])
+        fake = fixture.command
+
+        def stop_fixture_worker(args, **kwargs):
+            if args[:2] == ["systemctl", "stop"] and args[-1].endswith(".service"):
+                os.kill(worker_pid, signal.SIGKILL)
+            return fake(args, **kwargs)
+
+        fixture.command = stop_fixture_worker
+        fixture.recover("a" * 32)
+    else:
+        unittest.main()

scripts/deploy.py

@@ -0,0 +1,540 @@
+"""Root-only server deployment transactions. Commands are supplied by the Nix wrapper.
+
+The deployment lock excludes other deployments and the WAN watchdog. A separate
+systemd timer can stop a stuck worker before taking that lock and rolling back.
+State and GC roots survive the invoking terminal, service and machine reboot.
+"""
+
+import argparse
+import contextlib
+import fcntl
+import ipaddress
+import json
+import os
+from pathlib import Path
+import re
+import signal
+import socket
+import subprocess
+import sys
+import time
+import uuid
+
+TERMINAL = {"committed", "rolled-back", "build-failed"}
+
+
+def run(args, timeout=120):
+    # A timeout must kill descendants too, not leave an activation writing in
+    # parallel with recovery. systemd also kills the entire worker cgroup.
+    proc = subprocess.Popen(
+        args,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        start_new_session=True,
+    )
+    try:
+        output, _ = proc.communicate(timeout=timeout)
+    except subprocess.TimeoutExpired:
+        os.killpg(proc.pid, signal.SIGKILL)
+        proc.communicate()
+        raise RuntimeError(f"Command timed out: {args[0]}") from None
+    if proc.returncode:
+        raise RuntimeError(
+            f"Command failed ({proc.returncode}): {' '.join(map(str, args))}\n{output}"
+        )
+    return output.strip()
+
+
+def sync_directory(path):
+    fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        os.fsync(fd)
+    finally:
+        os.close(fd)
+
+
+def atomic_json(path, value):
+    temp = path.with_suffix(".tmp")
+    with temp.open("w") as out:
+        json.dump(value, out)
+        out.flush()
+        os.fsync(out.fileno())
+    os.replace(temp, path)
+    sync_directory(path.parent)
+
+
+class Deployment:
+    def __init__(
+        self,
+        runner,
+        state="/var/lib/fred-deploy",
+        roots="/nix/var/nix/gcroots/fred-deploy",
+        lock="/run/lock/fred-deploy.lock",
+        command=run,
+    ):
+        self.runner = str(Path(runner).resolve())
+        self.state, self.roots, self.lock = Path(state), Path(roots), Path(lock)
+        self.command = command
+        self.profile = Path("/nix/var/nix/profiles/system")
+        self.current = Path("/run/current-system")
+        self.state.mkdir(mode=0o700, parents=True, exist_ok=True)
+        self.roots.mkdir(mode=0o700, parents=True, exist_ok=True)
+        sync_directory(self.state.parent)
+        sync_directory(self.roots.parent)
+
+    @contextlib.contextmanager
+    def exclusive(self):
+        with self.lock.open("a") as lock:
+            # Queue deployments; watchdog uses a nonblocking acquisition.
+            fcntl.flock(lock, fcntl.LOCK_EX)
+            yield
+
+    def read(self):
+        file = self.state / "last-result.json"
+        return json.loads(file.read_text()) if file.exists() else None
+
+    def save(self, record, phase, **fields):
+        record.update(phase=phase, **fields)
+        atomic_json(self.state / "last-result.json", record)
+        print(f"Deployment {record['id']}: {phase}", flush=True)
+
+    def pin(self, name, target):
+        link = self.roots / name
+        temporary = self.roots / (name + ".new")
+        temporary.unlink(missing_ok=True)
+        temporary.symlink_to(target)
+        os.replace(temporary, link)
+        sync_directory(self.roots)
+
+    def cleanup(self):
+        for name in ("candidate", "old-running", "old-boot", "runner"):
+            (self.roots / name).unlink(missing_ok=True)
+
+    def healthy(self):
+        c = self.command
+        c(
+            [
+                "systemctl",
+                "is-active",
+                "--quiet",
+                "systemd-networkd",
+                "dnsmasq",
+                "adguardhome",
+                "nginx",
+                "sshd",
+                "nftables",
+                "router-ui",
+            ],
+            timeout=10,
+        )
+        addresses = json.loads(
+            c(["ip", "-j", "-4", "address", "show", "dev", "eth0"], timeout=5)
+        )
+        if not any(
+            a["local"] == "10.0.0.1" and a["prefixlen"] == 24
+            for interface in addresses
+            for a in interface["addr_info"]
+        ):
+            raise RuntimeError("LAN address 10.0.0.1/24 missing")
+        routes = json.loads(
+            c(["ip", "-j", "-4", "route", "show", "default", "dev", "eno1"], timeout=5)
+        )
+        if (
+            not routes
+            or Path("/proc/sys/net/ipv4/ip_forward").read_text().strip() != "1"
+        ):
+            raise RuntimeError("WAN route or IPv4 forwarding missing")
+        rules = json.loads(
+            c(["nft", "-j", "list", "table", "inet", "filter"], timeout=5)
+        )
+        chains = [r["chain"] for r in rules["nftables"] if "chain" in r]
+        for hook in ("input", "forward"):
+            if not any(
+                ch.get("hook") == hook and ch.get("policy") == "drop" for ch in chains
+            ):
+                raise RuntimeError(f"Default-drop {hook} firewall missing")
+        nat = json.loads(
+            c(["nft", "-j", "list", "table", "ip", "router-nat"], timeout=5)
+        )
+        hooks = {r["chain"].get("hook") for r in nat["nftables"] if "chain" in r}
+        if not {"prerouting", "postrouting"} <= hooks:
+            raise RuntimeError("Router NAT hooks missing")
+        expressions = [
+            expr
+            for item in nat["nftables"]
+            if "rule" in item
+            for expr in item["rule"].get("expr", [])
+        ]
+        if not all(
+            any(action in expr for expr in expressions)
+            for action in ("dnat", "masquerade")
+        ):
+            raise RuntimeError("Router DNAT or masquerade rules missing")
+        try:
+            c(["ping", "-c1", "-W3", "-I", "eno1", "1.1.1.1"], timeout=5)
+        except RuntimeError:
+            c(["ping", "-c1", "-W3", "-I", "eno1", "8.8.8.8"], timeout=5)
+        answer = c(
+            ["dig", "@10.0.0.1", "example.com", "A", "+short", "+time=2", "+tries=1"],
+            timeout=5,
+        )
+        if not any(self.is_ip(line) for line in answer.splitlines()):
+            raise RuntimeError("LAN DNS query failed")
+        with socket.create_connection(("10.0.0.1", 22), timeout=3) as ssh:
+            if not ssh.recv(128).startswith(b"SSH-2.0-"):
+                raise RuntimeError("SSH did not return a protocol banner")
+        c(
+            [
+                "curl",
+                "--fail",
+                "--silent",
+                "--show-error",
+                "--max-time",
+                "5",
+                "http://127.0.0.1:8086/",
+            ],
+            timeout=7,
+        )
+        code = c(
+            [
+                "curl",
+                "--silent",
+                "--show-error",
+                "--max-time",
+                "5",
+                "--output",
+                "/dev/null",
+                "--write-out",
+                "%{http_code}",
+                "--resolve",
+                "router.nordhammer.it:443:10.0.0.1",
+                "https://router.nordhammer.it/",
+            ],
+            timeout=7,
+        )
+        if code not in {"200", "301", "302", "303", "307", "308", "401", "403"}:
+            raise RuntimeError(f"Management proxy returned HTTP {code}")
+
+    @staticmethod
+    def is_ip(value):
+        try:
+            ipaddress.IPv4Address(value)
+            return True
+        except ValueError:
+            return False
+
+    def wait_healthy(self, duration=90):
+        deadline = time.monotonic() + duration
+        while True:
+            try:
+                self.healthy()
+                return
+            except Exception as error:
+                if time.monotonic() >= deadline:
+                    raise RuntimeError(
+                        f"Router health checks failed: {error}"
+                    ) from error
+                print(f"Waiting for router health: {error}", flush=True)
+                time.sleep(5)
+
+    def disarm(self, record):
+        # Do not stop the recovery service: it may already be checking state.
+        self.command(
+            ["systemctl", "stop", f"fred-rollback-{record['id']}.timer"], timeout=15
+        )
+
+    def rollback_locked(self, record):
+        try:
+            self.save(record, "rolling-back")
+        except OSError as error:
+            # A full/read-only state filesystem must not prevent restoration.
+            print(f"Cannot record recovery start: {error}", flush=True)
+        errors = []
+        # Always attempt runtime recovery, even if bootloader restoration fails.
+        for args in (
+            ["nix-env", "--profile", str(self.profile), "--set", record["old_boot"]],
+            [record["old_boot"] + "/bin/switch-to-configuration", "boot"],
+            [record["old_running"] + "/bin/switch-to-configuration", "test"],
+        ):
+            try:
+                self.command(args, timeout=120)
+            except Exception as error:
+                errors.append(str(error))
+        try:
+            self.wait_healthy()
+        except Exception as error:
+            errors.append(str(error))
+        if errors:
+            self.save(record, "rollback-failed", errors=errors)
+            raise RuntimeError(
+                "Recovery needs attention; GC roots retained: " + "; ".join(errors)
+            )
+        self.save(record, "rolled-back")
+        # Cancellation failure cannot turn a completed rollback into success
+        # for the deployment. A late timer sees terminal state and does nothing.
+        try:
+            self.disarm(record)
+        except Exception as error:
+            print(f"Could not cancel completed recovery timer: {error}", flush=True)
+        self.cleanup()
+
+    def recover(self, expected=None, boot=False):
+        record = self.read()
+        if (
+            not record
+            or record["phase"] in TERMINAL
+            or (expected and expected != record["id"])
+        ):
+            return
+        if boot and record.get("boot_id") == self.boot_id():
+            return
+        expected = record["id"]
+        # Never wait behind a hung activation holding the transaction lock.
+        # Timer and boot recovery execute in their own cgroups.
+        try:
+            self.command(["systemctl", "stop", record["unit"]], timeout=30)
+        except RuntimeError:
+            if (
+                self.command(
+                    [
+                        "systemctl",
+                        "show",
+                        "--property=LoadState",
+                        "--value",
+                        record["unit"],
+                    ]
+                )
+                != "not-found"
+            ):
+                raise
+        with self.exclusive():
+            record = self.read()
+            if (
+                record
+                and record["phase"] not in TERMINAL
+                and (not expected or expected == record["id"])
+            ):
+                if record["phase"] == "building":
+                    self.save(
+                        record,
+                        "build-failed",
+                        error="Builder interrupted; no activation attempted",
+                    )
+                    self.cleanup()
+                else:
+                    self.rollback_locked(record)
+
+    @staticmethod
+    def boot_id():
+        return Path("/proc/sys/kernel/random/boot_id").read_text().strip()
+
+    def deploy(self, ident, flake, reboot=False):
+        with self.exclusive():
+            previous = self.read()
+            if (
+                previous
+                and previous["phase"] not in TERMINAL
+                and previous["phase"] != "building"
+                and previous.get("boot_id") != self.boot_id()
+            ):
+                # A boot-time update may win the lock before boot recovery.
+                # The former worker cannot exist in this boot: restore first.
+                self.rollback_locked(previous)
+            if previous and previous["phase"] == "building":
+                # Owning the lock proves the previous builder is gone. No
+                # activation was attempted, so a fresh build is safe.
+                self.save(
+                    previous, "build-failed", error="Previous builder interrupted"
+                )
+                self.cleanup()
+            if previous and previous["phase"] not in TERMINAL:
+                raise RuntimeError(
+                    "An interrupted deployment needs recovery first: sudo fred-deploy recover"
+                )
+            record = {
+                "id": ident,
+                "unit": f"fred-deploy-{ident}.service",
+                "flake": flake,
+                "boot_id": self.boot_id(),
+                "old_running": str(self.current.resolve(strict=True)),
+                "old_boot": str(self.profile.resolve(strict=True)),
+            }
+            self.pin("old-running", record["old_running"])
+            self.pin("old-boot", record["old_boot"])
+            self.pin("runner", str(Path(self.runner).parents[1]))
+            self.save(record, "building")
+            try:
+                self.command(
+                    [
+                        "nix",
+                        "build",
+                        "--refresh",
+                        "--out-link",
+                        str(self.roots / "candidate"),
+                        flake
+                        + "#nixosConfigurations.FredOS-Mediaserver.config.system.build.toplevel",
+                    ],
+                    timeout=5400,
+                )
+                candidate = str((self.roots / "candidate").resolve(strict=True))
+                if not Path(candidate, "bin/switch-to-configuration").is_file():
+                    raise RuntimeError("Build did not produce a NixOS system")
+                if not Path(
+                    candidate, "etc/systemd/system/fred-deploy-recover.service"
+                ).is_file():
+                    raise RuntimeError("Candidate is missing the boot recovery service")
+                self.pin("candidate", candidate)
+            except Exception as error:
+                self.save(record, "build-failed", error=str(error))
+                self.cleanup()
+                raise
+            # Durable intent precedes timer setup; boot recovery can handle a
+            # crash even between starting the timer and invoking activation.
+            self.save(record, "armed", candidate=candidate)
+            try:
+                self.command(
+                    [
+                        "systemd-run",
+                        "--collect",
+                        "--unit",
+                        f"fred-rollback-{ident}",
+                        "--on-active=10min",
+                        "--timer-property=AccuracySec=1s",
+                        "--timer-property=RemainAfterElapse=no",
+                        "--property=Type=exec",
+                        "--property=RuntimeMaxSec=12min",
+                        "--property=TimeoutStopSec=10s",
+                        self.runner,
+                        "recover",
+                        "--id",
+                        ident,
+                    ],
+                    timeout=20,
+                )
+                self.save(record, "activating")
+                self.command(
+                    [candidate + "/bin/switch-to-configuration", "test"], timeout=120
+                )
+                self.wait_healthy()
+                self.save(record, "committing")
+                self.command(
+                    ["nix-env", "--profile", str(self.profile), "--set", candidate],
+                    timeout=30,
+                )
+                self.command(
+                    [candidate + "/bin/switch-to-configuration", "boot"], timeout=120
+                )
+                self.healthy()
+                self.save(record, "committed")
+            except Exception as error:
+                # Timer owns recovery after SIGKILL; ordinary errors recover now.
+                print(f"Deployment failed: {error}", flush=True)
+                record["error"] = str(error)
+                self.rollback_locked(record)
+                raise RuntimeError(
+                    "Deployment failed; previous runtime and boot configuration restored"
+                ) from error
+            try:
+                self.disarm(record)
+            except Exception as error:
+                print(
+                    f"Timer cleanup failed (committed state is durable): {error}",
+                    flush=True,
+                )
+            self.cleanup()
+            recorder = Path("/run/current-system/sw/bin/record-update")
+            if recorder.exists():
+                try:
+                    self.command(
+                        [str(recorder), record["old_running"], candidate], timeout=30
+                    )
+                except Exception as error:
+                    print(
+                        f"Deployment committed; history recording failed: {error}",
+                        flush=True,
+                    )
+            if reboot:
+                components = ("kernel", "initrd", "kernel-modules")
+                if any(
+                    Path("/run/booted-system", p).resolve()
+                    != Path(candidate, p).resolve()
+                    for p in components
+                ):
+                    self.command(["systemctl", "reboot", "--no-block"])
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="fred-deploy",
+        description="Build and safely deploy the mediaserver configuration.",
+        usage="%(prog)s [--flake SOURCE] [--reboot]\n       %(prog)s recover",
+    )
+    parser.add_argument(
+        "action",
+        choices=["launch", "launch-recovery", "deploy", "recover"],
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--flake",
+        default="git+https://forg.gregersen.it/rope/nixos",
+        help="Flake source without a #fragment; defaults to the Forgejo repository",
+    )
+    parser.add_argument("--id", help=argparse.SUPPRESS)
+    parser.add_argument(
+        "--reboot",
+        action="store_true",
+        help="Reboot after success if boot components changed",
+    )
+    parser.add_argument("--boot", action="store_true", help=argparse.SUPPRESS)
+    args = parser.parse_args()
+    if os.geteuid() != 0:
+        parser.error("run this command with sudo")
+    if "#" in args.flake:
+        parser.error("pass a flake source without a #fragment")
+    if args.id and not re.fullmatch(r"[0-9a-f]{32}", args.id):
+        parser.error("invalid transaction ID")
+    runner = os.environ["FRED_DEPLOY_RUNNER"]
+    if args.action in {"launch", "launch-recovery"}:
+        ident = uuid.uuid4().hex
+        recovering = args.action == "launch-recovery"
+        unit = f"fred-recover-{ident}" if recovering else f"fred-deploy-{ident}"
+        print(f"Deployment logs: journalctl -fu {unit}", flush=True)
+        command = [
+            "systemd-run",
+            "--wait",
+            "--collect",
+            "--unit",
+            unit,
+            "--property=Type=exec",
+            "--property=RuntimeMaxSec=100min",
+            "--property=TimeoutStopSec=10s",
+            "--property=WorkingDirectory=/",
+            runner,
+        ]
+        command += (
+            ["recover"]
+            if recovering
+            else ["deploy", "--id", ident, "--flake", args.flake]
+        )
+        if args.reboot:
+            command.append("--reboot")
+        result = subprocess.run(command)
+        subprocess.run(["journalctl", "--no-pager", "-o", "cat", "-u", unit])
+        return result.returncode
+    deployment = Deployment(runner)
+    if args.action == "recover":
+        deployment.recover(args.id, args.boot)
+    else:
+        if not args.id:
+            parser.error("deploy requires a transaction ID")
+        deployment.deploy(args.id, args.flake, args.reboot)
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except Exception as error:
+        print(str(error), file=sys.stderr, flush=True)
+        sys.exit(1)

scripts/greeter-check.py

@@ -0,0 +1,129 @@
+"""Test greeter failure/retry/launch against a fake greetd server.
+
+Usage: QML2_IMPORT_PATH=<qt5compat>/lib/qt-6/qml python3 scripts/greeter-check.py <auth-bundle>
+Requires qs on PATH. No real authentication or desktop launch occurs.
+"""
+
+import json, os, socket, struct, subprocess as sp, tempfile, time, shutil, sys
+from pathlib import Path
+
+qs = shutil.which("qs")
+assert qs, "qs must be on PATH"
+root = Path(tempfile.mkdtemp(prefix="fred-greetd-check."))
+root.chmod(0o700)
+shutil.copytree(str(Path(sys.argv[1]) / "greeter"), root / "config")
+p = root / "config/shell.qml"
+p.chmod(0o600)
+s = (
+    p.read_text()
+    .replace("import Quickshell\n", "import Quickshell\nimport Quickshell.Io\n")
+    .replace("    AuthView {", "    AuthView {\n        id: auth")
+)
+s = (
+    s.rstrip()[:-1]
+    + """    IpcHandler { target: "test"; function submit(password: string): void { auth.submitted(password); } }
+}
+"""
+)
+p.write_text(s)
+server = socket.socket(socket.AF_UNIX)
+server.bind(str(root / "greetd.sock"))
+server.listen()
+server.settimeout(10)
+env = {
+    **os.environ,
+    "HOME": str(root),
+    "XDG_RUNTIME_DIR": str(root),
+    "GREETD_SOCK": str(root / "greetd.sock"),
+    "FRED_GREETER": "1",
+    "QT_QPA_PLATFORM": "offscreen",
+    "QT_QUICK_BACKEND": "software",
+    "FRED_SESSION_COMMAND": "/fixture/authenticated-session",
+}
+log = open(root / "log", "w")
+proc = sp.Popen([qs, "-p", str(root / "config")], env=env, stdout=log, stderr=log)
+try:
+    conn, _ = server.accept()
+    conn.settimeout(5)
+
+    def recv():
+        data = b""
+        while len(data) < 4:
+            data += conn.recv(4 - len(data))
+        size = struct.unpack("<I", data)[0]
+        data = b""
+        while len(data) < size:
+            data += conn.recv(size - len(data))
+        value = json.loads(data)
+        print(value, flush=True)
+        return value
+
+    def send(value):
+        data = json.dumps(value).encode()
+        conn.sendall(struct.pack("<I", len(data)) + data)
+
+    def submit():
+        for _ in range(30):
+            result = sp.run(
+                [
+                    qs,
+                    "ipc",
+                    "--pid",
+                    str(proc.pid),
+                    "call",
+                    "test",
+                    "submit",
+                    "fixture-password",
+                ],
+                env=env,
+                capture_output=True,
+                text=True,
+            )
+            if result.returncode == 0:
+                return
+            time.sleep(0.1)
+        raise AssertionError(result.stderr)
+
+    for success in [False, True]:
+        submit()
+        assert recv() == {"type": "create_session", "username": "fred"}
+        send(
+            {
+                "type": "auth_message",
+                "auth_message_type": "secret",
+                "auth_message": "Password:",
+            }
+        )
+        assert recv() == {
+            "type": "post_auth_message_response",
+            "response": "fixture-password",
+        }
+        if success:
+            send({"type": "success"})
+            launch = recv()
+            assert (
+                launch["type"] == "start_session"
+                and launch["cmd"] == ["/fixture/authenticated-session"]
+                and "XDG_SESSION_TYPE=wayland" in launch["env"]
+            )
+            send({"type": "success"})
+            assert proc.wait(timeout=5) == 0
+        else:
+            send(
+                {
+                    "type": "error",
+                    "error_type": "auth_error",
+                    "description": "fixture rejection",
+                }
+            )
+            assert recv() == {"type": "cancel_session"}
+            send({"type": "success"})
+            time.sleep(0.3)
+    print(
+        "Greeter: failed password does not launch; retry and authenticated launch pass"
+    )
+finally:
+    proc.terminate()
+    proc.wait()
+    log.close()
+    print((root / "log").read_text())

scripts/lock-session.sh

@@ -0,0 +1,18 @@
+# Invoked by the fred-lock wrapper, with coreutils, systemd and qs on PATH.
+set -euo pipefail
+
+timeout 5s systemctl --user start quickshell-lock.service
+deadline=$((SECONDS + 10))
+while (( SECONDS < deadline )); do
+    # Select the running service, including one from an older generation.
+    locker_pid=$(timeout 1s systemctl --user show quickshell-lock.service --property=MainPID --value)
+    if [[ "$locker_pid" =~ ^[1-9][0-9]*$ ]] &&
+        timeout 1s qs ipc --pid "$locker_pid" call lock lock >/dev/null 2>&1 &&
+        secure=$(timeout 1s qs ipc --pid "$locker_pid" call lock secure 2>/dev/null) &&
+        [[ "$secure" == true ]]; then
+        exit 0
+    fi
+    sleep 0.1
+done
+echo "The compositor did not confirm a secure session lock" >&2
+exit 1

scripts/media-storage-check.py

@@ -0,0 +1,61 @@
+"""Test the generated storage readiness script in a private mount namespace.
+
+Usage: sudo python3 scripts/media-storage-check.py <media-storage-ready-unit>
+Uses temporary tmpfs mounts and NSS fixtures; host media and mount state are untouched.
+"""
+
+import os, re, subprocess as sp, sys, tempfile
+from pathlib import Path
+
+original = os.environ.get("FRED_TEST_ORIGINAL_MNTNS")
+if not original:
+    os.execvpe(
+        "unshare",
+        ["unshare", "-m", sys.executable, *sys.argv],
+        {**os.environ, "FRED_TEST_ORIGINAL_MNTNS": os.readlink("/proc/self/ns/mnt")},
+    )
+assert original != os.readlink("/proc/self/ns/mnt")
+sp.run(["mount", "--make-rprivate", "/"], check=True)
+sp.run(["mount", "-t", "tmpfs", "tmpfs", "/mnt"], check=True)
+for name in ["storage", "disk1", "disk2", "disk3"]:
+    Path("/mnt", name).mkdir()
+unit = (Path(sys.argv[1]) / "media-storage-ready.service").read_text()
+script = re.search(r"^ExecStart=(\S+)", unit, re.M)[1]
+
+
+def ready(expected):
+    r = sp.run([script], capture_output=True, text=True)
+    assert (r.returncode == 0) == expected, r.stderr
+    print(r.stderr.strip() or "Readiness succeeded", flush=True)
+
+
+ready(False)
+assert list(Path("/mnt/storage").iterdir()) == []
+for name in ["storage", "disk1", "disk2"]:
+    sp.run(["mount", "-t", "tmpfs", "tmpfs", "/mnt/" + name], check=True)
+ready(False)
+assert list(Path("/mnt/storage").iterdir()) == []
+# Isolated NSS fixtures let tmpfiles resolve the real configured names.
+root = Path(tempfile.mkdtemp(prefix="fred-storage-users."))
+(root / "passwd").write_text(
+    "root:x:0:0:root:/root:/bin/sh\n"
+    + "".join(
+        f"{name}:x:0:0::{root}:/bin/sh\n"
+        for name in ["fred", "qbittorrent", "sonarr", "radarr", "sabnzbd"]
+    )
+)
+(root / "group").write_text("root:x:0:\nmedia:x:0:\n")
+for name in ["passwd", "group"]:
+    sp.run(["mount", "--bind", str(root / name), "/etc/" + name], check=True)
+sp.run(["mount", "-t", "tmpfs", "tmpfs", "/mnt/disk3"], check=True)
+ready(True)
+assert Path("/mnt/storage/torrents/shows").is_dir()
+sp.run(["umount", "/mnt/disk2"], check=True)
+ready(False)
+sp.run(["umount", "/mnt/storage"], check=True)
+assert list(Path("/mnt/storage").iterdir()) == [], (
+    "wrote into underlying root directory"
+)
+print(
+    "Storage guard: missing pool/branch and lost branch refused; mounted pool prepared; underlying directory untouched"
+)

scripts/router-firewall-check.py

@@ -0,0 +1,142 @@
+"""Exercise evaluated router rules in private user/network namespaces.
+
+Run: python3 scripts/router-firewall-check.py /tmp/router-rules.nft
+Requires ip, nft, nsenter and unshare on PATH. Never runs against the host network.
+The extra NAT table models Docker publishing allowed and unlisted host ports.
+"""
+
+import os
+from pathlib import Path
+import subprocess as sp
+import sys
+
+original = os.environ.get("FRED_TEST_ORIGINAL_NETNS")
+if not original:
+    os.execvpe(
+        "unshare",
+        ["unshare", "-Urn", sys.executable, *sys.argv],
+        {**os.environ, "FRED_TEST_ORIGINAL_NETNS": os.readlink("/proc/self/ns/net")},
+    )
+if os.readlink("/proc/self/ns/net") == original:
+    sys.exit("Refusing to run outside a private network namespace")
+
+children = []
+
+
+def run(*args, **kw):
+    return sp.run(args, check=True, text=True, **kw)
+
+
+def net(pid, *args, **kw):
+    return run("nsenter", "-t", str(pid), "-n", *args, **kw)
+
+
+def peer(interface, address, peer_address):
+    proc = sp.Popen(["unshare", "-n", "sleep", "300"])
+    children.append(proc)
+    # unshare must finish before moving the peer into its namespace.
+    import time
+
+    for _ in range(100):
+        if os.readlink(f"/proc/{proc.pid}/ns/net") != os.readlink("/proc/self/ns/net"):
+            break
+        time.sleep(0.01)
+    run("ip", "link", "add", interface, "type", "veth", "peer", "name", "peer0")
+    run("ip", "link", "set", "peer0", "netns", str(proc.pid))
+    run("ip", "addr", "add", address + "/24", "dev", interface)
+    run("ip", "link", "set", interface, "up")
+    net(proc.pid, "ip", "addr", "add", peer_address + "/24", "dev", "peer0")
+    net(proc.pid, "ip", "link", "set", "peer0", "up")
+    net(proc.pid, "ip", "link", "set", "lo", "up")
+    net(proc.pid, "ip", "route", "add", "default", "via", address)
+    return proc.pid
+
+
+server = """
+import socket,threading,time
+ports=PORTS
+for port in ports:
+ for kind in (socket.SOCK_STREAM,socket.SOCK_DGRAM):
+  s=socket.socket(socket.AF_INET,kind);s.bind(('0.0.0.0',port))
+  if kind==socket.SOCK_STREAM:s.listen()
+  def serve(s=s,kind=kind):
+   while True:
+    if kind==socket.SOCK_STREAM:
+     c,a=s.accept()
+     with c:
+      data=c.recv(16)
+      if data:c.sendall(data)
+    else:
+     data,a=s.recvfrom(16);s.sendto(data,a)
+  threading.Thread(target=serve,daemon=True).start()
+print('ready',flush=True)
+time.sleep(300)
+"""
+
+
+def serve(pid, ports):
+    cmd = [sys.executable, "-u", "-c", server.replace("PORTS", repr(ports))]
+    if pid:
+        cmd = ["nsenter", "-t", str(pid), "-n"] + cmd
+    proc = sp.Popen(cmd, stdout=sp.PIPE, text=True)
+    children.append(proc)
+    assert proc.stdout.readline().strip() == "ready"
+
+
+def check(pid, port, proto, expected):
+    client = f"""
+import socket
+s=socket.socket(socket.AF_INET,socket.SOCK_{"STREAM" if proto == "tcp" else "DGRAM"})
+s.settimeout(.4)
+try:
+ s.connect(('{"10.0.0.1" if pid == lan else "198.18.0.1"}',{port}))
+ s.send(b'hello')
+ ok=s.recv(16)==b'hello'
+except OSError:ok=False
+assert ok=={expected!r}, 'unexpected reachability: {proto}/{port}, expected {expected}'
+"""
+    net(pid, sys.executable, "-c", client)
+    print(
+        f"{'WAN' if pid == wan else 'LAN'} {proto}/{port}: {'allowed' if expected else 'blocked'}"
+    )
+
+
+try:
+    run("ip", "link", "set", "lo", "up")
+    wan = peer("eno1", "198.18.0.1", "198.18.0.2")
+    lan = peer("eth0", "10.0.0.1", "10.0.0.2")
+    container = peer("docker0", "172.18.0.1", "172.18.0.2")
+    Path("/proc/sys/net/ipv4/ip_forward").write_text("1")
+    run("nft", "-f", sys.argv[1])
+    run(
+        "nft",
+        "-f",
+        "-",
+        input="""
+      table ip docker_test {
+        chain prerouting {
+          type nat hook prerouting priority -101; policy accept;
+          iifname "eno1" tcp dport { 25565, 25600, 2022 } dnat to 172.18.0.2
+          iifname "eno1" udp dport { 25565, 25600, 2022 } dnat to 172.18.0.2
+          iifname "eno1" tcp dport { 25564, 25601, 9999 } dnat to 172.18.0.2:9000
+        }
+        chain forward { type filter hook forward priority 0; policy accept; }
+      }
+    """,
+    )
+    serve(None, [22, 443, 2022])
+    serve(container, [25565, 25600, 2022, 9000])
+    for port in [22, 443, 25565, 25600]:
+        check(wan, port, "tcp", True)
+    for port in [25565, 25600]:
+        check(wan, port, "udp", True)
+    for port in [2022, 25564, 25601, 9999]:
+        check(wan, port, "tcp", False)
+    check(wan, 2022, "udp", False)
+    check(lan, 2022, "tcp", True)
+    print("Router namespace checks passed")
+finally:
+    for proc in reversed(children):
+        proc.terminate()
+    for proc in children:
+        proc.wait()

scripts/router-forwards-check.nix

@@ -0,0 +1,27 @@
+{ lib }:
+let
+  validate = import ../services/validate-forwards.nix { inherit lib; };
+  valid = builtins.fromTOML (builtins.readFile ../ports.toml);
+  accepts = data: (builtins.tryEval (builtins.deepSeq (validate data) true)).success;
+  extra = entry: valid // { forward = valid.forward ++ [ entry ]; };
+  forward = { name = "test"; port = 9999; protocol = "tcp"; };
+  invalid = [
+    (valid // { dest_default = "10.0.0.2"; })
+    (valid // { forward = builtins.filter (f: f.name != "SSH") valid.forward; })
+    (valid // { forward = map (f: if f.name == "HTTPS" then f // { dest = "10.0.0.2"; } else f) valid.forward; })
+    (extra (forward // { name = "bad\" label"; }))
+    (extra (forward // { name = "SSH"; }))
+    (extra (forward // { protocol = "sctp"; }))
+    (extra (forward // { dest = "10.0.0.5; accept"; }))
+    (extra (forward // { dest = "10.0.0.255"; }))
+    (extra (forward // { port = 0; }))
+    (extra (forward // { port = 65536; }))
+    (extra (forward // { ports = "10000-10001"; }))
+    (extra { name = "overlap"; ports = "25600-25601"; protocol = "udp"; })
+    (extra { name = "backwards"; ports = "9001-9000"; protocol = "udp"; })
+  ];
+in
+assert accepts valid;
+assert lib.all (data: !accepts data) invalid;
+assert accepts (extra (forward // { port = 443; protocol = "udp"; dest = "10.0.0.2"; }));
+"Forward validation: valid inventory and separate protocols accepted; all ${toString (builtins.length invalid)} invalid cases rejected"

scripts/router-ui-check.py

@@ -174,3 +174,22 @@ assert got[0] == {"ts": "2026-08-15 12:00:55", "down": 508.9, "up": 70.7,
                   "ping": 18.8, "server": "Preston"}, got[0]
 
 print("\nall checks passed", file=sys.stderr)
+
+# A save must not inject nft syntax, introduce ambiguous forwards, or move
+# management ports to another machine while appearing to keep them open.
+for extra in [
+    {"name": 'bad" rule', "port": "9999", "protocol": "tcp"},
+    {"name": "bad destination", "port": "9999", "protocol": "tcp", "dest": "10.0.0.5; accept"},
+    {"name": "outside LAN", "port": "9999", "protocol": "tcp", "dest": "192.168.1.2"},
+    {"name": "broadcast", "port": "9999", "protocol": "tcp", "dest": "10.0.0.255"},
+    {"name": "overlap", "port": "25600-25610", "protocol": "udp"},
+    {"name": "SSH", "port": "9999", "protocol": "udp"},
+    {"name": "ambiguous", "port": "9999", "ports": "10000-10001", "protocol": "udp"},
+]:
+    refuses(r.check_forwards, mk() + [extra])
+for name in ("SSH", "HTTPS"):
+    entries = mk()
+    next(f for f in entries if f["name"] == name)["dest"] = "10.0.0.2"
+    refuses(r.check_forwards, entries)
+r.check_forwards(mk() + [{"name": "different protocol", "port": "443", "protocol": "udp", "dest": "10.0.0.2"}])
+print("Forward failure-path checks passed")

scripts/router-ui.py

@@ -377,14 +377,20 @@ def check_devices(items, self_mac, self_ip):
 
 def check_forwards(items):
     covered = set()
+    names = set()
     for f in items:
         name = str(f.get("name", "")).strip()
         proto = str(f.get("protocol", "")).strip()
-        if not name:
-            raise Refused("every forward needs a name")
+        if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9 _().-]{0,79}", name):
+            raise Refused("forward names need 1–80 letters, digits, spaces or _().-")
+        if name in names:
+            raise Refused(f"duplicate forward name: {name}")
+        names.add(name)
         if proto not in ("tcp", "udp", "both"):
             raise Refused(f"'{proto}' is not tcp, udp or both")
         # Accept either key so re-validating an already-normalised entry works.
+        if f.get("port") and f.get("ports"):
+            raise Refused("use either port or ports, not both")
         spec = str(f.get("port") or f.get("ports") or "").strip()
         if not PORT_RE.match(spec):
             raise Refused(f"'{spec}' is not a port or range (443, or 26901-26902)")
@@ -403,12 +409,18 @@ def check_forwards(items):
         else:
             f["port"] = bounds[0]
         dest = str(f.get("dest", "")).strip()
+        if dest and not re.fullmatch(r"10\.0\.0\.([1-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-4])", dest):
+            raise Refused("forward destination must be a LAN address in 10.0.0.1–254")
         if dest:
             f["dest"] = dest
         else:
             f.pop("dest", None)
         for p in range(bounds[0], bounds[-1] + 1):
             for pr in (("tcp", "udp") if proto == "both" else (proto,)):
+                if (pr, str(p)) in covered:
+                    raise Refused(f"overlapping forward for {pr}/{p}")
+                if (pr, str(p)) in REQUIRED_FORWARDS and dest not in ("", ROUTER_IP):
+                    raise Refused(f"{pr}/{p} must still reach the router ({ROUTER_IP})")
                 covered.add((pr, str(p)))
     for need in REQUIRED_FORWARDS:
         if need not in covered:

scripts/session-lock-check.py

@@ -0,0 +1,121 @@
+"""Test compositor lock confirmation and helper failure when IPC disappears.
+
+Usage: QML2_IMPORT_PATH=<qt5compat>/lib/qt-6/qml python3 scripts/session-lock-check.py <auth-bundle>
+Requires qs and sway on PATH. Starts only a private, headless compositor.
+"""
+
+import os, subprocess as sp, tempfile, time, shutil, sys
+from pathlib import Path
+
+root = Path(tempfile.mkdtemp(prefix="fred-wayland-check."))
+root.chmod(0o700)
+(root / "sway.conf").write_text(
+    "output HEADLESS-1 mode 1280x720\nseat seat0 fallback true\n"
+)
+env = {
+    **os.environ,
+    "HOME": str(root),
+    "XDG_RUNTIME_DIR": str(root),
+    "WLR_BACKENDS": "headless",
+    "WLR_RENDERER": "pixman",
+    "WLR_LIBINPUT_NO_DEVICES": "1",
+    "QT_QUICK_BACKEND": "software",
+}
+env.pop("WAYLAND_DISPLAY", None)
+env.pop("DISPLAY", None)
+qs = shutil.which("qs")
+assert qs and shutil.which("sway"), "qs and sway must be on PATH"
+log = open(root / "sway.log", "w")
+sway = sp.Popen(
+    [shutil.which("sway"), "-c", str(root / "sway.conf")],
+    env=env,
+    stdout=log,
+    stderr=log,
+)
+lock = None
+try:
+    for _ in range(100):
+        sockets = [p for p in root.glob("wayland-*") if not str(p).endswith(".lock")]
+        if sockets:
+            break
+        if sway.poll() is not None:
+            raise RuntimeError("sway failed")
+        time.sleep(0.1)
+    assert sockets
+    env["WAYLAND_DISPLAY"] = sockets[0].name
+    env["QT_QPA_PLATFORM"] = "wayland"
+    locklog = open(root / "lock.log", "w")
+    lock = sp.Popen(
+        [qs, "-p", str(Path(sys.argv[1]) / "lock")],
+        env=env,
+        stdout=locklog,
+        stderr=locklog,
+    )
+    for _ in range(100):
+        result = sp.run(
+            [qs, "ipc", "--pid", str(lock.pid), "call", "lock", "secure"],
+            env=env,
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode == 0 and result.stdout.strip() == "true":
+            break
+        if lock.poll() is not None:
+            raise RuntimeError("lock failed")
+        time.sleep(0.1)
+    else:
+        raise AssertionError(result.stdout + result.stderr)
+    print("Wayland compositor confirms secure lock", flush=True)
+    mock = root / "bin"
+    mock.mkdir()
+    (mock / "systemctl").write_text(
+        '#!/bin/sh\ncase "$*" in *show*) echo ' + str(lock.pid) + ";; esac\n"
+    )
+    (mock / "systemctl").chmod(0o700)
+    (mock / "qs").symlink_to(qs)
+    testenv = {**env, "PATH": str(mock) + ":" + os.environ["PATH"]}
+    result = sp.run(
+        ["bash", str(Path(__file__).with_name("lock-session.sh"))],
+        env=testenv,
+        capture_output=True,
+        text=True,
+        timeout=20,
+    )
+    assert result.returncode == 0, result.stderr
+    print("Lock helper returns success only with secure confirmation", flush=True)
+    lock.terminate()
+    lock.wait()
+    result = sp.run(
+        ["bash", str(Path(__file__).with_name("lock-session.sh"))],
+        env=testenv,
+        capture_output=True,
+        text=True,
+        timeout=20,
+    )
+    assert result.returncode != 0 and "did not confirm" in result.stderr, result
+    print("Lock helper fails when lock IPC disappears", flush=True)
+    (mock / "qs").unlink()
+    (mock / "qs").write_text('#!/bin/sh\ncase "$*" in *secure*) echo false;; esac\n')
+    (mock / "qs").chmod(0o700)
+    result = sp.run(
+        ["bash", str(Path(__file__).with_name("lock-session.sh"))],
+        env=testenv,
+        capture_output=True,
+        text=True,
+        timeout=20,
+    )
+    assert result.returncode != 0 and "did not confirm" in result.stderr, result
+    print(
+        "Lock helper fails when IPC works but the compositor never confirms coverage",
+        flush=True,
+    )
+finally:
+    if lock and lock.poll() is None:
+        lock.terminate()
+        lock.wait()
+    sway.terminate()
+    sway.wait()
+    log.close()
+    for name in ["lock.log", "sway.log"]:
+        if (root / name).exists():
+            print(name, (root / name).read_text()[-4000:])