rope/nixos
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Harden deployment, authentication, firewall and media storage failure paths

Browse source
This commit is contained in:
rope 2026-09-14 11:50:22 +01:00
parent 64ede7bd05
commit 7145a35927
35 changed files with 2529 additions and 499 deletions
Download patch file Download diff file Expand all files Collapse all files

75
services/deployment.nix Normal file
View file

@ -0,0 +1,75 @@
# One protected activation path for manual, UI and nightly server deployments.
{ config, lib, pkgs, ... }:
let
worker = pkgs.writeShellApplication {
name = "fred-deploy-worker";
runtimeInputs = [
config.nix.package pkgs.systemd pkgs.iproute2 pkgs.iputils
pkgs.nftables pkgs.dig pkgs.curl
];
text = ''
export FRED_DEPLOY_RUNNER="$0"
exec ${pkgs.python3}/bin/python3 ${../scripts/deploy.py} "$@"
'';
};
command = pkgs.writeShellApplication {
name = "fred-deploy";
text = ''
if [ "''${1-}" = recover ]; then
shift
if [ -x /nix/var/nix/gcroots/fred-deploy/runner/bin/fred-deploy-worker ]; then
exec /nix/var/nix/gcroots/fred-deploy/runner/bin/fred-deploy-worker launch-recovery "$@"
fi
exec ${worker}/bin/fred-deploy-worker launch-recovery "$@"
fi
exec ${worker}/bin/fred-deploy-worker launch "$@"
'';
};
in {
config = lib.mkIf (config.networking.hostName == "FredOS-Mediaserver") {
environment.systemPackages = [ command ];
system.build.deploy = command;
systemd.services.nixos-upgrade = {
description = "Build, verify and commit the nightly server deployment";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
restartIfChanged = false;
unitConfig.X-StopOnRemoval = false;
serviceConfig = {
Type = "oneshot";
ExecStart = "${command}/bin/fred-deploy --reboot";
TimeoutStartSec = "105min";
};
};
systemd.timers.nixos-upgrade = {
wantedBy = [ "timers.target" ];
timerConfig = { OnCalendar = "05:15"; Persistent = true; };
};
# Transient rollback timers do not survive reboot. Recover unfinished
# transactions from durable state using the pinned transaction runner.
systemd.services.fred-deploy-recover = {
description = "Recover an interrupted server deployment";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
before = [ "wan-watchdog.service" "nixos-upgrade.service" ];
restartIfChanged = false;
unitConfig.X-StopOnRemoval = false;
# A oneshot WantedBy=multi-user would hold up that target while the
# activation script waits for target start jobs: use exec so boot can
# finish while recovery holds the shared deployment lock.
serviceConfig = {
Type = "exec";
RuntimeMaxSec = "15min";
TimeoutStopSec = "10s";
};
script = ''
if [ -x /nix/var/nix/gcroots/fred-deploy/runner/bin/fred-deploy-worker ]; then
exec /nix/var/nix/gcroots/fred-deploy/runner/bin/fred-deploy-worker recover --boot
fi
exec ${worker}/bin/fred-deploy-worker recover --boot
'';
};
};
}

5
services/pelican.nix
View file

@ -122,8 +122,9 @@
ports = [
# Daemon API + console websocket — nginx fronts it as games.nordhammer.it.
"127.0.0.1:8443:8080"
# SFTP into server files. LAN-reachable; not in ports.toml, so no WAN.
"2022:2022"
# SFTP into server files. Bind LAN explicitly; router.nix also denies
# new WAN connections outside the ports.toml allowlist after Docker NAT.
"10.0.0.1:2022:2022"
];
environment.TZ = "Europe/Stockholm";
# Image is distroless with no entrypoint script — needs a tty allocated.

3
services/qbittorrent-nox.nix
View file

@ -25,9 +25,6 @@
"d /var/lib/qbittorrent 0755 qbittorrent media -"
"Z /var/lib/qbittorrent 0755 qbittorrent media -"
# Storage - qbittorrent downloads here
"d /mnt/storage/torrents/downloads 2775 qbittorrent media -"
"Z /mnt/storage/torrents/downloads 2775 qbittorrent media -"
];
systemd.services.qbittorrent-nox = {

6
services/radarr.nix
View file

@ -36,10 +36,6 @@
};
# Set up directory structure with proper permissions
systemd.tmpfiles.rules = [
# Media folders - radarr writes here
"d /mnt/storage/torrents/movies 2775 radarr media -"
"Z /mnt/storage/torrents/movies 2775 radarr media -"
];
# Pool directories and mount dependencies live in server-permissions.nix.
};
}

64
services/router-ui.nix
View file

@ -6,15 +6,8 @@
# deploy that exact commit. services/router.nix reads those same files, so the
# UI is a constrained editor for config that is still fully reviewable in git.
#
# Apply is deliberately paranoid, because a bad generation here takes the
# household's internet with it:
#
# nixos-rebuild test -> eval/build failure aborts before anything activates
# wait, then health-check WAN ping + dnsmasq + LAN address + NAT table
# healthy -> nixos-rebuild switch (only now does it become the boot default)
# unhealthy -> switch-to-configuration test on the old profile, + ntfy alert
#
# `test` never touches the boot default, so a reboot is always an escape hatch.
# Apply delegates to the shared deployment transaction in deployment.nix.
# Builds precede activation; an independent timer recovers partial failures.
#
# Requires a Forgejo token with write:repository on rope/nixos:
# printf '%s' '<token>' | sudo tee /var/secrets/forgejo-router-token
@ -34,7 +27,7 @@ let
sudo = "/run/wrappers/bin/sudo";
applyScript = pkgs.writeShellScript "router-apply" ''
set -uo pipefail
set -euo pipefail
rev="$1"
flake="${flakeUrl}?rev=$rev"
secret=/var/secrets/ntfy-url
@ -47,45 +40,16 @@ let
-d "$1" "$url" >/dev/null 2>&1 || true
}
old=$(${pkgs.coreutils}/bin/readlink /run/current-system)
echo "==> building + activating $rev (test: boot default untouched)"
if ! nixos-rebuild test --refresh --flake "$flake" -L; then
echo "!! eval or build failed — nothing was activated, old config still running"
notify "Router apply ''${rev:0:8}: build failed. Nothing changed."
if [[ ! "$rev" =~ ^[0-9a-f]{40}$ ]]; then
echo "Invalid commit revision" >&2
exit 1
fi
echo "==> activated, waiting 45s for the network to settle"
sleep 45
healthy=1
${pkgs.iputils}/bin/ping -c1 -W3 -I eno1 1.1.1.1 >/dev/null 2>&1 \
|| ${pkgs.iputils}/bin/ping -c1 -W3 -I eno1 8.8.8.8 >/dev/null 2>&1 \
|| { echo "!! no WAN"; healthy=0; }
${systemctl} is-active --quiet dnsmasq || { echo "!! dnsmasq down"; healthy=0; }
${pkgs.iproute2}/bin/ip -4 addr show eth0 \
| ${pkgs.gnugrep}/bin/grep -q '10\.0\.0\.1' || { echo "!! LAN address gone"; healthy=0; }
${pkgs.nftables}/bin/nft list table ip router-nat >/dev/null 2>&1 \
|| { echo "!! NAT table missing"; healthy=0; }
if [ "$healthy" = 1 ]; then
echo "==> healthy — making $rev the boot default"
if nixos-rebuild switch --refresh --flake "$flake" -L; then
${pkgs.coreutils}/bin/printf '%s' "$rev" > ${stateDir}/last-rev
${pkgs.coreutils}/bin/chown router-ui ${stateDir}/last-rev || true
record-update "$old" /run/current-system || true
notify "Router applied ''${rev:0:8}."
echo "==> done"
else
echo "!! switch failed after a healthy test — running config is fine, boot default is not"
notify "Router apply ''${rev:0:8}: switch failed after a healthy test. Check the box."
exit 1
fi
if ${config.system.build.deploy}/bin/fred-deploy --flake "$flake"; then
${pkgs.coreutils}/bin/printf '%s' "$rev" > ${stateDir}/last-rev
${pkgs.coreutils}/bin/chown router-ui ${stateDir}/last-rev
notify "Router applied ''${rev:0:8}."
else
echo "!! health check failed — reverting to the previous generation"
/nix/var/nix/profiles/system/bin/switch-to-configuration test
notify "Router apply ''${rev:0:8} FAILED health check. Rolled back."
notify "Router apply ''${rev:0:8} failed. Check the apply journal and /var/lib/fred-deploy/last-result.json for recovery status."
exit 1
fi
'';
@ -160,13 +124,15 @@ in
# Deploy one specific commit. Instance name is the git rev.
systemd.services."router-apply@" = {
description = "Deploy router config %i (test, health check, then switch)";
path = [ pkgs.nixos-rebuild pkgs.nix pkgs.systemd "/run/current-system/sw" ];
description = "Deploy router config %i through the shared transaction";
restartIfChanged = false;
unitConfig.X-StopOnRemoval = false;
path = [ pkgs.systemd ];
environment.HOME = "/root";
serviceConfig = {
Type = "oneshot";
ExecStart = "${applyScript} %i";
TimeoutStartSec = "90min";
TimeoutStartSec = "105min";
};
};

14
services/router.nix
View file

@ -17,7 +17,8 @@
{ config, lib, pkgs, ... }:
let
portsData = builtins.fromTOML (builtins.readFile ../ports.toml);
portsData = (import ./validate-forwards.nix { inherit lib; })
(builtins.fromTOML (builtins.readFile ../ports.toml));
destDefault = portsData.dest_default;
devices = (builtins.fromTOML (builtins.readFile ../devices.toml)).device or [ ];
@ -95,6 +96,13 @@ let
(f: ''iifname "eno1" ${f.proto} dport ${f.port} accept comment "${f.name}"'')
forwards;
# Docker also creates DNAT rules. A DNAT flag alone is not permission to
# enter from WAN: check the published/original port, before translation.
# The declared Pelican allocation range remains automatic for new servers.
wanPortForwardRules = lib.concatMapStringsSep "\n "
(f: ''iifname "eno1" ct status dnat meta l4proto ${f.proto} ct original proto-dst ${f.port} accept comment "${f.name}"'')
forwards;
in
{
config = lib.mkIf (config.networking.hostName == "FredOS-Mediaserver") {
@ -221,8 +229,8 @@ in
# etc.). br-* = Docker custom networks, podman* = Podman rootful.
# pelican0 = wings' bridge; it does not match br-*.
iifname { "docker0", "br-*", "podman*", "pelican0" } accept
# WAN → any port-forward target (LAN host or docker container)
iifname "eno1" ct status dnat accept
# WAN → explicitly published ports only, including Docker DNAT.
${wanPortForwardRules}
}
chain output {
type filter hook output priority 0; policy accept;

6
services/sabnzbd.nix
View file

@ -50,10 +50,6 @@ in
systemd.tmpfiles.rules = [
"d /var/lib/sabnzbd 0755 sabnzbd media -"
"Z /var/lib/sabnzbd 0755 sabnzbd media -"
"d /mnt/storage/usenet/downloads 2775 sabnzbd media -"
"Z /mnt/storage/usenet/downloads 2775 sabnzbd media -"
"d /mnt/storage/usenet/incomplete 2775 sabnzbd media -"
"Z /mnt/storage/usenet/incomplete 2775 sabnzbd media -"
];
systemd.services.sabnzbd = {
@ -80,4 +76,4 @@ in
};
};
};
}
}

60
services/server-permissions.nix
View file

@ -1,11 +1,57 @@
# services/server-permissions.nix
{ config, pkgs, lib, ... }:
# Media writes are allowed only after the pool and all its branches mount.
# Keep these directory rules out of global tmpfiles: nofail mounts may be
# absent when global tmpfiles runs, and creating paths then fills the root disk.
{ config, pkgs, lib, utils, ... }:
let
pool = "/mnt/storage";
branches = config.fileSystems.${pool}.depends or [ ];
mounts = map (p: "${utils.escapeSystemdPath p}.mount") ([ pool ] ++ branches);
consumers = [
"jellyfin" "sonarr" "radarr" "bazarr" "qbittorrent-nox" "sabnzbd"
"docker-shelfarr" "arr-interconnect" "bazarr-sync" "bazarr-sync-full"
];
directories = pkgs.writeText "media-directories.conf" ''
d /mnt/storage/torrents/downloads 2775 qbittorrent media -
Z /mnt/storage/torrents/downloads 2775 qbittorrent media -
d /mnt/storage/torrents/shows 2775 sonarr media -
Z /mnt/storage/torrents/shows 2775 sonarr media -
d /mnt/storage/torrents/movies 2775 radarr media -
Z /mnt/storage/torrents/movies 2775 radarr media -
d /mnt/storage/torrents/audiobooks 2775 fred media -
Z /mnt/storage/torrents/audiobooks 2775 fred media -
d /mnt/storage/usenet/downloads 2775 sabnzbd media -
Z /mnt/storage/usenet/downloads 2775 sabnzbd media -
d /mnt/storage/usenet/incomplete 2775 sabnzbd media -
Z /mnt/storage/usenet/incomplete 2775 sabnzbd media -
'';
in
{
config = lib.mkIf (config.networking.hostName == "FredOS-Mediaserver") {
systemd.tmpfiles.rules = [
# Audiobooks - manually managed, no dedicated service yet
"d /mnt/storage/torrents/audiobooks 2775 fred media -"
"Z /mnt/storage/torrents/audiobooks 2775 fred media -"
];
systemd.services = {
media-storage-ready = {
description = "Prepare media directories on the mounted pool";
requires = mounts;
bindsTo = mounts;
after = mounts;
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
script = ''
set -euo pipefail
${lib.concatMapStringsSep "\n" (p: ''
${pkgs.util-linux}/bin/mountpoint -q ${lib.escapeShellArg p} || {
echo "Required media mount missing: ${p}" >&2
exit 1
}
'') ([ pool ] ++ branches)}
${pkgs.systemd}/bin/systemd-tmpfiles --create ${directories}
'';
};
} // lib.genAttrs consumers (_: {
requires = [ "media-storage-ready.service" ];
after = [ "media-storage-ready.service" ];
bindsTo = [ "media-storage-ready.service" ];
});
};
}

6
services/sonarr.nix
View file

@ -35,10 +35,6 @@
extraGroups = [ "media" ];
};
systemd.tmpfiles.rules = [
# Shows - sonarr organises, bazarr writes subtitles
"d /mnt/storage/torrents/shows 2775 sonarr media -"
"Z /mnt/storage/torrents/shows 2775 sonarr media -"
];
# Pool directories and mount dependencies live in server-permissions.nix.
};
}

41
services/validate-forwards.nix Normal file
View file

@ -0,0 +1,41 @@
# Validate hand-edited TOML as well as UI changes, before generating nft syntax.
{ lib }:
data:
let
fail = message: throw "ports.toml: ${message}";
lanIP = value: builtins.isString value && builtins.match
"10[.]0[.]0[.]([1-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-4])" value != null;
normalize = entry:
let
name = entry.name or "";
proto = entry.protocol or "";
single = entry ? port;
range = entry ? ports;
spec = if single then toString entry.port else entry.ports or "";
validSpec = builtins.isString spec && builtins.match "[0-9]+(-[0-9]+)?" spec != null;
bounds = map lib.toInt (lib.splitString "-" spec);
first = builtins.head bounds;
last = lib.last bounds;
dest = entry.dest or data.dest_default;
in
if !builtins.isString name || builtins.match "[A-Za-z0-9][A-Za-z0-9 _().-]{0,79}" name == null
then fail "invalid forward name"
else if !(lib.elem proto [ "tcp" "udp" "both" ]) then fail "${name}: invalid protocol"
else if single == range || !validSpec then fail "${name}: use one port or one port range"
else if (single && !builtins.isInt entry.port) || first < 1 || last > 65535 || (range && first >= last)
then fail "${name}: invalid port bounds"
else if !lanIP dest then fail "${name}: destination must be a LAN IPv4 address"
else { inherit name first last dest; protos = if proto == "both" then [ "tcp" "udp" ] else [ proto ]; };
entries = map normalize (data.forward or [ ]);
names = map (e: e.name) entries;
overlaps = a: b: a.first <= b.last && b.first <= a.last && lib.intersectLists a.protos b.protos != [ ];
noOverlaps = list: list == [ ] || (
lib.all (b: !overlaps (builtins.head list) b) (builtins.tail list)
&& noOverlaps (builtins.tail list));
reachable = port: lib.any (e: lib.elem "tcp" e.protos && e.first <= port && port <= e.last && e.dest == "10.0.0.1") entries;
in
if (data.dest_default or "") != "10.0.0.1" then fail "dest_default must remain 10.0.0.1"
else if builtins.length names != builtins.length (lib.unique names) then fail "duplicate names"
else if !noOverlaps entries then fail "overlapping protocol/port ranges"
else if !lib.all reachable [ 22 443 ] then fail "TCP 22 and 443 must still reach the router"
else builtins.deepSeq entries data

4
services/wan-watchdog.nix
View file

@ -21,6 +21,10 @@ let
# as "up". Two anycast targets so one being down isn't a false alarm.
watchdog = pkgs.writeShellScript "wan-watchdog" ''
set -uo pipefail
# Hold this for the whole check/recovery operation. Never restart the
# network or flush conntrack during activation, verification or rollback.
exec 9>/run/lock/fred-deploy.lock
${pkgs.util-linux}/bin/flock -n 9 || exit 0
wan=eno1
state=/var/lib/wan-watchdog
ipfile=$state/wan-ip