diff --git a/services/valheim-autoupdate.nix b/services/valheim-autoupdate.nix index 9073489..896db61 100644 --- a/services/valheim-autoupdate.nix +++ b/services/valheim-autoupdate.nix @@ -1,14 +1,20 @@ # services/valheim-autoupdate.nix — restart the Valheim server only when Steam -# actually ships a new build. +# actually ships a new build, and only when nobody is playing. # # The Pelican egg already has AUTO_UPDATE=1, so steamcmd runs on every boot and # a restart *is* the update. The missing half is knowing when to bother, which -# a plain nightly restart doesn't answer (it reboots players for nothing 364 -# nights out of 365). So, hourly: +# a nightly restart doesn't answer (it reboots players for nothing 364 nights +# out of 365). So, hourly: # # 1. installed buildid <- steamapps/appmanifest_896660.acf in the server volume # 2. latest buildid <- the container's own steamcmd, public branch -# 3. differ, nobody playing -> POST restart to the wings daemon API +# 3. same -> exit. differ -> wait for an empty server, then restart. +# +# Waiting is a 10s poll held inside this one invocation rather than a re-check +# on the next hourly tick, because when Valve ships a build Steam auto-updates +# the *clients* — so everyone not currently online is locked out by a network +# version mismatch until the server catches up. Waiting an extra hour after the +# last player leaves is the expensive case, so we don't. # # Why the wings API and not `docker restart`: the egg's stop command is ^C, so # wings sends SIGINT and Valheim saves the world before exiting. docker's @@ -17,8 +23,7 @@ # nothing to rotate by hand. # # Valheim has no console, no RCON and no broadcast command, so players cannot be -# warned in-game — the only honest options are "restart on an empty server" or -# "kick people mid-raid". This does the former and pushes an ntfy note instead. +# warned in-game at all. ntfy gets the play-by-play instead. { config, lib, pkgs, ... }: let # Pelican server "hawk" (panel server id 3). Both the volume path and the @@ -26,6 +31,17 @@ let serverUuid = "185008d8-8a04-4077-be78-63853dc06c0e"; steamAppId = "896660"; # valheim_server; 892970 is the client gamePort = "25569"; + # Valheim's Steam query port is always game port + 1, and wings publishes it + # on the LAN address, not localhost. + queryHost = "10.0.0.1"; + queryPort = "25570"; + + # How long to keep waiting for a busy server before giving up and letting the + # next hourly tick start over. Long enough for a full evening of play. + maxWaitSeconds = "21600"; # 6h + # Boot budget after the restart: steamcmd pulls the update, then Valheim + # loads the world (~40s on top). + bootTries = "90"; # x10s = 15min autoupdate = pkgs.writeShellScript "valheim-autoupdate" '' set -uo pipefail @@ -34,12 +50,17 @@ let appid=${steamAppId} manifest=/var/lib/pelican/volumes/$uuid/steamapps/appmanifest_$appid.acf wings_cfg=/etc/pelican/config.yml - state=/var/lib/valheim-autoupdate/announced secret=/var/secrets/ntfy-url + reply=/run/valheim-autoupdate/a2s-reply docker=${pkgs.docker}/bin/docker awk=${pkgs.gawk}/bin/awk curl=${pkgs.curl}/bin/curl + grep=${pkgs.gnugrep}/bin/grep + nc=${pkgs.netcat-openbsd}/bin/nc + sleep=${pkgs.coreutils}/bin/sleep + date=${pkgs.coreutils}/bin/date + timeout=${pkgs.coreutils}/bin/timeout notify() { # body [ -f "$secret" ] || return 0 @@ -49,11 +70,35 @@ let -d "$1" "$url" >/dev/null 2>&1 || true } + running() { + [ "$($docker inspect -f '{{.State.Running}}' "$uuid" 2>/dev/null)" = "true" ] + } + + # ponytail: UDP conntrack entries for the game port are the cheap stand-in + # for a player count — no A2S challenge handshake to parse. They linger + # ~2min past the last packet, so this reads as "played very recently", + # which errs toward leaving people alone. Upgrade path if that's ever too + # coarse: A2S_INFO the query port and read the players byte. + players_online() { + $grep -q "dport=${gamePort} " /proc/net/nf_conntrack 2>/dev/null + } + + # The server browser's own liveness test: anything at all back from the + # query port means Valheim is up and joinable. The first reply is a Steam + # challenge, which is enough — no need to complete the handshake. + query_answers() { + printf '\xff\xff\xff\xffTSource Engine Query\x00' \ + | $timeout 3 $nc -u -w2 ${queryHost} ${queryPort} > "$reply" 2>/dev/null || true + [ -s "$reply" ] + } + + installed_build() { $awk -F'"' '/"buildid"/{print $4; exit}' "$manifest" 2>/dev/null; } + # Nothing to restart, and no steamcmd to ask — a stopped server updates # itself whenever it next boots anyway. - [ "$($docker inspect -f '{{.State.Running}}' "$uuid" 2>/dev/null)" = "true" ] || exit 0 + running || exit 0 - installed=$($awk -F'"' '/"buildid"/{print $4; exit}' "$manifest" 2>/dev/null) + installed=$(installed_build) latest=$($docker exec "$uuid" \ ./steamcmd/steamcmd.sh +login anonymous +app_info_update 1 \ +app_info_print "$appid" +quit 2>/dev/null \ @@ -65,21 +110,22 @@ let case "$installed" in ""|*[!0-9]*) exit 0 ;; esac [ "$installed" = "$latest" ] && exit 0 - # ponytail: UDP conntrack entries for the game port are the cheap stand-in - # for a player count — no A2S challenge handshake to parse. They linger - # ~2min past the last packet, so this reads as "played very recently", - # which errs toward leaving people alone. Upgrade path if that's ever too - # coarse: query A2S_INFO on the query port (game port + 1) and read the - # players byte. - if ${pkgs.gnugrep}/bin/grep -q "dport=${gamePort} " /proc/net/nf_conntrack 2>/dev/null; then - if [ "$(${pkgs.coreutils}/bin/cat "$state" 2>/dev/null || true)" != "$latest" ]; then - notify "Valheim build $installed -> $latest is out. Someone is online, so the update waits until the server is empty." - ${pkgs.coreutils}/bin/printf '%s' "$latest" > "$state" - fi - echo "valheim-autoupdate: $installed -> $latest available, players connected, deferring" - exit 0 + if players_online; then + notify "Valheim build $installed -> $latest is out. Waiting for the server to empty before updating." + echo "valheim-autoupdate: $installed -> $latest available, players connected, waiting" + deadline=$(( $($date +%s) + ${maxWaitSeconds} )) + while players_online; do + if [ "$($date +%s)" -ge "$deadline" ]; then + echo "valheim-autoupdate: still busy after ${maxWaitSeconds}s, leaving it for the next run" + exit 0 + fi + $sleep 10 + done fi + # It may have been stopped by hand while we waited. + running || exit 0 + token=$($awk '/^token:/{print $2}' "$wings_cfg") if [ -z "$token" ]; then echo "valheim-autoupdate: no wings token in $wings_cfg, cannot restart" >&2 @@ -87,16 +133,35 @@ let fi echo "valheim-autoupdate: $installed -> $latest, server empty, restarting" - if $curl -fsS --max-time 30 -X POST \ + notify "Installing Valheim build $installed -> $latest. Server restarting now." + if ! $curl -fsS --max-time 30 -X POST \ -H "Authorization: Bearer $token" -H 'Content-Type: application/json' \ -d '{"action":"restart"}' \ "http://127.0.0.1:8443/api/servers/$uuid/power" >/dev/null; then - notify "Restarted for Valheim build $installed -> $latest." - ${pkgs.coreutils}/bin/printf '%s' "$latest" > "$state" - else + notify "Valheim restart request failed — server left on build $installed." echo "valheim-autoupdate: wings power request failed" >&2 exit 1 fi + + # Done when the manifest shows a different build than we started on (proves + # steamcmd finished) *and* the query port answers (proves it's joinable). + # Compared against $installed rather than $latest so an even newer build + # landing mid-wait still counts as success. + i=0 + while [ "$i" -lt ${bootTries} ]; do + i=$(( i + 1 )) + $sleep 10 + now_build=$(installed_build) + [ -n "$now_build" ] && [ "$now_build" != "$installed" ] || continue + query_answers || continue + echo "valheim-autoupdate: back up on build $now_build" + notify "Valheim is back up on build $now_build." + exit 0 + done + + notify "Valheim restart for build $latest was sent, but it has not come back after 15 min. Check the panel." + echo "valheim-autoupdate: server did not come back in time" >&2 + exit 1 ''; in { @@ -107,9 +172,10 @@ in serviceConfig = { Type = "oneshot"; ExecStart = autoupdate; - StateDirectory = "valheim-autoupdate"; - # steamcmd's app_info_update round-trip is the slow part (~1min worst case). - TimeoutStartSec = "5min"; + RuntimeDirectory = "valheim-autoupdate"; + # The script holds itself open while waiting for players to leave, so + # this is only a backstop above its own 6h deadline + boot budget. + TimeoutStartSec = "7h"; }; }; systemd.timers.valheim-autoupdate = {