valheim: wait for an empty server, notify on install and recovery

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

services/valheim-autoupdate.nix

@@ -1,14 +1,20 @@
 # services/valheim-autoupdate.nix — restart the Valheim server only when Steam
-# actually ships a new build.
+# actually ships a new build, and only when nobody is playing.
 #
 # The Pelican egg already has AUTO_UPDATE=1, so steamcmd runs on every boot and
 # a restart *is* the update. The missing half is knowing when to bother, which
-# a plain nightly restart doesn't answer (it reboots players for nothing 364
+# a nightly restart doesn't answer (it reboots players for nothing 364 nights
-# nights out of 365). So, hourly:
+# out of 365). So, hourly:
 #
 #   1. installed buildid  <- steamapps/appmanifest_896660.acf in the server volume
 #   2. latest buildid     <- the container's own steamcmd, public branch
-#   3. differ, nobody playing -> POST restart to the wings daemon API
+#   3. same -> exit. differ -> wait for an empty server, then restart.
+#
+# Waiting is a 10s poll held inside this one invocation rather than a re-check
+# on the next hourly tick, because when Valve ships a build Steam auto-updates
+# the *clients* — so everyone not currently online is locked out by a network
+# version mismatch until the server catches up. Waiting an extra hour after the
+# last player leaves is the expensive case, so we don't.
 #
 # Why the wings API and not `docker restart`: the egg's stop command is ^C, so
 # wings sends SIGINT and Valheim saves the world before exiting. docker's
@@ -17,8 +23,7 @@
 # nothing to rotate by hand.
 #
 # Valheim has no console, no RCON and no broadcast command, so players cannot be
-# warned in-game — the only honest options are "restart on an empty server" or
+# warned in-game at all. ntfy gets the play-by-play instead.
-# "kick people mid-raid". This does the former and pushes an ntfy note instead.
 { config, lib, pkgs, ... }:
 let
   # Pelican server "hawk" (panel server id 3). Both the volume path and the
@@ -26,6 +31,17 @@ let
   serverUuid = "185008d8-8a04-4077-be78-63853dc06c0e";
   steamAppId = "896660"; # valheim_server; 892970 is the client
   gamePort = "25569";
+  # Valheim's Steam query port is always game port + 1, and wings publishes it
+  # on the LAN address, not localhost.
+  queryHost = "10.0.0.1";
+  queryPort = "25570";
+
+  # How long to keep waiting for a busy server before giving up and letting the
+  # next hourly tick start over. Long enough for a full evening of play.
+  maxWaitSeconds = "21600"; # 6h
+  # Boot budget after the restart: steamcmd pulls the update, then Valheim
+  # loads the world (~40s on top).
+  bootTries = "90"; # x10s = 15min
 
   autoupdate = pkgs.writeShellScript "valheim-autoupdate" ''
     set -uo pipefail
@@ -34,12 +50,17 @@ let
     appid=${steamAppId}
     manifest=/var/lib/pelican/volumes/$uuid/steamapps/appmanifest_$appid.acf
     wings_cfg=/etc/pelican/config.yml
-    state=/var/lib/valheim-autoupdate/announced
     secret=/var/secrets/ntfy-url
+    reply=/run/valheim-autoupdate/a2s-reply
 
     docker=${pkgs.docker}/bin/docker
     awk=${pkgs.gawk}/bin/awk
     curl=${pkgs.curl}/bin/curl
+    grep=${pkgs.gnugrep}/bin/grep
+    nc=${pkgs.netcat-openbsd}/bin/nc
+    sleep=${pkgs.coreutils}/bin/sleep
+    date=${pkgs.coreutils}/bin/date
+    timeout=${pkgs.coreutils}/bin/timeout
 
     notify() { # body
       [ -f "$secret" ] || return 0
@@ -49,11 +70,35 @@ let
         -d "$1" "$url" >/dev/null 2>&1 || true
     }
 
+    running() {
+      [ "$($docker inspect -f '{{.State.Running}}' "$uuid" 2>/dev/null)" = "true" ]
+    }
+
+    # ponytail: UDP conntrack entries for the game port are the cheap stand-in
+    # for a player count — no A2S challenge handshake to parse. They linger
+    # ~2min past the last packet, so this reads as "played very recently",
+    # which errs toward leaving people alone. Upgrade path if that's ever too
+    # coarse: A2S_INFO the query port and read the players byte.
+    players_online() {
+      $grep -q "dport=${gamePort} " /proc/net/nf_conntrack 2>/dev/null
+    }
+
+    # The server browser's own liveness test: anything at all back from the
+    # query port means Valheim is up and joinable. The first reply is a Steam
+    # challenge, which is enough — no need to complete the handshake.
+    query_answers() {
+      printf '\xff\xff\xff\xffTSource Engine Query\x00' \
+        | $timeout 3 $nc -u -w2 ${queryHost} ${queryPort} > "$reply" 2>/dev/null || true
+      [ -s "$reply" ]
+    }
+
+    installed_build() { $awk -F'"' '/"buildid"/{print $4; exit}' "$manifest" 2>/dev/null; }
+
     # Nothing to restart, and no steamcmd to ask — a stopped server updates
     # itself whenever it next boots anyway.
-    [ "$($docker inspect -f '{{.State.Running}}' "$uuid" 2>/dev/null)" = "true" ] || exit 0
+    running || exit 0
 
-    installed=$($awk -F'"' '/"buildid"/{print $4; exit}' "$manifest" 2>/dev/null)
+    installed=$(installed_build)
     latest=$($docker exec "$uuid" \
       ./steamcmd/steamcmd.sh +login anonymous +app_info_update 1 \
         +app_info_print "$appid" +quit 2>/dev/null \
@@ -65,20 +110,21 @@ let
     case "$installed" in ""|*[!0-9]*) exit 0 ;; esac
     [ "$installed" = "$latest" ] && exit 0
 
-    # ponytail: UDP conntrack entries for the game port are the cheap stand-in
+    if players_online; then
-    # for a player count — no A2S challenge handshake to parse. They linger
+      notify "Valheim build $installed -> $latest is out. Waiting for the server to empty before updating."
-    # ~2min past the last packet, so this reads as "played very recently",
+      echo "valheim-autoupdate: $installed -> $latest available, players connected, waiting"
-    # which errs toward leaving people alone. Upgrade path if that's ever too
+      deadline=$(( $($date +%s) + ${maxWaitSeconds} ))
-    # coarse: query A2S_INFO on the query port (game port + 1) and read the
+      while players_online; do
-    # players byte.
+        if [ "$($date +%s)" -ge "$deadline" ]; then
-    if ${pkgs.gnugrep}/bin/grep -q "dport=${gamePort} " /proc/net/nf_conntrack 2>/dev/null; then
+          echo "valheim-autoupdate: still busy after ${maxWaitSeconds}s, leaving it for the next run"
-      if [ "$(${pkgs.coreutils}/bin/cat "$state" 2>/dev/null || true)" != "$latest" ]; then
-        notify "Valheim build $installed -> $latest is out. Someone is online, so the update waits until the server is empty."
-        ${pkgs.coreutils}/bin/printf '%s' "$latest" > "$state"
-      fi
-      echo "valheim-autoupdate: $installed -> $latest available, players connected, deferring"
           exit 0
         fi
+        $sleep 10
+      done
+    fi
+
+    # It may have been stopped by hand while we waited.
+    running || exit 0
 
     token=$($awk '/^token:/{print $2}' "$wings_cfg")
     if [ -z "$token" ]; then
@@ -87,16 +133,35 @@ let
     fi
 
     echo "valheim-autoupdate: $installed -> $latest, server empty, restarting"
-    if $curl -fsS --max-time 30 -X POST \
+    notify "Installing Valheim build $installed -> $latest. Server restarting now."
+    if ! $curl -fsS --max-time 30 -X POST \
         -H "Authorization: Bearer $token" -H 'Content-Type: application/json' \
         -d '{"action":"restart"}' \
         "http://127.0.0.1:8443/api/servers/$uuid/power" >/dev/null; then
-      notify "Restarted for Valheim build $installed -> $latest."
+      notify "Valheim restart request failed — server left on build $installed."
-      ${pkgs.coreutils}/bin/printf '%s' "$latest" > "$state"
-    else
       echo "valheim-autoupdate: wings power request failed" >&2
       exit 1
     fi
+
+    # Done when the manifest shows a different build than we started on (proves
+    # steamcmd finished) *and* the query port answers (proves it's joinable).
+    # Compared against $installed rather than $latest so an even newer build
+    # landing mid-wait still counts as success.
+    i=0
+    while [ "$i" -lt ${bootTries} ]; do
+      i=$(( i + 1 ))
+      $sleep 10
+      now_build=$(installed_build)
+      [ -n "$now_build" ] && [ "$now_build" != "$installed" ] || continue
+      query_answers || continue
+      echo "valheim-autoupdate: back up on build $now_build"
+      notify "Valheim is back up on build $now_build."
+      exit 0
+    done
+
+    notify "Valheim restart for build $latest was sent, but it has not come back after 15 min. Check the panel."
+    echo "valheim-autoupdate: server did not come back in time" >&2
+    exit 1
   '';
 in
 {
@@ -107,9 +172,10 @@ in
       serviceConfig = {
         Type = "oneshot";
         ExecStart = autoupdate;
-        StateDirectory = "valheim-autoupdate";
+        RuntimeDirectory = "valheim-autoupdate";
-        # steamcmd's app_info_update round-trip is the slow part (~1min worst case).
+        # The script holds itself open while waiting for players to leave, so
-        TimeoutStartSec = "5min";
+        # this is only a backstop above its own 6h deadline + boot budget.
+        TimeoutStartSec = "7h";
       };
     };
     systemd.timers.valheim-autoupdate = {