hardware-health: ntfy alert on new MCE/QPI errors, geometric thresholds

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

services/hardware-health.nix

@@ -13,8 +13,67 @@
 #   - hardware watchdog: if userspace hangs again, systemd stops petting
 #     /dev/watchdog0 and the chipset watchdog reboots the box (~30s),
 #     restoring the LAN without physical access.
+#   - mce-monitor: after the Jul 2026 CPU repaste/reseat, watch for the QPI
+#     fault returning. Baselines at the current rasdaemon count (so only NEW
+#     errors count) and pushes an ntfy alert on geometric thresholds — the
+#     1st, 10th, 100th, 1000th... new error — so the first one pings
+#     immediately and a storm spaces out instead of thousands of pushes.
+#     Reuses the /var/secrets/ntfy-url topic (see services/service-health.nix).
 
 { config, lib, pkgs, ... }:
+let
+  mceMonitor = pkgs.writeShellScript "mce-monitor" ''
+    set -uo pipefail
+    host="${config.networking.hostName}"
+    secret=/var/secrets/ntfy-url
+    state=/var/lib/mce-monitor/state
+    rasctl=${pkgs.rasdaemon}/bin/ras-mc-ctl
+    grep=${pkgs.gnugrep}/bin/grep
+
+    # Cumulative decoded-MCE count from rasdaemon's DB (persists across reboots).
+    current=$($rasctl --errors 2>/dev/null | $grep -cE '^[0-9]+ 20' || echo 0)
+
+    # First run: record baseline, alert on nothing historical.
+    if [ ! -f "$state" ]; then
+      echo "$current 0" > "$state"
+      exit 0
+    fi
+    read baseline last < "$state"
+
+    # DB wiped/shrank -> re-baseline, don't alert.
+    if [ "$current" -lt "$baseline" ]; then
+      echo "$current 0" > "$state"
+      exit 0
+    fi
+
+    new=$(( current - baseline ))
+    [ "$new" -lt 1 ] && exit 0
+
+    # Highest geometric threshold crossed by the new-error count.
+    crossed=0
+    for t in 1 10 100 1000 10000 100000; do
+      [ "$new" -ge "$t" ] && crossed=$t
+    done
+    # Already alerted at this bucket -> stay quiet (increasing intervals).
+    [ "$crossed" -le "$last" ] && exit 0
+
+    if [ ! -f "$secret" ]; then
+      echo "mce-monitor: $secret missing; cannot notify" >&2
+      exit 0
+    fi
+    url=$(${pkgs.coreutils}/bin/tr -d '\n' < "$secret")
+    newest=$($rasctl --errors 2>/dev/null | $grep -E '^[0-9]+ 20' \
+      | ${pkgs.coreutils}/bin/tail -1 | ${pkgs.coreutils}/bin/cut -d' ' -f2-4)
+    if [ "$new" -ge 1000 ]; then prio=urgent; tag=rotating_light; else prio=high; tag=warning; fi
+
+    ${pkgs.curl}/bin/curl -fsS --max-time 10 \
+      -H "Title: MCE/QPI errors returned on $host" -H "Priority: $prio" -H "Tags: $tag" \
+      -d "$new new machine-check (QPI) errors since the Jul repaste. Newest: $newest" \
+      "$url" >/dev/null 2>&1 || true
+
+    echo "$baseline $crossed" > "$state"
+  '';
+in
 {
   config = lib.mkIf (config.networking.hostName == "FredOS-Mediaserver") {
 
@@ -31,5 +90,23 @@
       RuntimeWatchdogSec = "30s";
       RebootWatchdogSec = "10min";
     };
+
+    # Watch for the QPI fault returning; ntfy on increasing thresholds.
+    systemd.services.mce-monitor = {
+      description = "Alert (ntfy) on new machine-check/QPI errors";
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = mceMonitor;
+        StateDirectory = "mce-monitor";
+      };
+    };
+    systemd.timers.mce-monitor = {
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnBootSec = "3min";
+        OnUnitActiveSec = "5min";
+        Persistent = true;
+      };
+    };
   };
 }