diff --git a/services/hardware-health.nix b/services/hardware-health.nix index 6d2d525..a205cdb 100644 --- a/services/hardware-health.nix +++ b/services/hardware-health.nix @@ -13,8 +13,67 @@ # - hardware watchdog: if userspace hangs again, systemd stops petting # /dev/watchdog0 and the chipset watchdog reboots the box (~30s), # restoring the LAN without physical access. +# - mce-monitor: after the Jul 2026 CPU repaste/reseat, watch for the QPI +# fault returning. Baselines at the current rasdaemon count (so only NEW +# errors count) and pushes an ntfy alert on geometric thresholds — the +# 1st, 10th, 100th, 1000th... new error — so the first one pings +# immediately and a storm spaces out instead of thousands of pushes. +# Reuses the /var/secrets/ntfy-url topic (see services/service-health.nix). { config, lib, pkgs, ... }: +let + mceMonitor = pkgs.writeShellScript "mce-monitor" '' + set -uo pipefail + host="${config.networking.hostName}" + secret=/var/secrets/ntfy-url + state=/var/lib/mce-monitor/state + rasctl=${pkgs.rasdaemon}/bin/ras-mc-ctl + grep=${pkgs.gnugrep}/bin/grep + + # Cumulative decoded-MCE count from rasdaemon's DB (persists across reboots). + current=$($rasctl --errors 2>/dev/null | $grep -cE '^[0-9]+ 20' || echo 0) + + # First run: record baseline, alert on nothing historical. + if [ ! -f "$state" ]; then + echo "$current 0" > "$state" + exit 0 + fi + read baseline last < "$state" + + # DB wiped/shrank -> re-baseline, don't alert. + if [ "$current" -lt "$baseline" ]; then + echo "$current 0" > "$state" + exit 0 + fi + + new=$(( current - baseline )) + [ "$new" -lt 1 ] && exit 0 + + # Highest geometric threshold crossed by the new-error count. + crossed=0 + for t in 1 10 100 1000 10000 100000; do + [ "$new" -ge "$t" ] && crossed=$t + done + # Already alerted at this bucket -> stay quiet (increasing intervals). + [ "$crossed" -le "$last" ] && exit 0 + + if [ ! -f "$secret" ]; then + echo "mce-monitor: $secret missing; cannot notify" >&2 + exit 0 + fi + url=$(${pkgs.coreutils}/bin/tr -d '\n' < "$secret") + newest=$($rasctl --errors 2>/dev/null | $grep -E '^[0-9]+ 20' \ + | ${pkgs.coreutils}/bin/tail -1 | ${pkgs.coreutils}/bin/cut -d' ' -f2-4) + if [ "$new" -ge 1000 ]; then prio=urgent; tag=rotating_light; else prio=high; tag=warning; fi + + ${pkgs.curl}/bin/curl -fsS --max-time 10 \ + -H "Title: MCE/QPI errors returned on $host" -H "Priority: $prio" -H "Tags: $tag" \ + -d "$new new machine-check (QPI) errors since the Jul repaste. Newest: $newest" \ + "$url" >/dev/null 2>&1 || true + + echo "$baseline $crossed" > "$state" + ''; +in { config = lib.mkIf (config.networking.hostName == "FredOS-Mediaserver") { @@ -31,5 +90,23 @@ RuntimeWatchdogSec = "30s"; RebootWatchdogSec = "10min"; }; + + # Watch for the QPI fault returning; ntfy on increasing thresholds. + systemd.services.mce-monitor = { + description = "Alert (ntfy) on new machine-check/QPI errors"; + serviceConfig = { + Type = "oneshot"; + ExecStart = mceMonitor; + StateDirectory = "mce-monitor"; + }; + }; + systemd.timers.mce-monitor = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnBootSec = "3min"; + OnUnitActiveSec = "5min"; + Persistent = true; + }; + }; }; }