nixos/services/hardware-health.nix

# services/hardware-health.nix — RAS error attribution + watchdog auto-recovery
#
# Context: Jun 2026 the dual Xeon E5-2697 v3 began throwing a storm of
# *corrected* Machine Check Exceptions on both sockets (Bank 5 / Bank 20),
# ~18k events in 36h, eventually hanging the box. Since this host is the
# router, a hang takes the whole LAN offline until a manual power-cycle.
#
# This module:
#   - rasdaemon: decodes every MCE to a specific DIMM/channel/socket and
#     persists a per-component error DB, so a failing part can be named
#     (needed for the seller's warranty claim). Query with `ras-mc-ctl
#     --error-count` and `ras-mc-ctl --summary`.
#   - hardware watchdog: if userspace hangs again, systemd stops petting
#     /dev/watchdog0 and the chipset watchdog reboots the box (~30s),
#     restoring the LAN without physical access.
#   - mce-monitor: after the Jul 2026 CPU repaste/reseat, watch for the QPI
#     fault returning. Baselines at the current rasdaemon count (so only NEW
#     errors count) and pushes an ntfy alert on geometric thresholds — the
#     1st, 10th, 100th, 1000th... new error — so the first one pings
#     immediately and a storm spaces out instead of thousands of pushes.
#     Reuses the /var/secrets/ntfy-url topic (see services/service-health.nix).

{ config, lib, pkgs, ... }:
let
  mceMonitor = pkgs.writeShellScript "mce-monitor" ''
    set -uo pipefail
    host="${config.networking.hostName}"
    secret=/var/secrets/ntfy-url
    state=/var/lib/mce-monitor/state
    rasctl=${pkgs.rasdaemon}/bin/ras-mc-ctl
    grep=${pkgs.gnugrep}/bin/grep

    # Cumulative decoded-MCE count from rasdaemon's DB (persists across reboots).
    current=$($rasctl --errors 2>/dev/null | $grep -cE '^[0-9]+ 20' || echo 0)

    # First run: record baseline, alert on nothing historical.
    if [ ! -f "$state" ]; then
      echo "$current 0" > "$state"
      exit 0
    fi
    read baseline last < "$state"

    # DB wiped/shrank -> re-baseline, don't alert.
    if [ "$current" -lt "$baseline" ]; then
      echo "$current 0" > "$state"
      exit 0
    fi

    new=$(( current - baseline ))
    [ "$new" -lt 1 ] && exit 0

    # Highest geometric threshold crossed by the new-error count.
    crossed=0
    for t in 1 10 100 1000 10000 100000; do
      [ "$new" -ge "$t" ] && crossed=$t
    done
    # Already alerted at this bucket -> stay quiet (increasing intervals).
    [ "$crossed" -le "$last" ] && exit 0

    if [ ! -f "$secret" ]; then
      echo "mce-monitor: $secret missing; cannot notify" >&2
      exit 0
    fi
    url=$(${pkgs.coreutils}/bin/tr -d '\n' < "$secret")
    newest=$($rasctl --errors 2>/dev/null | $grep -E '^[0-9]+ 20' \
      | ${pkgs.coreutils}/bin/tail -1 | ${pkgs.coreutils}/bin/cut -d' ' -f2-4)
    if [ "$new" -ge 1000 ]; then prio=urgent; tag=rotating_light; else prio=high; tag=warning; fi

    ${pkgs.curl}/bin/curl -fsS --max-time 10 \
      -H "Title: MCE/QPI errors returned on $host" -H "Priority: $prio" -H "Tags: $tag" \
      -d "$new new machine-check (QPI) errors since the Jul repaste. Newest: $newest" \
      "$url" >/dev/null 2>&1 || true

    echo "$baseline $crossed" > "$state"
  '';
in
{
  config = lib.mkIf (config.networking.hostName == "FredOS-Mediaserver") {

    # Decode + log + persist machine-check / memory errors per component.
    hardware.rasdaemon.enable = true;

    # ras-mc-ctl on PATH for manual inspection.
    environment.systemPackages = [ pkgs.rasdaemon ];

    # Hardware watchdog: auto-reboot a hung box instead of a dead LAN.
    # systemd pets /dev/watchdog0 at half the runtime interval; if it stops
    # (hang), the chipset resets after RuntimeWatchdogSec.
    systemd.settings.Manager = {
      RuntimeWatchdogSec = "30s";
      RebootWatchdogSec = "10min";
    };

    # Watch for the QPI fault returning; ntfy on increasing thresholds.
    systemd.services.mce-monitor = {
      description = "Alert (ntfy) on new machine-check/QPI errors";
      serviceConfig = {
        Type = "oneshot";
        ExecStart = mceMonitor;
        StateDirectory = "mce-monitor";
      };
    };
    systemd.timers.mce-monitor = {
      wantedBy = [ "timers.target" ];
      timerConfig = {
        OnBootSec = "3min";
        OnUnitActiveSec = "5min";
        Persistent = true;
      };
    };
  };
}