nixos/services/hardware-health.nix

# services/hardware-health.nix — RAS error attribution + watchdog auto-recovery
#
# Context: Jun 2026 the dual Xeon E5-2697 v3 began throwing a storm of
# *corrected* Machine Check Exceptions on both sockets (Bank 5 / Bank 20),
# ~18k events in 36h, eventually hanging the box. Since this host is the
# router, a hang takes the whole LAN offline until a manual power-cycle.
#
# This module:
#   - rasdaemon: decodes every MCE to a specific DIMM/channel/socket and
#     persists a per-component error DB, so a failing part can be named
#     (needed for the seller's warranty claim). Query with `ras-mc-ctl
#     --error-count` and `ras-mc-ctl --summary`.
#   - hardware watchdog: if userspace hangs again, systemd stops petting
#     /dev/watchdog0 and the chipset watchdog reboots the box (~30s),
#     restoring the LAN without physical access.
#   - mce-monitor: after the Jul 2026 CPU repaste/reseat, watch for the QPI
#     fault returning. Baselines at the current rasdaemon count (so only NEW
#     errors count) and pushes an ntfy alert on geometric thresholds — the
#     1st, 10th, 100th, 1000th... new error — so the first one pings
#     immediately and a storm spaces out instead of thousands of pushes.
#     Reuses the /var/secrets/ntfy-url topic (see services/service-health.nix).

{ config, lib, pkgs, ... }:
let
  # Watches TWO rasdaemon error classes independently, each with its own
  # geometric-threshold alerting (1/10/100/... new since baseline):
  #   - MCE / QPI  (rows carry "bank=0x")        -> the CPU interconnect fault
  #   - disk I/O   (rows carry "error='I/O error'") -> failing drive OR SATA link
  # rasdaemon lumps both in `--errors`; counting them together used to make a
  # disk glitch masquerade as "QPI returned", so we grep each class separately.
  # State: "mce_base mce_last disk_base disk_last" (old 2-field files re-baseline).
  mceMonitor = pkgs.writeShellScript "mce-monitor" ''
    set -uo pipefail
    host="${config.networking.hostName}"
    secret=/var/secrets/ntfy-url
    state=/var/lib/mce-monitor/state
    rasctl=${pkgs.rasdaemon}/bin/ras-mc-ctl
    grep=${pkgs.gnugrep}/bin/grep
    tail=${pkgs.coreutils}/bin/tail
    cut=${pkgs.coreutils}/bin/cut

    errs=$($rasctl --errors 2>/dev/null || true)
    mce_cur=$(printf '%s\n' "$errs" | $grep -cE 'bank=0x' || true)
    disk_cur=$(printf '%s\n' "$errs" | $grep -cF "error='I/O error'" || true)

    # First run (or migration from the old 2-field state): baseline, no alert.
    if [ ! -f "$state" ] || [ "$(${pkgs.coreutils}/bin/wc -w < "$state")" -ne 4 ]; then
      echo "$mce_cur 0 $disk_cur 0" > "$state"
      exit 0
    fi
    read mce_base mce_last disk_base disk_last < "$state"

    # Counter shrank (DB wiped) -> re-baseline that class, don't alert.
    [ "$mce_cur"  -lt "$mce_base"  ] && { mce_base=$mce_cur;   mce_last=0; }
    [ "$disk_cur" -lt "$disk_base" ] && { disk_base=$disk_cur; disk_last=0; }

    notify() { # prio tag body
      [ -f "$secret" ] || return 0
      url=$(${pkgs.coreutils}/bin/tr -d '\n' < "$secret")
      ${pkgs.curl}/bin/curl -fsS --max-time 10 \
        -H "Title: $host hardware alert" -H "Priority: $1" -H "Tags: $2" \
        -d "$3" "$url" >/dev/null 2>&1 || true
    }
    bucket() { # new -> highest geometric threshold crossed
      local n=$1 c=0 t
      for t in 1 10 100 1000 10000 100000; do [ "$n" -ge "$t" ] && c=$t; done
      echo "$c"
    }

    # --- MCE / QPI ---
    mce_new=$(( mce_cur - mce_base ))
    if [ "$mce_new" -ge 1 ]; then
      b=$(bucket "$mce_new")
      if [ "$b" -gt "$mce_last" ]; then
        newest=$(printf '%s\n' "$errs" | $grep -E 'bank=0x' | $tail -1 | $cut -d' ' -f2-4)
        if [ "$mce_new" -ge 1000 ]; then p=urgent; else p=high; fi
        echo "mce-monitor: $mce_new new MCE/QPI errors (newest $newest)"
        notify "$p" rotating_light "$mce_new new machine-check (QPI) errors since the Jul repaste. Newest: $newest"
        mce_last=$b
      fi
    fi

    # --- disk I/O / SATA ---
    disk_new=$(( disk_cur - disk_base ))
    if [ "$disk_new" -ge 1 ]; then
      b=$(bucket "$disk_new")
      if [ "$b" -gt "$disk_last" ]; then
        dev=$(printf '%s\n' "$errs" | $grep -F "error='I/O error'" | $tail -1 | $grep -oE 'dev=[0-9]+:[0-9]+' | $tail -1)
        echo "mce-monitor: $disk_new new disk I/O errors ($dev)"
        notify high floppy_disk "$disk_new new disk I/O errors on $host ($dev). Check SATA cable / SMART."
        disk_last=$b
      fi
    fi

    echo "$mce_base $mce_last $disk_base $disk_last" > "$state"
  '';
in
{
  config = lib.mkIf (config.networking.hostName == "FredOS-Mediaserver") {

    # Decode + log + persist machine-check / memory errors per component.
    hardware.rasdaemon.enable = true;

    # ras-mc-ctl on PATH for manual inspection.
    environment.systemPackages = [ pkgs.rasdaemon ];

    # Hardware watchdog: auto-reboot a hung box instead of a dead LAN.
    # systemd pets /dev/watchdog0 at half the runtime interval; if it stops
    # (hang), the chipset resets after RuntimeWatchdogSec.
    systemd.settings.Manager = {
      RuntimeWatchdogSec = "30s";
      RebootWatchdogSec = "10min";
    };

    # Watch for the QPI fault returning; ntfy on increasing thresholds.
    systemd.services.mce-monitor = {
      description = "Alert (ntfy) on new machine-check/QPI errors";
      serviceConfig = {
        Type = "oneshot";
        ExecStart = mceMonitor;
        StateDirectory = "mce-monitor";
      };
    };
    systemd.timers.mce-monitor = {
      wantedBy = [ "timers.target" ];
      timerConfig = {
        OnBootSec = "3min";
        OnUnitActiveSec = "5min";
        Persistent = true;
      };
    };
  };
}