mce-monitor: split MCE/QPI vs disk I/O error tracking + separate alerts

A disk SATA glitch was miscounted as new MCEs and paged as 'QPI returned'.
Now counts bank=0x (MCE) and I/O-error (disk) rows independently, each with
its own threshold alert. Old 2-field state auto-re-baselines.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

services/hardware-health.nix

@@ -22,6 +22,13 @@
 
 { config, lib, pkgs, ... }:
 let
+  # Watches TWO rasdaemon error classes independently, each with its own
+  # geometric-threshold alerting (1/10/100/... new since baseline):
+  #   - MCE / QPI  (rows carry "bank=0x")        -> the CPU interconnect fault
+  #   - disk I/O   (rows carry "error='I/O error'") -> failing drive OR SATA link
+  # rasdaemon lumps both in `--errors`; counting them together used to make a
+  # disk glitch masquerade as "QPI returned", so we grep each class separately.
+  # State: "mce_base mce_last disk_base disk_last" (old 2-field files re-baseline).
   mceMonitor = pkgs.writeShellScript "mce-monitor" ''
     set -uo pipefail
     host="${config.networking.hostName}"
@@ -29,49 +36,63 @@ let
     state=/var/lib/mce-monitor/state
     rasctl=${pkgs.rasdaemon}/bin/ras-mc-ctl
     grep=${pkgs.gnugrep}/bin/grep
+    tail=${pkgs.coreutils}/bin/tail
+    cut=${pkgs.coreutils}/bin/cut
 
-    # Cumulative decoded-MCE count from rasdaemon's DB (persists across reboots).
+    errs=$($rasctl --errors 2>/dev/null || true)
-    current=$($rasctl --errors 2>/dev/null | $grep -cE '^[0-9]+ 20' || echo 0)
+    mce_cur=$(printf '%s\n' "$errs" | $grep -cE 'bank=0x' || true)
+    disk_cur=$(printf '%s\n' "$errs" | $grep -cF "error='I/O error'" || true)
 
-    # First run: record baseline, alert on nothing historical.
+    # First run (or migration from the old 2-field state): baseline, no alert.
-    if [ ! -f "$state" ]; then
+    if [ ! -f "$state" ] || [ "$(${pkgs.coreutils}/bin/wc -w < "$state")" -ne 4 ]; then
-      echo "$current 0" > "$state"
+      echo "$mce_cur 0 $disk_cur 0" > "$state"
       exit 0
     fi
-    read baseline last < "$state"
+    read mce_base mce_last disk_base disk_last < "$state"
 
-    # DB wiped/shrank -> re-baseline, don't alert.
+    # Counter shrank (DB wiped) -> re-baseline that class, don't alert.
-    if [ "$current" -lt "$baseline" ]; then
+    [ "$mce_cur"  -lt "$mce_base"  ] && { mce_base=$mce_cur;   mce_last=0; }
-      echo "$current 0" > "$state"
+    [ "$disk_cur" -lt "$disk_base" ] && { disk_base=$disk_cur; disk_last=0; }
-      exit 0
-    fi
 
-    new=$(( current - baseline ))
+    notify() { # prio tag body
-    [ "$new" -lt 1 ] && exit 0
+      [ -f "$secret" ] || return 0
-
-    # Highest geometric threshold crossed by the new-error count.
-    crossed=0
-    for t in 1 10 100 1000 10000 100000; do
-      [ "$new" -ge "$t" ] && crossed=$t
-    done
-    # Already alerted at this bucket -> stay quiet (increasing intervals).
-    [ "$crossed" -le "$last" ] && exit 0
-
-    if [ ! -f "$secret" ]; then
-      echo "mce-monitor: $secret missing; cannot notify" >&2
-      exit 0
-    fi
       url=$(${pkgs.coreutils}/bin/tr -d '\n' < "$secret")
-    newest=$($rasctl --errors 2>/dev/null | $grep -E '^[0-9]+ 20' \
-      | ${pkgs.coreutils}/bin/tail -1 | ${pkgs.coreutils}/bin/cut -d' ' -f2-4)
-    if [ "$new" -ge 1000 ]; then prio=urgent; tag=rotating_light; else prio=high; tag=warning; fi
-
       ${pkgs.curl}/bin/curl -fsS --max-time 10 \
-      -H "Title: MCE/QPI errors returned on $host" -H "Priority: $prio" -H "Tags: $tag" \
+        -H "Title: $host hardware alert" -H "Priority: $1" -H "Tags: $2" \
-      -d "$new new machine-check (QPI) errors since the Jul repaste. Newest: $newest" \
+        -d "$3" "$url" >/dev/null 2>&1 || true
-      "$url" >/dev/null 2>&1 || true
+    }
+    bucket() { # new -> highest geometric threshold crossed
+      local n=$1 c=0 t
+      for t in 1 10 100 1000 10000 100000; do [ "$n" -ge "$t" ] && c=$t; done
+      echo "$c"
+    }
 
-    echo "$baseline $crossed" > "$state"
+    # --- MCE / QPI ---
+    mce_new=$(( mce_cur - mce_base ))
+    if [ "$mce_new" -ge 1 ]; then
+      b=$(bucket "$mce_new")
+      if [ "$b" -gt "$mce_last" ]; then
+        newest=$(printf '%s\n' "$errs" | $grep -E 'bank=0x' | $tail -1 | $cut -d' ' -f2-4)
+        if [ "$mce_new" -ge 1000 ]; then p=urgent; else p=high; fi
+        echo "mce-monitor: $mce_new new MCE/QPI errors (newest $newest)"
+        notify "$p" rotating_light "$mce_new new machine-check (QPI) errors since the Jul repaste. Newest: $newest"
+        mce_last=$b
+      fi
+    fi
+
+    # --- disk I/O / SATA ---
+    disk_new=$(( disk_cur - disk_base ))
+    if [ "$disk_new" -ge 1 ]; then
+      b=$(bucket "$disk_new")
+      if [ "$b" -gt "$disk_last" ]; then
+        dev=$(printf '%s\n' "$errs" | $grep -F "error='I/O error'" | $tail -1 | $grep -oE 'dev=[0-9]+:[0-9]+' | $tail -1)
+        echo "mce-monitor: $disk_new new disk I/O errors ($dev)"
+        notify high floppy_disk "$disk_new new disk I/O errors on $host ($dev). Check SATA cable / SMART."
+        disk_last=$b
+      fi
+    fi
+
+    echo "$mce_base $mce_last $disk_base $disk_last" > "$state"
   '';
 in
 {