diff --git a/services/hardware-health.nix b/services/hardware-health.nix index a205cdb..eb908ef 100644 --- a/services/hardware-health.nix +++ b/services/hardware-health.nix @@ -22,6 +22,13 @@ { config, lib, pkgs, ... }: let + # Watches TWO rasdaemon error classes independently, each with its own + # geometric-threshold alerting (1/10/100/... new since baseline): + # - MCE / QPI (rows carry "bank=0x") -> the CPU interconnect fault + # - disk I/O (rows carry "error='I/O error'") -> failing drive OR SATA link + # rasdaemon lumps both in `--errors`; counting them together used to make a + # disk glitch masquerade as "QPI returned", so we grep each class separately. + # State: "mce_base mce_last disk_base disk_last" (old 2-field files re-baseline). mceMonitor = pkgs.writeShellScript "mce-monitor" '' set -uo pipefail host="${config.networking.hostName}" @@ -29,49 +36,63 @@ let state=/var/lib/mce-monitor/state rasctl=${pkgs.rasdaemon}/bin/ras-mc-ctl grep=${pkgs.gnugrep}/bin/grep + tail=${pkgs.coreutils}/bin/tail + cut=${pkgs.coreutils}/bin/cut - # Cumulative decoded-MCE count from rasdaemon's DB (persists across reboots). - current=$($rasctl --errors 2>/dev/null | $grep -cE '^[0-9]+ 20' || echo 0) + errs=$($rasctl --errors 2>/dev/null || true) + mce_cur=$(printf '%s\n' "$errs" | $grep -cE 'bank=0x' || true) + disk_cur=$(printf '%s\n' "$errs" | $grep -cF "error='I/O error'" || true) - # First run: record baseline, alert on nothing historical. - if [ ! -f "$state" ]; then - echo "$current 0" > "$state" + # First run (or migration from the old 2-field state): baseline, no alert. + if [ ! -f "$state" ] || [ "$(${pkgs.coreutils}/bin/wc -w < "$state")" -ne 4 ]; then + echo "$mce_cur 0 $disk_cur 0" > "$state" exit 0 fi - read baseline last < "$state" + read mce_base mce_last disk_base disk_last < "$state" - # DB wiped/shrank -> re-baseline, don't alert. - if [ "$current" -lt "$baseline" ]; then - echo "$current 0" > "$state" - exit 0 + # Counter shrank (DB wiped) -> re-baseline that class, don't alert. + [ "$mce_cur" -lt "$mce_base" ] && { mce_base=$mce_cur; mce_last=0; } + [ "$disk_cur" -lt "$disk_base" ] && { disk_base=$disk_cur; disk_last=0; } + + notify() { # prio tag body + [ -f "$secret" ] || return 0 + url=$(${pkgs.coreutils}/bin/tr -d '\n' < "$secret") + ${pkgs.curl}/bin/curl -fsS --max-time 10 \ + -H "Title: $host hardware alert" -H "Priority: $1" -H "Tags: $2" \ + -d "$3" "$url" >/dev/null 2>&1 || true + } + bucket() { # new -> highest geometric threshold crossed + local n=$1 c=0 t + for t in 1 10 100 1000 10000 100000; do [ "$n" -ge "$t" ] && c=$t; done + echo "$c" + } + + # --- MCE / QPI --- + mce_new=$(( mce_cur - mce_base )) + if [ "$mce_new" -ge 1 ]; then + b=$(bucket "$mce_new") + if [ "$b" -gt "$mce_last" ]; then + newest=$(printf '%s\n' "$errs" | $grep -E 'bank=0x' | $tail -1 | $cut -d' ' -f2-4) + if [ "$mce_new" -ge 1000 ]; then p=urgent; else p=high; fi + echo "mce-monitor: $mce_new new MCE/QPI errors (newest $newest)" + notify "$p" rotating_light "$mce_new new machine-check (QPI) errors since the Jul repaste. Newest: $newest" + mce_last=$b + fi fi - new=$(( current - baseline )) - [ "$new" -lt 1 ] && exit 0 - - # Highest geometric threshold crossed by the new-error count. - crossed=0 - for t in 1 10 100 1000 10000 100000; do - [ "$new" -ge "$t" ] && crossed=$t - done - # Already alerted at this bucket -> stay quiet (increasing intervals). - [ "$crossed" -le "$last" ] && exit 0 - - if [ ! -f "$secret" ]; then - echo "mce-monitor: $secret missing; cannot notify" >&2 - exit 0 + # --- disk I/O / SATA --- + disk_new=$(( disk_cur - disk_base )) + if [ "$disk_new" -ge 1 ]; then + b=$(bucket "$disk_new") + if [ "$b" -gt "$disk_last" ]; then + dev=$(printf '%s\n' "$errs" | $grep -F "error='I/O error'" | $tail -1 | $grep -oE 'dev=[0-9]+:[0-9]+' | $tail -1) + echo "mce-monitor: $disk_new new disk I/O errors ($dev)" + notify high floppy_disk "$disk_new new disk I/O errors on $host ($dev). Check SATA cable / SMART." + disk_last=$b + fi fi - url=$(${pkgs.coreutils}/bin/tr -d '\n' < "$secret") - newest=$($rasctl --errors 2>/dev/null | $grep -E '^[0-9]+ 20' \ - | ${pkgs.coreutils}/bin/tail -1 | ${pkgs.coreutils}/bin/cut -d' ' -f2-4) - if [ "$new" -ge 1000 ]; then prio=urgent; tag=rotating_light; else prio=high; tag=warning; fi - ${pkgs.curl}/bin/curl -fsS --max-time 10 \ - -H "Title: MCE/QPI errors returned on $host" -H "Priority: $prio" -H "Tags: $tag" \ - -d "$new new machine-check (QPI) errors since the Jul repaste. Newest: $newest" \ - "$url" >/dev/null 2>&1 || true - - echo "$baseline $crossed" > "$state" + echo "$mce_base $mce_last $disk_base $disk_last" > "$state" ''; in {