112 lines
4.2 KiB
Nix
112 lines
4.2 KiB
Nix
# services/hardware-health.nix — RAS error attribution + watchdog auto-recovery
|
|
#
|
|
# Context: Jun 2026 the dual Xeon E5-2697 v3 began throwing a storm of
|
|
# *corrected* Machine Check Exceptions on both sockets (Bank 5 / Bank 20),
|
|
# ~18k events in 36h, eventually hanging the box. Since this host is the
|
|
# router, a hang takes the whole LAN offline until a manual power-cycle.
|
|
#
|
|
# This module:
|
|
# - rasdaemon: decodes every MCE to a specific DIMM/channel/socket and
|
|
# persists a per-component error DB, so a failing part can be named
|
|
# (needed for the seller's warranty claim). Query with `ras-mc-ctl
|
|
# --error-count` and `ras-mc-ctl --summary`.
|
|
# - hardware watchdog: if userspace hangs again, systemd stops petting
|
|
# /dev/watchdog0 and the chipset watchdog reboots the box (~30s),
|
|
# restoring the LAN without physical access.
|
|
# - mce-monitor: after the Jul 2026 CPU repaste/reseat, watch for the QPI
|
|
# fault returning. Baselines at the current rasdaemon count (so only NEW
|
|
# errors count) and pushes an ntfy alert on geometric thresholds — the
|
|
# 1st, 10th, 100th, 1000th... new error — so the first one pings
|
|
# immediately and a storm spaces out instead of thousands of pushes.
|
|
# Reuses the /var/secrets/ntfy-url topic (see services/service-health.nix).
|
|
|
|
{ config, lib, pkgs, ... }:
|
|
let
|
|
mceMonitor = pkgs.writeShellScript "mce-monitor" ''
|
|
set -uo pipefail
|
|
host="${config.networking.hostName}"
|
|
secret=/var/secrets/ntfy-url
|
|
state=/var/lib/mce-monitor/state
|
|
rasctl=${pkgs.rasdaemon}/bin/ras-mc-ctl
|
|
grep=${pkgs.gnugrep}/bin/grep
|
|
|
|
# Cumulative decoded-MCE count from rasdaemon's DB (persists across reboots).
|
|
current=$($rasctl --errors 2>/dev/null | $grep -cE '^[0-9]+ 20' || echo 0)
|
|
|
|
# First run: record baseline, alert on nothing historical.
|
|
if [ ! -f "$state" ]; then
|
|
echo "$current 0" > "$state"
|
|
exit 0
|
|
fi
|
|
read baseline last < "$state"
|
|
|
|
# DB wiped/shrank -> re-baseline, don't alert.
|
|
if [ "$current" -lt "$baseline" ]; then
|
|
echo "$current 0" > "$state"
|
|
exit 0
|
|
fi
|
|
|
|
new=$(( current - baseline ))
|
|
[ "$new" -lt 1 ] && exit 0
|
|
|
|
# Highest geometric threshold crossed by the new-error count.
|
|
crossed=0
|
|
for t in 1 10 100 1000 10000 100000; do
|
|
[ "$new" -ge "$t" ] && crossed=$t
|
|
done
|
|
# Already alerted at this bucket -> stay quiet (increasing intervals).
|
|
[ "$crossed" -le "$last" ] && exit 0
|
|
|
|
if [ ! -f "$secret" ]; then
|
|
echo "mce-monitor: $secret missing; cannot notify" >&2
|
|
exit 0
|
|
fi
|
|
url=$(${pkgs.coreutils}/bin/tr -d '\n' < "$secret")
|
|
newest=$($rasctl --errors 2>/dev/null | $grep -E '^[0-9]+ 20' \
|
|
| ${pkgs.coreutils}/bin/tail -1 | ${pkgs.coreutils}/bin/cut -d' ' -f2-4)
|
|
if [ "$new" -ge 1000 ]; then prio=urgent; tag=rotating_light; else prio=high; tag=warning; fi
|
|
|
|
${pkgs.curl}/bin/curl -fsS --max-time 10 \
|
|
-H "Title: MCE/QPI errors returned on $host" -H "Priority: $prio" -H "Tags: $tag" \
|
|
-d "$new new machine-check (QPI) errors since the Jul repaste. Newest: $newest" \
|
|
"$url" >/dev/null 2>&1 || true
|
|
|
|
echo "$baseline $crossed" > "$state"
|
|
'';
|
|
in
|
|
{
|
|
config = lib.mkIf (config.networking.hostName == "FredOS-Mediaserver") {
|
|
|
|
# Decode + log + persist machine-check / memory errors per component.
|
|
hardware.rasdaemon.enable = true;
|
|
|
|
# ras-mc-ctl on PATH for manual inspection.
|
|
environment.systemPackages = [ pkgs.rasdaemon ];
|
|
|
|
# Hardware watchdog: auto-reboot a hung box instead of a dead LAN.
|
|
# systemd pets /dev/watchdog0 at half the runtime interval; if it stops
|
|
# (hang), the chipset resets after RuntimeWatchdogSec.
|
|
systemd.settings.Manager = {
|
|
RuntimeWatchdogSec = "30s";
|
|
RebootWatchdogSec = "10min";
|
|
};
|
|
|
|
# Watch for the QPI fault returning; ntfy on increasing thresholds.
|
|
systemd.services.mce-monitor = {
|
|
description = "Alert (ntfy) on new machine-check/QPI errors";
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
ExecStart = mceMonitor;
|
|
StateDirectory = "mce-monitor";
|
|
};
|
|
};
|
|
systemd.timers.mce-monitor = {
|
|
wantedBy = [ "timers.target" ];
|
|
timerConfig = {
|
|
OnBootSec = "3min";
|
|
OnUnitActiveSec = "5min";
|
|
Persistent = true;
|
|
};
|
|
};
|
|
};
|
|
}
|