mce-monitor: split MCE/QPI vs disk I/O error tracking + separate alerts
A disk SATA glitch was miscounted as new MCEs and paged as 'QPI returned'. Now counts bank=0x (MCE) and I/O-error (disk) rows independently, each with its own threshold alert. Old 2-field state auto-re-baselines. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
58d87dfb52
commit
3654bc6ca7
1 changed files with 55 additions and 34 deletions
|
|
@ -22,6 +22,13 @@
|
|||
|
||||
{ config, lib, pkgs, ... }:
|
||||
let
|
||||
# Watches TWO rasdaemon error classes independently, each with its own
|
||||
# geometric-threshold alerting (1/10/100/... new since baseline):
|
||||
# - MCE / QPI (rows carry "bank=0x") -> the CPU interconnect fault
|
||||
# - disk I/O (rows carry "error='I/O error'") -> failing drive OR SATA link
|
||||
# rasdaemon lumps both in `--errors`; counting them together used to make a
|
||||
# disk glitch masquerade as "QPI returned", so we grep each class separately.
|
||||
# State: "mce_base mce_last disk_base disk_last" (old 2-field files re-baseline).
|
||||
mceMonitor = pkgs.writeShellScript "mce-monitor" ''
|
||||
set -uo pipefail
|
||||
host="${config.networking.hostName}"
|
||||
|
|
@ -29,49 +36,63 @@ let
|
|||
state=/var/lib/mce-monitor/state
|
||||
rasctl=${pkgs.rasdaemon}/bin/ras-mc-ctl
|
||||
grep=${pkgs.gnugrep}/bin/grep
|
||||
tail=${pkgs.coreutils}/bin/tail
|
||||
cut=${pkgs.coreutils}/bin/cut
|
||||
|
||||
# Cumulative decoded-MCE count from rasdaemon's DB (persists across reboots).
|
||||
current=$($rasctl --errors 2>/dev/null | $grep -cE '^[0-9]+ 20' || echo 0)
|
||||
errs=$($rasctl --errors 2>/dev/null || true)
|
||||
mce_cur=$(printf '%s\n' "$errs" | $grep -cE 'bank=0x' || true)
|
||||
disk_cur=$(printf '%s\n' "$errs" | $grep -cF "error='I/O error'" || true)
|
||||
|
||||
# First run: record baseline, alert on nothing historical.
|
||||
if [ ! -f "$state" ]; then
|
||||
echo "$current 0" > "$state"
|
||||
# First run (or migration from the old 2-field state): baseline, no alert.
|
||||
if [ ! -f "$state" ] || [ "$(${pkgs.coreutils}/bin/wc -w < "$state")" -ne 4 ]; then
|
||||
echo "$mce_cur 0 $disk_cur 0" > "$state"
|
||||
exit 0
|
||||
fi
|
||||
read baseline last < "$state"
|
||||
read mce_base mce_last disk_base disk_last < "$state"
|
||||
|
||||
# DB wiped/shrank -> re-baseline, don't alert.
|
||||
if [ "$current" -lt "$baseline" ]; then
|
||||
echo "$current 0" > "$state"
|
||||
exit 0
|
||||
# Counter shrank (DB wiped) -> re-baseline that class, don't alert.
|
||||
[ "$mce_cur" -lt "$mce_base" ] && { mce_base=$mce_cur; mce_last=0; }
|
||||
[ "$disk_cur" -lt "$disk_base" ] && { disk_base=$disk_cur; disk_last=0; }
|
||||
|
||||
notify() { # prio tag body
|
||||
[ -f "$secret" ] || return 0
|
||||
url=$(${pkgs.coreutils}/bin/tr -d '\n' < "$secret")
|
||||
${pkgs.curl}/bin/curl -fsS --max-time 10 \
|
||||
-H "Title: $host hardware alert" -H "Priority: $1" -H "Tags: $2" \
|
||||
-d "$3" "$url" >/dev/null 2>&1 || true
|
||||
}
|
||||
bucket() { # new -> highest geometric threshold crossed
|
||||
local n=$1 c=0 t
|
||||
for t in 1 10 100 1000 10000 100000; do [ "$n" -ge "$t" ] && c=$t; done
|
||||
echo "$c"
|
||||
}
|
||||
|
||||
# --- MCE / QPI ---
|
||||
mce_new=$(( mce_cur - mce_base ))
|
||||
if [ "$mce_new" -ge 1 ]; then
|
||||
b=$(bucket "$mce_new")
|
||||
if [ "$b" -gt "$mce_last" ]; then
|
||||
newest=$(printf '%s\n' "$errs" | $grep -E 'bank=0x' | $tail -1 | $cut -d' ' -f2-4)
|
||||
if [ "$mce_new" -ge 1000 ]; then p=urgent; else p=high; fi
|
||||
echo "mce-monitor: $mce_new new MCE/QPI errors (newest $newest)"
|
||||
notify "$p" rotating_light "$mce_new new machine-check (QPI) errors since the Jul repaste. Newest: $newest"
|
||||
mce_last=$b
|
||||
fi
|
||||
fi
|
||||
|
||||
new=$(( current - baseline ))
|
||||
[ "$new" -lt 1 ] && exit 0
|
||||
|
||||
# Highest geometric threshold crossed by the new-error count.
|
||||
crossed=0
|
||||
for t in 1 10 100 1000 10000 100000; do
|
||||
[ "$new" -ge "$t" ] && crossed=$t
|
||||
done
|
||||
# Already alerted at this bucket -> stay quiet (increasing intervals).
|
||||
[ "$crossed" -le "$last" ] && exit 0
|
||||
|
||||
if [ ! -f "$secret" ]; then
|
||||
echo "mce-monitor: $secret missing; cannot notify" >&2
|
||||
exit 0
|
||||
# --- disk I/O / SATA ---
|
||||
disk_new=$(( disk_cur - disk_base ))
|
||||
if [ "$disk_new" -ge 1 ]; then
|
||||
b=$(bucket "$disk_new")
|
||||
if [ "$b" -gt "$disk_last" ]; then
|
||||
dev=$(printf '%s\n' "$errs" | $grep -F "error='I/O error'" | $tail -1 | $grep -oE 'dev=[0-9]+:[0-9]+' | $tail -1)
|
||||
echo "mce-monitor: $disk_new new disk I/O errors ($dev)"
|
||||
notify high floppy_disk "$disk_new new disk I/O errors on $host ($dev). Check SATA cable / SMART."
|
||||
disk_last=$b
|
||||
fi
|
||||
fi
|
||||
url=$(${pkgs.coreutils}/bin/tr -d '\n' < "$secret")
|
||||
newest=$($rasctl --errors 2>/dev/null | $grep -E '^[0-9]+ 20' \
|
||||
| ${pkgs.coreutils}/bin/tail -1 | ${pkgs.coreutils}/bin/cut -d' ' -f2-4)
|
||||
if [ "$new" -ge 1000 ]; then prio=urgent; tag=rotating_light; else prio=high; tag=warning; fi
|
||||
|
||||
${pkgs.curl}/bin/curl -fsS --max-time 10 \
|
||||
-H "Title: MCE/QPI errors returned on $host" -H "Priority: $prio" -H "Tags: $tag" \
|
||||
-d "$new new machine-check (QPI) errors since the Jul repaste. Newest: $newest" \
|
||||
"$url" >/dev/null 2>&1 || true
|
||||
|
||||
echo "$baseline $crossed" > "$state"
|
||||
echo "$mce_base $mce_last $disk_base $disk_last" > "$state"
|
||||
'';
|
||||
in
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue