From c00a1f85bccbab2efdf55559dae4a12d3e228ac1 Mon Sep 17 00:00:00 2001 From: rope Date: Sat, 15 Aug 2026 12:42:50 +0100 Subject: [PATCH] router-ui: per-device traffic, last seen, and saner device saves - nftables dynamic sets count bytes per LAN address; a 1-min tick folds them into totals that survive a ruleset reload - devices page shows last seen + per-device down/up - saving no longer pushes every DHCP lease into devices.toml, only rows with a reservation, note or block - empty-state text on the traffic graphs instead of blank space Co-Authored-By: Claude Opus 5 --- scripts/router-ui-check.py | 33 +++++ scripts/router-ui.py | 277 +++++++++++++++++++++++++++++++------ services/router-ui.nix | 27 ++++ services/router.nix | 38 +++++ 4 files changed, 336 insertions(+), 39 deletions(-) diff --git a/scripts/router-ui-check.py b/scripts/router-ui-check.py index 4f2f4ce..e217479 100644 --- a/scripts/router-ui-check.py +++ b/scripts/router-ui-check.py @@ -129,6 +129,39 @@ entry = tomlkit.dumps(ddoc).split("[[device]]")[1] assert "ip = " not in entry, entry assert "blocked = true" in entry, entry +# --- accounting: deltas, and counter resets on ruleset reload ---------------- +state = tempfile.mkdtemp() +r.TRAFFIC_FILE = os.path.join(state, "traffic.json") +r.LEASES = os.path.join(state, "leases") +open(r.LEASES, "w").write("999 aa:bb:cc:dd:ee:01 10.0.0.50 phone *\n") + +samples = [] +r.nft_counters = lambda: samples[-1] # noqa: E731 - stand in for the live sets + + +def tick(up, down): + samples.append({"10.0.0.50": {"up": up, "down": down}}) + return r.accounting_tick()["totals"]["aa:bb:cc:dd:ee:01"] + + +t = tick(100, 200) +assert (t["up"], t["down"]) == (100, 200), t +t = tick(150, 700) # normal growth -> delta added +assert (t["up"], t["down"]) == (150, 700), t +t = tick(10, 5) # counters reset -> whole sample counts +assert (t["up"], t["down"]) == (160, 705), t +t = tick(10, 5) # unchanged -> nothing added +assert (t["up"], t["down"]) == (160, 705), t +assert t["ip"] == "10.0.0.50" + +seen = t["last_seen"] +t = tick(10, 5) # idle tick must not refresh last_seen +assert t["last_seen"] == seen, (t["last_seen"], seen) + +# an address with no lease is keyed by IP so it still shows up +samples.append({"10.0.0.77": {"up": 5, "down": 5}}) +assert "10.0.0.77" in r.accounting_tick()["totals"] + # --- speedtest parsing (bytes/s -> Mbps, ns -> ms) --------------------------- sample = ('{"timestamp":"2026-08-15 12:00:55","servers":[{"name":"Preston",' '"latency":18796260,"dl_speed":63607890.17,"ul_speed":8839605.08}]}\n') diff --git a/scripts/router-ui.py b/scripts/router-ui.py index 69b4cc4..82a7545 100644 --- a/scripts/router-ui.py +++ b/scripts/router-ui.py @@ -19,6 +19,7 @@ import os import re import socket import subprocess +import sys import time import urllib.error import urllib.request @@ -44,6 +45,7 @@ LEASES = os.environ.get("DNSMASQ_LEASES", "/var/lib/dnsmasq/dnsmasq.leases") OUI_FILE = os.environ.get("OUI_FILE", "") SPEEDTEST_LOG = os.path.join(STATE, "speedtest.jsonl") LAST_REV = os.path.join(STATE, "last-rev") +TRAFFIC_FILE = os.path.join(STATE, "traffic.json") # Absolute, injected by the unit. sudo matches on argv, so the systemctl path # here must be byte-identical to the one in the sudoers rule. @@ -52,6 +54,7 @@ SYSTEMCTL = os.environ.get("SYSTEMCTL_BIN", "systemctl") JOURNALCTL = os.environ.get("JOURNALCTL_BIN", "journalctl") IP = os.environ.get("IP_BIN", "ip") VNSTAT = os.environ.get("VNSTAT_BIN", "vnstat") +NFT = os.environ.get("NFT_BIN", "nft") # Forwards that must always survive an edit — losing either locks us out. REQUIRED_FORWARDS = [("tcp", "22"), ("tcp", "443")] @@ -216,14 +219,90 @@ def leases(): def neighbours(): - """MACs the kernel has recently exchanged packets with, i.e. online-ish.""" - online = {} + """{mac: ARP state}. REACHABLE/DELAY/PROBE mean traffic in the last few + seconds; STALE only means we spoke to it at some point since boot, which + is why it doesn't count as online on its own.""" + states = {} for line in run(IP, "neigh", "show", "dev", LAN_IF).splitlines(): parts = line.split() if len(parts) >= 5 and parts[1] == "lladdr": - state = parts[-1] - online[parts[2].lower()] = state not in ("FAILED", "INCOMPLETE") - return online + states[parts[2].lower()] = parts[-1] + return states + + +LIVE_STATES = ("REACHABLE", "DELAY", "PROBE") + + +def nft_counters(): + """{lan_ip: {"up": bytes, "down": bytes}} from the accounting table.""" + out = {} + for direction in ("up", "down"): + try: + blob = json.loads(run(NFT, "-j", "list", "set", "ip", "accounting", direction)) + except ValueError: + continue + for node in blob.get("nftables", []): + if not isinstance(node, dict): + continue + for elem in node.get("set", {}).get("elem", []): + e = elem.get("elem", elem) if isinstance(elem, dict) else None + if not isinstance(e, dict): + continue + ip, counter = e.get("val"), e.get("counter") or {} + if isinstance(ip, str) and "bytes" in counter: + out.setdefault(ip, {"up": 0, "down": 0})[direction] = counter["bytes"] + return out + + +def load_traffic(): + try: + with open(TRAFFIC_FILE) as fh: + data = json.load(fh) + except (OSError, ValueError): + data = {} + data.setdefault("totals", {}) + data.setdefault("snapshot", {}) + data.setdefault("since", int(time.time())) + return data + + +def accounting_tick(): + """Fold the live nft counters into per-device totals that survive. + + Run as root from router-accounting.service. The nft counters restart at + zero every time the ruleset reloads, so a sample below the previous one is + read as a reset and counted in full rather than as a negative delta. + + Totals are keyed by MAC where a DHCP lease gives us one, so a device + keeps its history across an address change. + """ + data = load_traffic() + by_ip = {lease["ip"]: mac for mac, lease in leases().items()} + now = int(time.time()) + current = nft_counters() + + for ip, counts in current.items(): + key = by_ip.get(ip, ip) + was = data["snapshot"].get(ip, {}) + entry = data["totals"].setdefault(key, {"up": 0, "down": 0, "last_seen": 0}) + moved = False + for direction in ("up", "down"): + sample, previous = counts.get(direction, 0), was.get(direction, 0) + delta = sample - previous if sample >= previous else sample + if delta > 0: + entry[direction] += delta + moved = True + if moved: + entry["last_seen"] = now + entry["ip"] = ip + + data["snapshot"] = current + tmp = TRAFFIC_FILE + ".tmp" + with open(tmp, "w") as fh: + json.dump(data, fh) + os.replace(tmp, TRAFFIC_FILE) + os.chmod(TRAFFIC_FILE, 0o644) # the web app runs as router-ui and only reads + return data def caller_ip(handler): @@ -374,28 +453,40 @@ def device_view(): doc, sha = git_file("devices.toml") known = devices_from(doc) lease = leases() - online = neighbours() - by_mac = {d["mac"]: d for d in known} + states = neighbours() + totals = load_traffic()["totals"] + now = int(time.time()) - for d in known: - seen = lease.get(d["mac"], {}) + def enrich(d, seen): d["lease_ip"] = seen.get("ip", "") d["hostname"] = seen.get("lease_name", "") - d["online"] = online.get(d["mac"], False) d["vendor"] = vendor(d["mac"]) + # Counters are keyed by MAC once a lease exists, by raw IP before that. + stats = totals.get(d["mac"]) or totals.get(d["lease_ip"]) or {} + d["up"] = stats.get("up", 0) + d["down"] = stats.get("down", 0) + d["last_seen"] = stats.get("last_seen", 0) + # ARP says "right now"; the accounting tick only resolves to a minute, + # so treat a recent tick as live too for devices that idle quietly. + d["online"] = bool(states.get(d["mac"]) in LIVE_STATES + or (d["last_seen"] and now - d["last_seen"] < 180)) + return d + + by_mac = {d["mac"]: d for d in known} + for d in known: + enrich(d, lease.get(d["mac"], {})) d["known"] = True unknown = [] for mac, seen in sorted(lease.items(), key=lambda kv: kv[1]["ip"]): if mac in by_mac: continue - unknown.append({ + unknown.append(enrich({ "_i": None, "mac": mac, "name": seen["lease_name"] or mac.replace(":", ""), - "ip": "", "blocked": False, "note": "", - "lease_ip": seen["ip"], "hostname": seen["lease_name"], - "online": online.get(mac, False), "vendor": vendor(mac), "known": False, - }) - return {"sha": sha, "devices": known + unknown} + "ip": "", "blocked": False, "note": "", "known": False, + }, seen)) + return {"sha": sha, "devices": known + unknown, + "since": load_traffic().get("since", 0)} def wan_stats(): @@ -487,12 +578,38 @@ def speedtests(limit=30): return out[-limit:] +def device_traffic(): + """Per-device totals, newest names first. Reads the deployed devices.toml + rather than git so the traffic tab needs no Forgejo round-trip.""" + data = load_traffic() + names = {} + doc = deployed("devices.toml") + if doc: + for d in doc.get("device", []): + names[str(d.get("mac", "")).lower()] = str(d.get("name", "")) + lease = leases() + rows = [ + { + "key": key, + "name": names.get(key) or lease.get(key, {}).get("lease_name") or "", + "ip": t.get("ip", ""), + "up": t.get("up", 0), + "down": t.get("down", 0), + "last_seen": t.get("last_seen", 0), + } + for key, t in data["totals"].items() + ] + rows.sort(key=lambda r: r["up"] + r["down"], reverse=True) + return {"since": data.get("since", 0), "devices": rows} + + def traffic(): + per_device = device_traffic() raw = run(VNSTAT, "--json", "-i", WAN_IF, timeout=20) try: iface = json.loads(raw)["interfaces"][0]["traffic"] except (ValueError, KeyError, IndexError): - return {"day": [], "month": [], "hour": []} + return {"day": [], "month": [], "hour": [], **per_device} def series(key): return [ @@ -507,7 +624,7 @@ def traffic(): ] return {"hour": series("hour")[-24:], "day": series("day")[-30:], - "month": series("month")[-12:]} + "month": series("month")[-12:], **per_device} # --- actions ----------------------------------------------------------------- @@ -687,9 +804,13 @@ select{background:#0e1016;color:var(--fg);border:1px solid var(--line); border:1px solid var(--line);background:var(--card);display:none;font-size:13px} .msg.bad{border-color:var(--bad);color:var(--bad)} .msg.ok{border-color:var(--ok);color:var(--ok)} -.bar{display:flex;align-items:flex-end;gap:2px;height:70px;margin-top:8px} -.bar i{flex:1;background:var(--accent);border-radius:2px 2px 0 0;min-height:1px} +.bar{display:flex;align-items:flex-end;gap:2px;height:70px;margin-top:8px; + border-bottom:1px solid var(--line)} +.bar i{flex:1;background:var(--accent);border-radius:2px 2px 0 0;min-height:2px} .bar i.up{background:var(--warn)} +.bar .empty{color:var(--dim);font-size:13px;font-style:normal;align-self:center; + background:none;flex:1;text-align:center;min-height:0} +.scroll{overflow-x:auto} pre{background:#0e1016;border:1px solid var(--line);border-radius:8px;padding:12px; max-height:340px;overflow:auto;font-size:12px;white-space:pre-wrap} .dim{color:var(--dim)}.right{text-align:right} @@ -718,9 +839,15 @@ pre{background:#0e1016;border:1px solid var(--line);border-radius:8px;padding:12
Saving commits devices.toml. Nothing reaches the router until you Apply.
- +
+
NameMACVendorLease
+
NameLast seenMACVendorLeaseDownUp Reserved IPBlockNote
+ +

Rows marked unregistered are live DHCP leases that + aren't in devices.toml. They're only written to git once you give one a + reserved IP, a note, or tick Block — otherwise Save leaves them alone.

@@ -736,7 +863,14 @@ pre{background:#0e1016;border:1px solid var(--line);border-radius:8px;padding:12

Last 24 hours

Last 30 days

Monthly

-

Blue = download, amber = upload.

+

Blue = download, amber = upload. Hover a bar for exact figures.

+

Per device

+
+ + +
DeviceAddressDownUpTotalLast seen
+
+

@@ -777,7 +911,20 @@ function dur(s) { const d = Math.floor(s / 86400), h = Math.floor(s % 86400 / 3600); return d ? d + 'd ' + h + 'h' : h + 'h ' + Math.floor(s % 3600 / 60) + 'm'; } -const gib = b => (b / 1024 / 1024 / 1024).toFixed(1) + ' GiB'; +function bytes(b) { + const u = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; + let i = 0; + while (b >= 1024 && i < u.length - 1) { b /= 1024; i++; } + return (i ? b.toFixed(1) : b) + ' ' + u[i]; +} +function ago(ts) { + if (!ts) return 'never'; + const s = Math.max(0, Math.floor(Date.now() / 1000) - ts); + if (s < 180) return 'just now'; + if (s < 3600) return Math.floor(s / 60) + ' min ago'; + if (s < 86400) return Math.floor(s / 3600) + 'h ago'; + return Math.floor(s / 86400) + 'd ago'; +} async function load(tab) { try { @@ -816,25 +963,58 @@ function bind(obj, key) { i.oninput = () => obj[key] = i.value.trim(); return i; } +// A lease that isn't in devices.toml stays out of git until you actually +// give it something to remember — mirrors what the Save button sends. +const registered = d => d.known || d.ip || d.blocked || (d.note || '').trim(); + function drawDevs() { const body = $('#devrows'); body.innerHTML = ''; devs.forEach((d, n) => { const tr = el('tr'); - const dot = el('span', { className: 'dot' + (d.online ? ' up' : '') }); + const dot = el('span', { + className: 'dot' + (d.online ? ' up' : ''), + title: d.online ? 'active now' : 'last seen ' + ago(d.last_seen) + }); const blocked = el('input', { type: 'checkbox', checked: d.blocked }); - blocked.onchange = () => d.blocked = blocked.checked; - const cells = [dot, bind(d, 'name'), d.mac, d.vendor || '—', - d.lease_ip || '—', bind(d, 'ip'), blocked, bind(d, 'note')]; - cells.forEach(c => { const td = el('td'); td.append(c.nodeType ? c : document.createTextNode(c)); tr.append(td); }); + blocked.onchange = () => { d.blocked = blocked.checked; drawDevs(); }; + const cells = [dot, bind(d, 'name'), d.online ? 'active now' : ago(d.last_seen), + d.mac, d.vendor || '—', d.lease_ip || '—', + bytes(d.down || 0), bytes(d.up || 0), + bind(d, 'ip'), blocked, bind(d, 'note')]; + cells.forEach((c, i) => { + const td = el('td', { className: i === 6 || i === 7 ? 'right' : '' }); + td.append(c.nodeType ? c : document.createTextNode(c)); + tr.append(td); + }); const td = el('td'); if (d.known) { const b = el('button', { className: 'danger', textContent: 'Forget' }); b.onclick = () => { devs.splice(n, 1); drawDevs(); }; td.append(b); - } else td.append(el('span', { className: 'dim', textContent: 'unregistered' })); + } else { + td.append(el('span', { + className: 'dim', + textContent: registered(d) ? 'will be added' : 'unregistered' + })); + } tr.append(td); body.append(tr); }); } +function drawDeviceTraffic(t) { + const body = $('#dtrows'); body.innerHTML = ''; + (t.devices || []).forEach(d => { + const tr = el('tr'); + [d.name || d.key, d.ip || d.key, bytes(d.down), bytes(d.up), + bytes(d.down + d.up), ago(d.last_seen)].forEach((v, i) => { + tr.append(el('td', { textContent: v, className: i >= 2 && i <= 4 ? 'right' : '' })); + }); + body.append(tr); + }); + $('#dtsince').textContent = (t.devices || []).length + ? 'Counting since ' + new Date((t.since || 0) * 1000).toLocaleString() + + '. Totals survive rebuilds; the underlying nftables counters do not.' + : 'No per-device data yet — the accounting tick runs every minute.'; +} function drawPorts() { const body = $('#portrows'); body.innerHTML = ''; ports.forEach((f, n) => { @@ -850,19 +1030,29 @@ function drawPorts() { td.append(b); tr.append(td); body.append(tr); }); } -function bars(node, rows) { +function bars(node, rows, empty) { node.innerHTML = ''; + if (!rows.length) { + node.append(el('i', { className: 'empty', textContent: empty })); + return; + } const max = Math.max(1, ...rows.map(r => Math.max(r.rx, r.tx))); rows.forEach(r => { - [['rx', ''], ['tx', ' up']].forEach(([k, cls]) => { - node.append(el('i', { - className: cls.trim(), title: r.label + ' ' + k + ' ' + gib(r[k]), - style: 'height:' + Math.max(1, r[k] / max * 100) + '%' - })); + [['rx', 'down'], ['tx', 'up']].forEach(([k, label]) => { + const i = el('i', { title: r.label + ' ' + label + ' ' + bytes(r[k]) }); + if (label === 'up') i.className = 'up'; + i.style.height = Math.max(2, r[k] / max * 100) + '%'; + node.append(i); }); }); } -function drawTraffic(t) { bars($('#th'), t.hour); bars($('#td'), t.day); bars($('#tm'), t.month); } +function drawTraffic(t) { + const wait = 'vnstat is still collecting — it writes every 5 minutes'; + bars($('#th'), t.hour, wait); + bars($('#td'), t.day, wait); + bars($('#tm'), t.month, wait); + drawDeviceTraffic(t); +} let prev = null, timer = null; async function live() { @@ -908,8 +1098,13 @@ function drawSpeed(rows) { $('#devsave').onclick = async () => { try { - const r = await post('/api/devices', { sha: devSha, devices: devs }); - head = r.rev; say('Committed ' + r.rev.slice(0, 8) + '. Go to Overview and Apply.'); + const payload = devs.filter(registered); + const added = payload.filter(d => !d.known).length; + const r = await post('/api/devices', { sha: devSha, devices: payload }); + head = r.rev; + say('Committed ' + r.rev.slice(0, 8) + + (added ? ' (' + added + ' newly registered)' : '') + + '. Go to Overview and Apply.'); load('devices'); } catch (e) { say(e.message, true); } }; @@ -959,4 +1154,8 @@ load('overview'); if __name__ == "__main__": os.makedirs(STATE, exist_ok=True) - ThreadingHTTPServer(("127.0.0.1", PORT), Handler).serve_forever() + if len(sys.argv) > 1 and sys.argv[1] == "tick": + # router-accounting.service, as root: reading nft sets needs NET_ADMIN. + accounting_tick() + else: + ThreadingHTTPServer(("127.0.0.1", PORT), Handler).serve_forever() diff --git a/services/router-ui.nix b/services/router-ui.nix index 00b3d36..d1dc800 100644 --- a/services/router-ui.nix +++ b/services/router-ui.nix @@ -170,6 +170,33 @@ in }; }; + # Snapshot the nftables per-device counters and fold them into totals that + # outlive a ruleset reload. Root, because reading an nft set needs + # NET_ADMIN; the web app only ever reads the resulting file. + systemd.services.router-accounting = { + description = "Accumulate per-device traffic counters"; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${pythonEnv}/bin/python3 ${../scripts/router-ui.py} tick"; + # No StateDirectory: this unit is root, and StateDirectory would chown + # /var/lib/router-ui away from the router-ui user on every tick. The + # script mkdir -p's it, so a cold start before the web app is fine. + }; + environment = { + ROUTER_UI_STATE = stateDir; + DNSMASQ_LEASES = "/var/lib/dnsmasq/dnsmasq.leases"; + NFT_BIN = "${pkgs.nftables}/bin/nft"; + }; + }; + + systemd.timers.router-accounting = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnBootSec = "2min"; + OnUnitActiveSec = "1min"; + }; + }; + systemd.services.router-speedtest = { description = "Record a speedtest result"; serviceConfig = { diff --git a/services/router.nix b/services/router.nix index fd368d0..1685eb1 100644 --- a/services/router.nix +++ b/services/router.nix @@ -175,6 +175,44 @@ in } ''; }; + # Per-device byte counters for the router UI's traffic page. + # + # Dynamic sets keyed on the LAN address, so hosts appear on their own — + # nothing here needs to know which devices exist. prerouting/postrouting + # rather than forward, because forward misses LAN↔router traffic, and on + # this box that includes every Jellyfin stream. Priority -300 puts both + # chains ahead of NAT, so addresses are still the device's own. + # + # Counters reset whenever this table reloads (i.e. every switch that + # touches nftables). router-accounting.service snapshots them each + # minute and accumulates deltas into a file that survives, so the UI's + # totals are continuous even though these counters aren't. + tables.accounting = { + family = "ip"; + content = '' + set up { + type ipv4_addr + flags dynamic + counter + timeout 7d + } + set down { + type ipv4_addr + flags dynamic + counter + timeout 7d + } + chain pre { + type filter hook prerouting priority -300; policy accept; + iifname "eth0" update @up { ip saddr } + } + chain post { + type filter hook postrouting priority -300; policy accept; + oifname "eth0" update @down { ip daddr } + } + ''; + }; + # Use a distinct table name so we don't share `ip nat` with Docker — # Docker manages its own DOCKER/PREROUTING chains in `ip nat`, and # NixOS's nftables module rebuilds whichever tables it owns on every