router-ui: per-device traffic, last seen, and saner device saves

- nftables dynamic sets count bytes per LAN address; a 1-min tick folds
  them into totals that survive a ruleset reload
- devices page shows last seen + per-device down/up
- saving no longer pushes every DHCP lease into devices.toml, only rows
  with a reservation, note or block
- empty-state text on the traffic graphs instead of blank space

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
rope 2026-08-15 12:42:50 +01:00
parent 355e696c43
commit c00a1f85bc
4 changed files with 336 additions and 39 deletions

View file

@ -129,6 +129,39 @@ entry = tomlkit.dumps(ddoc).split("[[device]]")[1]
assert "ip = " not in entry, entry
assert "blocked = true" in entry, entry
# --- accounting: deltas, and counter resets on ruleset reload ----------------
state = tempfile.mkdtemp()
r.TRAFFIC_FILE = os.path.join(state, "traffic.json")
r.LEASES = os.path.join(state, "leases")
open(r.LEASES, "w").write("999 aa:bb:cc:dd:ee:01 10.0.0.50 phone *\n")
samples = []
r.nft_counters = lambda: samples[-1] # noqa: E731 - stand in for the live sets
def tick(up, down):
samples.append({"10.0.0.50": {"up": up, "down": down}})
return r.accounting_tick()["totals"]["aa:bb:cc:dd:ee:01"]
t = tick(100, 200)
assert (t["up"], t["down"]) == (100, 200), t
t = tick(150, 700) # normal growth -> delta added
assert (t["up"], t["down"]) == (150, 700), t
t = tick(10, 5) # counters reset -> whole sample counts
assert (t["up"], t["down"]) == (160, 705), t
t = tick(10, 5) # unchanged -> nothing added
assert (t["up"], t["down"]) == (160, 705), t
assert t["ip"] == "10.0.0.50"
seen = t["last_seen"]
t = tick(10, 5) # idle tick must not refresh last_seen
assert t["last_seen"] == seen, (t["last_seen"], seen)
# an address with no lease is keyed by IP so it still shows up
samples.append({"10.0.0.77": {"up": 5, "down": 5}})
assert "10.0.0.77" in r.accounting_tick()["totals"]
# --- speedtest parsing (bytes/s -> Mbps, ns -> ms) ---------------------------
sample = ('{"timestamp":"2026-08-15 12:00:55","servers":[{"name":"Preston",'
'"latency":18796260,"dl_speed":63607890.17,"ul_speed":8839605.08}]}\n')

View file

@ -19,6 +19,7 @@ import os
import re
import socket
import subprocess
import sys
import time
import urllib.error
import urllib.request
@ -44,6 +45,7 @@ LEASES = os.environ.get("DNSMASQ_LEASES", "/var/lib/dnsmasq/dnsmasq.leases")
OUI_FILE = os.environ.get("OUI_FILE", "")
SPEEDTEST_LOG = os.path.join(STATE, "speedtest.jsonl")
LAST_REV = os.path.join(STATE, "last-rev")
TRAFFIC_FILE = os.path.join(STATE, "traffic.json")
# Absolute, injected by the unit. sudo matches on argv, so the systemctl path
# here must be byte-identical to the one in the sudoers rule.
@ -52,6 +54,7 @@ SYSTEMCTL = os.environ.get("SYSTEMCTL_BIN", "systemctl")
JOURNALCTL = os.environ.get("JOURNALCTL_BIN", "journalctl")
IP = os.environ.get("IP_BIN", "ip")
VNSTAT = os.environ.get("VNSTAT_BIN", "vnstat")
NFT = os.environ.get("NFT_BIN", "nft")
# Forwards that must always survive an edit — losing either locks us out.
REQUIRED_FORWARDS = [("tcp", "22"), ("tcp", "443")]
@ -216,14 +219,90 @@ def leases():
def neighbours():
"""MACs the kernel has recently exchanged packets with, i.e. online-ish."""
online = {}
"""{mac: ARP state}. REACHABLE/DELAY/PROBE mean traffic in the last few
seconds; STALE only means we spoke to it at some point since boot, which
is why it doesn't count as online on its own."""
states = {}
for line in run(IP, "neigh", "show", "dev", LAN_IF).splitlines():
parts = line.split()
if len(parts) >= 5 and parts[1] == "lladdr":
state = parts[-1]
online[parts[2].lower()] = state not in ("FAILED", "INCOMPLETE")
return online
states[parts[2].lower()] = parts[-1]
return states
LIVE_STATES = ("REACHABLE", "DELAY", "PROBE")
def nft_counters():
"""{lan_ip: {"up": bytes, "down": bytes}} from the accounting table."""
out = {}
for direction in ("up", "down"):
try:
blob = json.loads(run(NFT, "-j", "list", "set", "ip", "accounting", direction))
except ValueError:
continue
for node in blob.get("nftables", []):
if not isinstance(node, dict):
continue
for elem in node.get("set", {}).get("elem", []):
e = elem.get("elem", elem) if isinstance(elem, dict) else None
if not isinstance(e, dict):
continue
ip, counter = e.get("val"), e.get("counter") or {}
if isinstance(ip, str) and "bytes" in counter:
out.setdefault(ip, {"up": 0, "down": 0})[direction] = counter["bytes"]
return out
def load_traffic():
try:
with open(TRAFFIC_FILE) as fh:
data = json.load(fh)
except (OSError, ValueError):
data = {}
data.setdefault("totals", {})
data.setdefault("snapshot", {})
data.setdefault("since", int(time.time()))
return data
def accounting_tick():
"""Fold the live nft counters into per-device totals that survive.
Run as root from router-accounting.service. The nft counters restart at
zero every time the ruleset reloads, so a sample below the previous one is
read as a reset and counted in full rather than as a negative delta.
Totals are keyed by MAC where a DHCP lease gives us one, so a device
keeps its history across an address change.
"""
data = load_traffic()
by_ip = {lease["ip"]: mac for mac, lease in leases().items()}
now = int(time.time())
current = nft_counters()
for ip, counts in current.items():
key = by_ip.get(ip, ip)
was = data["snapshot"].get(ip, {})
entry = data["totals"].setdefault(key, {"up": 0, "down": 0, "last_seen": 0})
moved = False
for direction in ("up", "down"):
sample, previous = counts.get(direction, 0), was.get(direction, 0)
delta = sample - previous if sample >= previous else sample
if delta > 0:
entry[direction] += delta
moved = True
if moved:
entry["last_seen"] = now
entry["ip"] = ip
data["snapshot"] = current
tmp = TRAFFIC_FILE + ".tmp"
with open(tmp, "w") as fh:
json.dump(data, fh)
os.replace(tmp, TRAFFIC_FILE)
os.chmod(TRAFFIC_FILE, 0o644) # the web app runs as router-ui and only reads
return data
def caller_ip(handler):
@ -374,28 +453,40 @@ def device_view():
doc, sha = git_file("devices.toml")
known = devices_from(doc)
lease = leases()
online = neighbours()
by_mac = {d["mac"]: d for d in known}
states = neighbours()
totals = load_traffic()["totals"]
now = int(time.time())
for d in known:
seen = lease.get(d["mac"], {})
def enrich(d, seen):
d["lease_ip"] = seen.get("ip", "")
d["hostname"] = seen.get("lease_name", "")
d["online"] = online.get(d["mac"], False)
d["vendor"] = vendor(d["mac"])
# Counters are keyed by MAC once a lease exists, by raw IP before that.
stats = totals.get(d["mac"]) or totals.get(d["lease_ip"]) or {}
d["up"] = stats.get("up", 0)
d["down"] = stats.get("down", 0)
d["last_seen"] = stats.get("last_seen", 0)
# ARP says "right now"; the accounting tick only resolves to a minute,
# so treat a recent tick as live too for devices that idle quietly.
d["online"] = bool(states.get(d["mac"]) in LIVE_STATES
or (d["last_seen"] and now - d["last_seen"] < 180))
return d
by_mac = {d["mac"]: d for d in known}
for d in known:
enrich(d, lease.get(d["mac"], {}))
d["known"] = True
unknown = []
for mac, seen in sorted(lease.items(), key=lambda kv: kv[1]["ip"]):
if mac in by_mac:
continue
unknown.append({
unknown.append(enrich({
"_i": None, "mac": mac, "name": seen["lease_name"] or mac.replace(":", ""),
"ip": "", "blocked": False, "note": "",
"lease_ip": seen["ip"], "hostname": seen["lease_name"],
"online": online.get(mac, False), "vendor": vendor(mac), "known": False,
})
return {"sha": sha, "devices": known + unknown}
"ip": "", "blocked": False, "note": "", "known": False,
}, seen))
return {"sha": sha, "devices": known + unknown,
"since": load_traffic().get("since", 0)}
def wan_stats():
@ -487,12 +578,38 @@ def speedtests(limit=30):
return out[-limit:]
def device_traffic():
"""Per-device totals, newest names first. Reads the deployed devices.toml
rather than git so the traffic tab needs no Forgejo round-trip."""
data = load_traffic()
names = {}
doc = deployed("devices.toml")
if doc:
for d in doc.get("device", []):
names[str(d.get("mac", "")).lower()] = str(d.get("name", ""))
lease = leases()
rows = [
{
"key": key,
"name": names.get(key) or lease.get(key, {}).get("lease_name") or "",
"ip": t.get("ip", ""),
"up": t.get("up", 0),
"down": t.get("down", 0),
"last_seen": t.get("last_seen", 0),
}
for key, t in data["totals"].items()
]
rows.sort(key=lambda r: r["up"] + r["down"], reverse=True)
return {"since": data.get("since", 0), "devices": rows}
def traffic():
per_device = device_traffic()
raw = run(VNSTAT, "--json", "-i", WAN_IF, timeout=20)
try:
iface = json.loads(raw)["interfaces"][0]["traffic"]
except (ValueError, KeyError, IndexError):
return {"day": [], "month": [], "hour": []}
return {"day": [], "month": [], "hour": [], **per_device}
def series(key):
return [
@ -507,7 +624,7 @@ def traffic():
]
return {"hour": series("hour")[-24:], "day": series("day")[-30:],
"month": series("month")[-12:]}
"month": series("month")[-12:], **per_device}
# --- actions -----------------------------------------------------------------
@ -687,9 +804,13 @@ select{background:#0e1016;color:var(--fg);border:1px solid var(--line);
border:1px solid var(--line);background:var(--card);display:none;font-size:13px}
.msg.bad{border-color:var(--bad);color:var(--bad)}
.msg.ok{border-color:var(--ok);color:var(--ok)}
.bar{display:flex;align-items:flex-end;gap:2px;height:70px;margin-top:8px}
.bar i{flex:1;background:var(--accent);border-radius:2px 2px 0 0;min-height:1px}
.bar{display:flex;align-items:flex-end;gap:2px;height:70px;margin-top:8px;
border-bottom:1px solid var(--line)}
.bar i{flex:1;background:var(--accent);border-radius:2px 2px 0 0;min-height:2px}
.bar i.up{background:var(--warn)}
.bar .empty{color:var(--dim);font-size:13px;font-style:normal;align-self:center;
background:none;flex:1;text-align:center;min-height:0}
.scroll{overflow-x:auto}
pre{background:#0e1016;border:1px solid var(--line);border-radius:8px;padding:12px;
max-height:340px;overflow:auto;font-size:12px;white-space:pre-wrap}
.dim{color:var(--dim)}.right{text-align:right}
@ -718,9 +839,15 @@ pre{background:#0e1016;border:1px solid var(--line);border-radius:8px;padding:12
<section id="devices">
<div class="row"><button class="go" id="devsave">Save to git</button>
<span class="dim">Saving commits devices.toml. Nothing reaches the router until you Apply.</span></div>
<table><thead><tr><th></th><th>Name</th><th>MAC</th><th>Vendor</th><th>Lease</th>
<div class="scroll">
<table><thead><tr><th></th><th>Name</th><th>Last seen</th><th>MAC</th><th>Vendor</th>
<th>Lease</th><th class="right">Down</th><th class="right">Up</th>
<th>Reserved IP</th><th>Block</th><th>Note</th><th></th></tr></thead>
<tbody id="devrows"></tbody></table>
</div>
<p class="dim">Rows marked <em>unregistered</em> are live DHCP leases that
aren't in devices.toml. They're only written to git once you give one a
reserved IP, a note, or tick Block otherwise Save leaves them alone.</p>
</section>
<section id="ports">
@ -736,7 +863,14 @@ pre{background:#0e1016;border:1px solid var(--line);border-radius:8px;padding:12
<h3>Last 24 hours</h3><div class="bar" id="th"></div>
<h3>Last 30 days</h3><div class="bar" id="td"></div>
<h3>Monthly</h3><div class="bar" id="tm"></div>
<p class="dim">Blue = download, amber = upload.</p>
<p class="dim">Blue = download, amber = upload. Hover a bar for exact figures.</p>
<h3>Per device</h3>
<div class="scroll">
<table><thead><tr><th>Device</th><th>Address</th><th class="right">Down</th>
<th class="right">Up</th><th class="right">Total</th><th>Last seen</th></tr></thead>
<tbody id="dtrows"></tbody></table>
</div>
<p class="dim" id="dtsince"></p>
</section>
<section id="speed">
@ -777,7 +911,20 @@ function dur(s) {
const d = Math.floor(s / 86400), h = Math.floor(s % 86400 / 3600);
return d ? d + 'd ' + h + 'h' : h + 'h ' + Math.floor(s % 3600 / 60) + 'm';
}
const gib = b => (b / 1024 / 1024 / 1024).toFixed(1) + ' GiB';
function bytes(b) {
const u = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
let i = 0;
while (b >= 1024 && i < u.length - 1) { b /= 1024; i++; }
return (i ? b.toFixed(1) : b) + ' ' + u[i];
}
function ago(ts) {
if (!ts) return 'never';
const s = Math.max(0, Math.floor(Date.now() / 1000) - ts);
if (s < 180) return 'just now';
if (s < 3600) return Math.floor(s / 60) + ' min ago';
if (s < 86400) return Math.floor(s / 3600) + 'h ago';
return Math.floor(s / 86400) + 'd ago';
}
async function load(tab) {
try {
@ -816,25 +963,58 @@ function bind(obj, key) {
i.oninput = () => obj[key] = i.value.trim();
return i;
}
// A lease that isn't in devices.toml stays out of git until you actually
// give it something to remember mirrors what the Save button sends.
const registered = d => d.known || d.ip || d.blocked || (d.note || '').trim();
function drawDevs() {
const body = $('#devrows'); body.innerHTML = '';
devs.forEach((d, n) => {
const tr = el('tr');
const dot = el('span', { className: 'dot' + (d.online ? ' up' : '') });
const dot = el('span', {
className: 'dot' + (d.online ? ' up' : ''),
title: d.online ? 'active now' : 'last seen ' + ago(d.last_seen)
});
const blocked = el('input', { type: 'checkbox', checked: d.blocked });
blocked.onchange = () => d.blocked = blocked.checked;
const cells = [dot, bind(d, 'name'), d.mac, d.vendor || '',
d.lease_ip || '', bind(d, 'ip'), blocked, bind(d, 'note')];
cells.forEach(c => { const td = el('td'); td.append(c.nodeType ? c : document.createTextNode(c)); tr.append(td); });
blocked.onchange = () => { d.blocked = blocked.checked; drawDevs(); };
const cells = [dot, bind(d, 'name'), d.online ? 'active now' : ago(d.last_seen),
d.mac, d.vendor || '', d.lease_ip || '',
bytes(d.down || 0), bytes(d.up || 0),
bind(d, 'ip'), blocked, bind(d, 'note')];
cells.forEach((c, i) => {
const td = el('td', { className: i === 6 || i === 7 ? 'right' : '' });
td.append(c.nodeType ? c : document.createTextNode(c));
tr.append(td);
});
const td = el('td');
if (d.known) {
const b = el('button', { className: 'danger', textContent: 'Forget' });
b.onclick = () => { devs.splice(n, 1); drawDevs(); };
td.append(b);
} else td.append(el('span', { className: 'dim', textContent: 'unregistered' }));
} else {
td.append(el('span', {
className: 'dim',
textContent: registered(d) ? 'will be added' : 'unregistered'
}));
}
tr.append(td); body.append(tr);
});
}
function drawDeviceTraffic(t) {
const body = $('#dtrows'); body.innerHTML = '';
(t.devices || []).forEach(d => {
const tr = el('tr');
[d.name || d.key, d.ip || d.key, bytes(d.down), bytes(d.up),
bytes(d.down + d.up), ago(d.last_seen)].forEach((v, i) => {
tr.append(el('td', { textContent: v, className: i >= 2 && i <= 4 ? 'right' : '' }));
});
body.append(tr);
});
$('#dtsince').textContent = (t.devices || []).length
? 'Counting since ' + new Date((t.since || 0) * 1000).toLocaleString()
+ '. Totals survive rebuilds; the underlying nftables counters do not.'
: 'No per-device data yet — the accounting tick runs every minute.';
}
function drawPorts() {
const body = $('#portrows'); body.innerHTML = '';
ports.forEach((f, n) => {
@ -850,19 +1030,29 @@ function drawPorts() {
td.append(b); tr.append(td); body.append(tr);
});
}
function bars(node, rows) {
function bars(node, rows, empty) {
node.innerHTML = '';
if (!rows.length) {
node.append(el('i', { className: 'empty', textContent: empty }));
return;
}
const max = Math.max(1, ...rows.map(r => Math.max(r.rx, r.tx)));
rows.forEach(r => {
[['rx', ''], ['tx', ' up']].forEach(([k, cls]) => {
node.append(el('i', {
className: cls.trim(), title: r.label + ' ' + k + ' ' + gib(r[k]),
style: 'height:' + Math.max(1, r[k] / max * 100) + '%'
}));
[['rx', 'down'], ['tx', 'up']].forEach(([k, label]) => {
const i = el('i', { title: r.label + ' ' + label + ' ' + bytes(r[k]) });
if (label === 'up') i.className = 'up';
i.style.height = Math.max(2, r[k] / max * 100) + '%';
node.append(i);
});
});
}
function drawTraffic(t) { bars($('#th'), t.hour); bars($('#td'), t.day); bars($('#tm'), t.month); }
function drawTraffic(t) {
const wait = 'vnstat is still collecting — it writes every 5 minutes';
bars($('#th'), t.hour, wait);
bars($('#td'), t.day, wait);
bars($('#tm'), t.month, wait);
drawDeviceTraffic(t);
}
let prev = null, timer = null;
async function live() {
@ -908,8 +1098,13 @@ function drawSpeed(rows) {
$('#devsave').onclick = async () => {
try {
const r = await post('/api/devices', { sha: devSha, devices: devs });
head = r.rev; say('Committed ' + r.rev.slice(0, 8) + '. Go to Overview and Apply.');
const payload = devs.filter(registered);
const added = payload.filter(d => !d.known).length;
const r = await post('/api/devices', { sha: devSha, devices: payload });
head = r.rev;
say('Committed ' + r.rev.slice(0, 8)
+ (added ? ' (' + added + ' newly registered)' : '')
+ '. Go to Overview and Apply.');
load('devices');
} catch (e) { say(e.message, true); }
};
@ -959,4 +1154,8 @@ load('overview');
if __name__ == "__main__":
os.makedirs(STATE, exist_ok=True)
ThreadingHTTPServer(("127.0.0.1", PORT), Handler).serve_forever()
if len(sys.argv) > 1 and sys.argv[1] == "tick":
# router-accounting.service, as root: reading nft sets needs NET_ADMIN.
accounting_tick()
else:
ThreadingHTTPServer(("127.0.0.1", PORT), Handler).serve_forever()

View file

@ -170,6 +170,33 @@ in
};
};
# Snapshot the nftables per-device counters and fold them into totals that
# outlive a ruleset reload. Root, because reading an nft set needs
# NET_ADMIN; the web app only ever reads the resulting file.
systemd.services.router-accounting = {
description = "Accumulate per-device traffic counters";
serviceConfig = {
Type = "oneshot";
ExecStart = "${pythonEnv}/bin/python3 ${../scripts/router-ui.py} tick";
# No StateDirectory: this unit is root, and StateDirectory would chown
# /var/lib/router-ui away from the router-ui user on every tick. The
# script mkdir -p's it, so a cold start before the web app is fine.
};
environment = {
ROUTER_UI_STATE = stateDir;
DNSMASQ_LEASES = "/var/lib/dnsmasq/dnsmasq.leases";
NFT_BIN = "${pkgs.nftables}/bin/nft";
};
};
systemd.timers.router-accounting = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "2min";
OnUnitActiveSec = "1min";
};
};
systemd.services.router-speedtest = {
description = "Record a speedtest result";
serviceConfig = {

View file

@ -175,6 +175,44 @@ in
}
'';
};
# Per-device byte counters for the router UI's traffic page.
#
# Dynamic sets keyed on the LAN address, so hosts appear on their own —
# nothing here needs to know which devices exist. prerouting/postrouting
# rather than forward, because forward misses LAN↔router traffic, and on
# this box that includes every Jellyfin stream. Priority -300 puts both
# chains ahead of NAT, so addresses are still the device's own.
#
# Counters reset whenever this table reloads (i.e. every switch that
# touches nftables). router-accounting.service snapshots them each
# minute and accumulates deltas into a file that survives, so the UI's
# totals are continuous even though these counters aren't.
tables.accounting = {
family = "ip";
content = ''
set up {
type ipv4_addr
flags dynamic
counter
timeout 7d
}
set down {
type ipv4_addr
flags dynamic
counter
timeout 7d
}
chain pre {
type filter hook prerouting priority -300; policy accept;
iifname "eth0" update @up { ip saddr }
}
chain post {
type filter hook postrouting priority -300; policy accept;
oifname "eth0" update @down { ip daddr }
}
'';
};
# Use a distinct table name so we don't share `ip nat` with Docker —
# Docker manages its own DOCKER/PREROUTING chains in `ip nat`, and
# NixOS's nftables module rebuilds whichever tables it owns on every