bvandeusen/FabledSteward
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(snmp): devices you add, test and read on the SNMP page (milestone 491)
CI / lint (push) Successful in 2s
Details
CI / integration (push) Failing after 55s
Details
CI / unit (push) Successful in 1m24s
Details
CI / publish (push) Skipped
Details

Browse Source 
The operator, reviewing SNMP on #5214: "this system looks incomplete like
there's something missing from it". The one device shown was plugin.yaml's
example (core-switch 192.168.1.1, community public), polled every minute;
devices could only be defined by editing YAML in the image; a failed poll was
logged at DEBUG and stored nowhere, so the page said "waiting for the next
poll" forever; v3 ran as v2c; counters showed as lifetime totals.

- Devices are rows (snmp_devices, migration snmp_001): address, port,
  v1/v2c community or a v3 user with auth/priv protocols and keys (secrets
  encrypted), presets plus custom readings, an optional host binding, and
  the newest poll's outcome. The YAML device list and its example are gone.
- The SNMP page adds, edits and deletes them (admin, audited, with
  get_audit_entities so the audit log names and links a device). Readings
  come from presets (system uptime, interface traffic by number, UPS-MIB,
  Linux net-snmp) and custom "OID name [unit]" lines.
- One GET per device with every OID, devices polled at once. The outcome is
  written to the row in words ("No reply from … within 5 s. Check the address
  and port, that SNMP is on, and the community"), with readings it lacks.
  v1's whole-request error drops that OID and asks again. SNMPv3 signs in
  with USM (MD5/SHA-1/SHA-2, DES/3DES/AES).
- Test asks the unsaved form's device for its name, description, uptime and
  interfaces by number. It is an admin's explicit action; every page reads
  only stored rows (System 178's ruling).
- Each reading has a unit; a counter is stored as a per-second rate (bytes as
  bit/s), a reset or wrap skipped. The page, widget and host panel say values
  in units and why a device is silent; the device page draws a chart per
  reading on a dated axis.

Tests: readings and presets, the form and its secrets, v3 checks, rates,
the poller against a stand-in pysnmp (one GET, v1 retry, timeout wording,
USM), the pages, and an integration test of the poll writing outcomes and
rates on Postgres.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-10-08 10:54:11 -04:00
co-authored by Claude Opus 5.5
parent 14474f9be4
commit 9ca2e38fa1
27 changed files with 1873 additions and 703 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/snmp/routes.py
+275 -207
View File
@@ -1,251 +1,319 @@
# plugins/snmp/routes.py
"""The SNMP pages. Every GET reads Steward's own records: the device rows the
editor writes and the poller annotates, and the readings it stores. Only Test,
an admin's explicit action, asks a device anything (System 178's ruling)."""
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from quart import Blueprint, current_app, render_template, request
from sqlalchemy import select
from quart import Blueprint, abort, current_app, redirect, render_template, request, session
from sqlalchemy import delete, select, update
from steward.auth.middleware import require_role
from steward.models.users import UserRole
from steward.core.formatting import human_bytes, human_duration
from steward.core.audit import log_audit
from steward.core.metric_series import latest_by_resource
from steward.models.metrics import PluginMetric
from steward.models.metrics import PluginMetric, PluginMetricLatest
from steward.models.users import UserRole
from .models import SnmpDevice
from .readings import (INTERFACES_LABEL, PRESETS, PRESET_ORDER, UNITS, chart_axis, format_value,
stored_unit)
from .service import (FormError, device_state, form_credentials, form_values, parse_form,
readings_for)
snmp_bp = Blueprint("snmp", __name__, template_folder="templates")
async def _latest_readings(db, device_names: list[str]) -> dict[str, dict]:
"""{device_name: {"readings": {label: latest_value}, "at": newest reading's
time}} for the devices that have any readings."""
latest = await latest_by_resource(db, "snmp", device_names)
return {name: {"readings": {m: r.value for m, r in metrics.items()},
"at": max(r.recorded_at for r in metrics.values())}
for name, metrics in latest.items() if metrics}
# A device whose newest reading is older than this many poll intervals is not
# shown as reachable: the poller has stopped getting answers from it.
STALE_POLLS = 3
def _device_state(entry: dict | None, poll_seconds: int, now: datetime) -> dict:
"""What the page says about a device's readings: {"readings", "at",
"reach"}, reach being "up" (answered within STALE_POLLS polls), "stale"
(has readings, but none recent) or "pending" (never answered)."""
if not entry:
return {"readings": {}, "at": None, "reach": "pending"}
fresh = (now - entry["at"]).total_seconds() <= STALE_POLLS * max(int(poll_seconds), 1)
return {**entry, "reach": "up" if fresh else "stale"}
def _poll_seconds() -> int:
return int(current_app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60))
async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]:
"""Return {label: [(recorded_at, value), ...]} for a single device."""
since = datetime.now(timezone.utc) - timedelta(hours=hours)
result = await db.execute(
select(PluginMetric)
.where(
PluginMetric.source_module == "snmp",
PluginMetric.resource_name == device_name,
PluginMetric.recorded_at >= since,
)
.order_by(PluginMetric.recorded_at)
)
rows = result.scalars().all()
async def _all_devices(db) -> list[SnmpDevice]:
return list((await db.execute(select(SnmpDevice).order_by(SnmpDevice.name))).scalars().all())
out: dict[str, list] = {}
for row in rows:
out.setdefault(row.metric_name, []).append(
(row.recorded_at.isoformat(), row.value)
)
return out
async def _latest(db, names: list[str]) -> dict[str, dict]:
"""{device name: {key: value}} from plugin_metric_latest."""
latest = await latest_by_resource(db, "snmp", names)
return {name: {m: r.value for m, r in metrics.items()} for name, metrics in latest.items()}
def shown_readings(device: SnmpDevice, values: dict) -> list[dict]:
"""The device's readings in its own order, each named and in its unit;
"—" for one with no value yet."""
return [{"key": r.key, "name": r.name, "counter": r.counter,
"text": format_value(stored_unit(r), values.get(r.key))}
for r in readings_for(device)]
def view(device: SnmpDevice, values: dict, now: datetime) -> dict:
state = device_state(device, _poll_seconds(), now)
names = {r.key: r.name for r in readings_for(device)}
return {"id": device.id, "name": device.name, "address": device.address, "port": device.port,
"version": device.version, "reach": state.reach, "says": state.says,
"polled_at": device.last_polled_at, "ok_at": device.last_ok_at,
"readings": shown_readings(device, values),
"missing": [names.get(k, k) for k in device.missing or []]}
_REACH_ORDER = {"down": 0, "stale": 1, "pending": 2, "up": 3}
@snmp_bp.get("/")
@require_role(UserRole.viewer)
async def index():
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg]
async with current_app.db_sessionmaker() as db:
latest = await _latest_readings(db, device_names)
devices = await _all_devices(db)
latest = await _latest(db, [d.name for d in devices])
now = datetime.now(timezone.utc)
# Merge config + latest readings for the template
devices = []
for cfg in devices_cfg:
name = cfg.get("name") or cfg.get("host", "?")
devices.append({
"name": name,
"host": cfg.get("host", ""),
"oids": cfg.get("oids", []),
**_device_state(latest.get(name), _poll_seconds(), now),
})
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
return await render_template("snmp/index.html",
devices=devices,
poll_interval=poll_interval)
shown = [view(d, latest.get(d.name, {}), now) for d in devices]
shown.sort(key=lambda v: (_REACH_ORDER[v["reach"]], v["name"].lower()))
return await render_template("snmp/index.html", devices=shown)
def _device_binds_to_host(d: dict, keys: set[str], host_id: str) -> bool:
"""Does config device `d` belong on the host identified by `keys`
(lowercased {address, name}) / `host_id`?
# ── The editor ────────────────────────────────────────────────────────────────
A device may declare an **explicit** binding that decouples "which Steward
host this belongs to" from "where to poll" (the `host` field):
• `host_id` — the Steward Host UUID (exact match), the explicit link.
• `steward_host` — a friendly bind by Host name or address (case-insensitive).
An explicit binding is **exclusive**: a device bound to host A must not also
match host B by a coincidental poll-target string. Only when no explicit
binding is present do we fall back to the implicit `host`-string match
(backward compatible with configs that only set `host`).
"""
explicit_id = str(d.get("host_id", "")).strip()
explicit_name = str(d.get("steward_host", "")).strip().lower()
if explicit_id or explicit_name:
if explicit_id and host_id and explicit_id == host_id:
return True
return bool(explicit_name and explicit_name in keys)
return str(d.get("host", "")).strip().lower() in keys
def _devices_for_host(devices_cfg: list, address: str | None, name: str | None,
host_id: str | None = None) -> list[dict]:
"""SNMP devices that map onto a Steward host's page. Prefers an explicit
per-device binding (`host_id` / `steward_host`); otherwise falls back to
matching the device's `host` (poll target) against the host's address or
name (case-insensitive). See `_device_binds_to_host` for the precedence."""
keys = {(address or "").strip().lower(), (name or "").strip().lower()}
keys.discard("")
hid = (host_id or "").strip()
return [
d for d in devices_cfg
if isinstance(d, dict) and _device_binds_to_host(d, keys, hid)
]
@snmp_bp.get("/host/<host_id>")
@require_role(UserRole.viewer)
async def host_panel(host_id: str):
"""Per-host SNMP fragment for the Hosts hub. Surfaces any configured SNMP
device that maps to this host (by address or name) with its latest readings.
Renders nothing when no device maps, so hosts without SNMP carry no empty card.
"""
async def _hosts(db) -> list:
from steward.models.hosts import Host
return list((await db.execute(select(Host.id, Host.name).order_by(Host.name))).all())
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
async def _render_form(device: SnmpDevice | None, values: dict, error: str | None = None, status: int = 200):
async with current_app.db_sessionmaker() as db:
host = (await db.execute(
select(Host).where(Host.id == host_id))).scalar_one_or_none()
if host is None:
return ""
matched = _devices_for_host(devices_cfg, host.address, host.name, host.id)
if not matched:
return ""
names = [d.get("name") or d.get("host", "?") for d in matched]
latest = await _latest_readings(db, names)
hosts = await _hosts(db)
presets = [(pid, *(INTERFACES_LABEL if pid == "interfaces" else PRESETS[pid][:2])) for pid in PRESET_ORDER]
return await render_template("snmp/form.html", device=device, v=values, error=error, hosts=hosts,
presets=presets, units=UNITS), status
def _actor() -> tuple[str | None, str]:
return session.get("user_id"), session.get("username", "")
@snmp_bp.get("/devices/new")
@require_role(UserRole.admin)
async def new_device():
return await _render_form(None, form_values(None))
@snmp_bp.post("/devices/new")
@require_role(UserRole.admin)
async def create_device():
form = await request.form
try:
values = parse_form(form)
except FormError as exc:
return await _render_form(None, _typed(form), str(exc), 422)
async with current_app.db_sessionmaker() as db:
async with db.begin():
if await _name_taken(db, values["name"]):
return await _render_form(None, _typed(form), f"A device is already named “{values['name']}”.", 422)
device = SnmpDevice(**values)
db.add(device)
await db.flush()
device_id = device.id
await log_audit(current_app, *_actor(), "snmp.device_created", entity_type="snmp_device",
entity_id=device_id, detail={"name": values["name"], "address": values["address"]})
return redirect(f"/plugins/snmp/devices/{device_id}")
async def _name_taken(db, name: str, other_than: str | None = None) -> bool:
q = select(SnmpDevice.id).where(SnmpDevice.name == name)
if other_than:
q = q.where(SnmpDevice.id != other_than)
return (await db.execute(q)).first() is not None
def _typed(form) -> dict:
"""What was typed, to show the form again with an error. Secrets are not
echoed back."""
return {"name": form.get("name", ""), "address": form.get("address", ""), "port": form.get("port", ""),
"version": form.get("version", "2c"), "v3_user": form.get("v3_user", ""),
"v3_auth": form.get("v3_auth", "none"), "v3_priv": form.get("v3_priv", "none"),
"presets": form.getlist("presets"), "interfaces": form.get("interfaces", ""),
"custom": form.get("custom", ""), "host_id": form.get("host_id", "")}
async def _device_or_404(db, device_id: str) -> SnmpDevice:
device = await db.get(SnmpDevice, device_id)
if device is None:
abort(404)
return device
@snmp_bp.get("/devices/<device_id>/edit")
@require_role(UserRole.admin)
async def edit_device(device_id: str):
async with current_app.db_sessionmaker() as db:
device = await _device_or_404(db, device_id)
return await _render_form(device, form_values(device))
@snmp_bp.post("/devices/<device_id>/edit")
@require_role(UserRole.admin)
async def save_device(device_id: str):
form = await request.form
async with current_app.db_sessionmaker() as db:
async with db.begin():
device = await _device_or_404(db, device_id)
try:
values = parse_form(form, device)
except FormError as exc:
return await _render_form(device, {**_typed(form), **_kept_secrets(device)}, str(exc), 422)
if await _name_taken(db, values["name"], other_than=device.id):
return await _render_form(device, {**_typed(form), **_kept_secrets(device)},
f"A device is already named “{values['name']}”.", 422)
old_name = device.name
for column, value in values.items():
setattr(device, column, value)
# What it is asked, or how, may have changed: the last outcome no
# longer describes it, so the page says "not polled yet" until the next poll.
device.last_polled_at = device.last_ok_at = device.last_error = None
device.missing = []
if old_name != device.name:
# Its history is stored under its name; move it with the rename.
for model in (PluginMetric, PluginMetricLatest):
await db.execute(update(model).where(
model.source_module == "snmp", model.resource_name == old_name
).values(resource_name=device.name))
from . import scheduler
if scheduler.RATES is not None:
scheduler.RATES.forget(device_id)
detail = {"name": values["name"]}
if old_name != values["name"]:
detail["old_name"] = old_name
await log_audit(current_app, *_actor(), "snmp.device_updated", entity_type="snmp_device",
entity_id=device_id, detail=detail)
return redirect(f"/plugins/snmp/devices/{device_id}")
def _kept_secrets(device: SnmpDevice) -> dict:
return {"has_community": bool(device.community), "has_auth_key": bool(device.v3_auth_key),
"has_priv_key": bool(device.v3_priv_key)}
@snmp_bp.post("/devices/<device_id>/delete")
@require_role(UserRole.admin)
async def delete_device(device_id: str):
async with current_app.db_sessionmaker() as db:
async with db.begin():
device = await _device_or_404(db, device_id)
name = device.name
await db.delete(device)
# Its newest readings go with it, so nothing reads it as a device
# that stopped answering. Its history ages out with retention.
await db.execute(delete(PluginMetricLatest).where(
PluginMetricLatest.source_module == "snmp", PluginMetricLatest.resource_name == name))
await log_audit(current_app, *_actor(), "snmp.device_deleted", entity_type="snmp_device",
entity_id=device_id, detail={"name": name})
return redirect("/plugins/snmp/")
@snmp_bp.post("/devices/test")
@require_role(UserRole.admin)
async def test_device():
"""Ask the device on the (unsaved) form who it is and what interfaces it
has. An admin's explicit action, so it may contact the device."""
from .poller import describe_device
form = await request.form
existing = None
if form.get("device_id"):
async with current_app.db_sessionmaker() as db:
existing = await db.get(SnmpDevice, form["device_id"])
address, port, creds = form_credentials(form, existing)
if not address:
found = {"error": "Enter the device's address first."}
else:
found = await describe_device(address, port, creds)
return await render_template("snmp/_test.html", found=found, address=address, port=port)
# ── The device page, the dashboard widget, the host page's panel ─────────────
async def _history(db, device_name: str, since: datetime) -> dict[str, list]:
"""{key: [(iso time, value), ...]} for one device since `since`."""
rows = (await db.execute(
select(PluginMetric.metric_name, PluginMetric.recorded_at, PluginMetric.value)
.where(PluginMetric.source_module == "snmp", PluginMetric.resource_name == device_name,
PluginMetric.recorded_at >= since)
.order_by(PluginMetric.recorded_at))).all()
out: dict[str, list] = {}
for key, at, value in rows:
out.setdefault(key, []).append((at.isoformat(), value))
return out
RANGES = (1, 6, 24, 168)
@snmp_bp.get("/devices/<device_id>")
@require_role(UserRole.viewer)
async def device_detail(device_id: str):
hours = request.args.get("hours", type=int, default=24)
if hours not in RANGES:
hours = 24
now = datetime.now(timezone.utc)
devices = [{
"name": d.get("name") or d.get("host", "?"),
"host": d.get("host", ""),
"oids": d.get("oids", []),
**_device_state(latest.get(d.get("name") or d.get("host", "?")), _poll_seconds(), now),
"bound": bool(str(d.get("host_id", "")).strip()
or str(d.get("steward_host", "")).strip()),
} for d in matched]
return await render_template("snmp/host_panel.html", devices=devices)
async with current_app.db_sessionmaker() as db:
device = await _device_or_404(db, device_id)
latest = await _latest(db, [device.name])
hist = await _history(db, device.name, now - timedelta(hours=hours))
charts = []
for r in readings_for(device):
unit = stored_unit(r)
axis, factor = chart_axis(unit)
charts.append({"key": r.key, "name": r.name, "axis": axis,
"now": format_value(unit, latest.get(device.name, {}).get(r.key)),
"points": [(t, v * factor) for t, v in hist.get(r.key, [])]})
return await render_template("snmp/device.html", device=view(device, latest.get(device.name, {}), now),
charts=charts, charts_json=json.dumps(charts), hours=hours, ranges=RANGES)
# Labels by suffix: how a reading reads ("uptime_centisecs" → uptime 12d 4h).
# Counters (octets, packets) only ever grow, so the raw number says little;
# a device's key readings prefer the others.
_COUNTER_SUFFIXES = ("_octets", "_packets", "_pkts", "_errors", "_discards")
def snmp_reading(label: str, value: float) -> tuple[str, str]:
"""(name, value) as a reader would say them: units taken from the label's
suffix, underscores as spaces."""
if label.endswith("_centisecs"):
return label[:-len("_centisecs")].replace("_", " "), human_duration(value / 100)
if label.endswith("_pct"):
return label[:-len("_pct")].replace("_", " "), f"{value:.0f}%"
if label.endswith("_dV"):
return label[:-len("_dV")].replace("_", " "), f"{value / 10:.1f} V"
if label.endswith("_octets"):
return label[:-len("_octets")].replace("_", " "), f"{human_bytes(value)} total"
return label.replace("_", " "), f"{value:,.0f}" if float(value).is_integer() else f"{value:.4g}"
def key_readings(oids: list[dict], readings: dict, limit: int = 2) -> list[tuple[str, str]]:
"""The first `limit` readings in config order, gauges before counters."""
labels = [o.get("label") or o.get("oid") for o in oids if isinstance(o, dict)]
present = [lb for lb in labels if lb in readings]
ordered = ([lb for lb in present if not lb.endswith(_COUNTER_SUFFIXES)]
+ [lb for lb in present if lb.endswith(_COUNTER_SUFFIXES)])
return [snmp_reading(lb, readings[lb]) for lb in ordered[:limit]]
_REACH_ORDER = {"stale": 0, "pending": 1, "up": 2}
DEVICES_SHOWN = 10
@snmp_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
"""The dashboard SNMP widget (#5255), in the widget-shape order: a
verdict, devices not answering first with since when, then the rest, each
with its status in words and one or two readable readings."""
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg]
"""The dashboard SNMP widget (#5255), in widget shape: a verdict, devices
not answering first with why, then the rest with two readings each."""
async with current_app.db_sessionmaker() as db:
latest = await _latest_readings(db, device_names)
devices = await _all_devices(db)
latest = await _latest(db, [d.name for d in devices])
now = datetime.now(timezone.utc)
devices = []
for cfg in devices_cfg:
name = cfg.get("name") or cfg.get("host", "?")
state = _device_state(latest.get(name), _poll_seconds(), now)
devices.append({"name": name, **state,
"key": key_readings(cfg.get("oids", []), state["readings"])})
devices.sort(key=lambda d: _REACH_ORDER[d["reach"]])
counts = {r: sum(1 for d in devices if d["reach"] == r) for r in _REACH_ORDER}
return await render_template("snmp/widget.html",
devices=devices[:DEVICES_SHOWN],
more=max(0, len(devices) - DEVICES_SHOWN),
counts=counts, total=len(devices))
shown = [view(d, latest.get(d.name, {}), now) for d in devices]
shown.sort(key=lambda v: (_REACH_ORDER[v["reach"]], v["name"].lower()))
for v in shown:
v["key"] = [r for r in v["readings"] if r["text"] != "—"][:2]
counts = {r: sum(1 for d in shown if d["reach"] == r) for r in _REACH_ORDER}
return await render_template("snmp/widget.html", devices=shown[:DEVICES_SHOWN],
more=max(0, len(shown) - DEVICES_SHOWN), counts=counts, total=len(shown))
@snmp_bp.get("/device/<device_name>")
def devices_for_host(devices: list[SnmpDevice], host_id: str, address: str | None,
name: str | None) -> list[SnmpDevice]:
"""The devices shown on a host's page: those bound to it, and those bound
to no host whose address is the host's address or name. A binding is
exclusive, so a device bound elsewhere never matches by address."""
keys = {(address or "").strip().lower(), (name or "").strip().lower()} - {""}
return [d for d in devices
if (d.host_id == host_id if d.host_id else d.address.strip().lower() in keys)]
@snmp_bp.get("/host/<host_id>")
@require_role(UserRole.viewer)
async def device_detail(device_name: str):
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
device_cfg = next(
(d for d in devices_cfg if (d.get("name") or d.get("host")) == device_name),
None,
)
if device_cfg is None:
from quart import abort
abort(404)
hours = int(request.args.get("hours", 24))
async def host_panel(host_id: str):
"""The SNMP panel on a host's page; nothing when no device maps to it."""
from steward.models.hosts import Host
async with current_app.db_sessionmaker() as db:
hist = await _history(db, device_name, hours=hours)
import json
history_json = json.dumps(hist)
oid_labels = [o.get("label") or o.get("oid") for o in device_cfg.get("oids", [])]
return await render_template(
"snmp/device.html",
device=device_cfg,
device_name=device_name,
oid_labels=oid_labels,
history_json=history_json,
hours=hours,
)
host = await db.get(Host, host_id)
if host is None:
return ""
matched = devices_for_host(await _all_devices(db), host.id, host.address, host.name)
if not matched:
return ""
latest = await _latest(db, [d.name for d in matched])
now = datetime.now(timezone.utc)
return await render_template("snmp/host_panel.html",
devices=[{**view(d, latest.get(d.name, {}), now), "bound": bool(d.host_id)}
for d in matched])