#5210 "Problems first, with since-time, across every list" (milestone 477), second batch. - Host page: an "Alerts on this host" strip first (rules on its agent series, its containers or its monitors: alert_svc.watches_host); its monitors problems first with "Down 12m · reason"; Docker above Ansible. - Uptime page and widget: worst 24h uptime first, then 7d and 30d; hosts with no uptime yet last. - Agent panel and fleet widget: last seen as "3m ago", not a clock time. - SNMP: a device is "Reachable" only while it answered within the last 3 polls; older readings say "No answer" with their age, and every device says when it was last read (was: reachable whenever any reading existed). - Traefik: a router missing from 3 scrapes in a row is listed apart as no longer reported, by when it was last seen, instead of keeping its last state (a removed router stayed Failing). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
215 lines
8.4 KiB
Python
215 lines
8.4 KiB
Python
# plugins/snmp/routes.py
|
|
from __future__ import annotations
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from quart import Blueprint, current_app, render_template, request
|
|
from sqlalchemy import select
|
|
|
|
from steward.auth.middleware import require_role
|
|
from steward.models.users import UserRole
|
|
from steward.core.metric_series import latest_by_resource
|
|
from steward.models.metrics import PluginMetric
|
|
|
|
snmp_bp = Blueprint("snmp", __name__, template_folder="templates")
|
|
|
|
|
|
async def _latest_readings(db, device_names: list[str]) -> dict[str, dict]:
|
|
"""{device_name: {"readings": {label: latest_value}, "at": newest reading's
|
|
time}} for the devices that have any readings."""
|
|
latest = await latest_by_resource(db, "snmp", device_names)
|
|
return {name: {"readings": {m: r.value for m, r in metrics.items()},
|
|
"at": max(r.recorded_at for r in metrics.values())}
|
|
for name, metrics in latest.items() if metrics}
|
|
|
|
|
|
# A device whose newest reading is older than this many poll intervals is not
|
|
# shown as reachable: the poller has stopped getting answers from it.
|
|
STALE_POLLS = 3
|
|
|
|
|
|
def _device_state(entry: dict | None, poll_seconds: int, now: datetime) -> dict:
|
|
"""What the page says about a device's readings: {"readings", "at",
|
|
"reach"}, reach being "up" (answered within STALE_POLLS polls), "stale"
|
|
(has readings, but none recent) or "pending" (never answered)."""
|
|
if not entry:
|
|
return {"readings": {}, "at": None, "reach": "pending"}
|
|
fresh = (now - entry["at"]).total_seconds() <= STALE_POLLS * max(int(poll_seconds), 1)
|
|
return {**entry, "reach": "up" if fresh else "stale"}
|
|
|
|
|
|
def _poll_seconds() -> int:
|
|
return int(current_app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60))
|
|
|
|
|
|
async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]:
|
|
"""Return {label: [(recorded_at, value), ...]} for a single device."""
|
|
since = datetime.now(timezone.utc) - timedelta(hours=hours)
|
|
result = await db.execute(
|
|
select(PluginMetric)
|
|
.where(
|
|
PluginMetric.source_module == "snmp",
|
|
PluginMetric.resource_name == device_name,
|
|
PluginMetric.recorded_at >= since,
|
|
)
|
|
.order_by(PluginMetric.recorded_at)
|
|
)
|
|
rows = result.scalars().all()
|
|
|
|
out: dict[str, list] = {}
|
|
for row in rows:
|
|
out.setdefault(row.metric_name, []).append(
|
|
(row.recorded_at.isoformat(), row.value)
|
|
)
|
|
return out
|
|
|
|
|
|
@snmp_bp.get("/")
|
|
@require_role(UserRole.viewer)
|
|
async def index():
|
|
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
|
|
device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg]
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
latest = await _latest_readings(db, device_names)
|
|
now = datetime.now(timezone.utc)
|
|
|
|
# Merge config + latest readings for the template
|
|
devices = []
|
|
for cfg in devices_cfg:
|
|
name = cfg.get("name") or cfg.get("host", "?")
|
|
devices.append({
|
|
"name": name,
|
|
"host": cfg.get("host", ""),
|
|
"oids": cfg.get("oids", []),
|
|
**_device_state(latest.get(name), _poll_seconds(), now),
|
|
})
|
|
|
|
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
|
|
return await render_template("snmp/index.html",
|
|
devices=devices,
|
|
poll_interval=poll_interval)
|
|
|
|
|
|
def _device_binds_to_host(d: dict, keys: set[str], host_id: str) -> bool:
|
|
"""Does config device `d` belong on the host identified by `keys`
|
|
(lowercased {address, name}) / `host_id`?
|
|
|
|
A device may declare an **explicit** binding that decouples "which Steward
|
|
host this belongs to" from "where to poll" (the `host` field):
|
|
• `host_id` — the Steward Host UUID (exact match), the explicit link.
|
|
• `steward_host` — a friendly bind by Host name or address (case-insensitive).
|
|
An explicit binding is **exclusive**: a device bound to host A must not also
|
|
match host B by a coincidental poll-target string. Only when no explicit
|
|
binding is present do we fall back to the implicit `host`-string match
|
|
(backward compatible with configs that only set `host`).
|
|
"""
|
|
explicit_id = str(d.get("host_id", "")).strip()
|
|
explicit_name = str(d.get("steward_host", "")).strip().lower()
|
|
if explicit_id or explicit_name:
|
|
if explicit_id and host_id and explicit_id == host_id:
|
|
return True
|
|
return bool(explicit_name and explicit_name in keys)
|
|
return str(d.get("host", "")).strip().lower() in keys
|
|
|
|
|
|
def _devices_for_host(devices_cfg: list, address: str | None, name: str | None,
|
|
host_id: str | None = None) -> list[dict]:
|
|
"""SNMP devices that map onto a Steward host's page. Prefers an explicit
|
|
per-device binding (`host_id` / `steward_host`); otherwise falls back to
|
|
matching the device's `host` (poll target) against the host's address or
|
|
name (case-insensitive). See `_device_binds_to_host` for the precedence."""
|
|
keys = {(address or "").strip().lower(), (name or "").strip().lower()}
|
|
keys.discard("")
|
|
hid = (host_id or "").strip()
|
|
return [
|
|
d for d in devices_cfg
|
|
if isinstance(d, dict) and _device_binds_to_host(d, keys, hid)
|
|
]
|
|
|
|
|
|
@snmp_bp.get("/host/<host_id>")
|
|
@require_role(UserRole.viewer)
|
|
async def host_panel(host_id: str):
|
|
"""Per-host SNMP fragment for the Hosts hub. Surfaces any configured SNMP
|
|
device that maps to this host (by address or name) with its latest readings.
|
|
Renders nothing when no device maps, so hosts without SNMP carry no empty card.
|
|
"""
|
|
from steward.models.hosts import Host
|
|
|
|
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
|
|
async with current_app.db_sessionmaker() as db:
|
|
host = (await db.execute(
|
|
select(Host).where(Host.id == host_id))).scalar_one_or_none()
|
|
if host is None:
|
|
return ""
|
|
matched = _devices_for_host(devices_cfg, host.address, host.name, host.id)
|
|
if not matched:
|
|
return ""
|
|
names = [d.get("name") or d.get("host", "?") for d in matched]
|
|
latest = await _latest_readings(db, names)
|
|
now = datetime.now(timezone.utc)
|
|
|
|
devices = [{
|
|
"name": d.get("name") or d.get("host", "?"),
|
|
"host": d.get("host", ""),
|
|
"oids": d.get("oids", []),
|
|
**_device_state(latest.get(d.get("name") or d.get("host", "?")), _poll_seconds(), now),
|
|
"bound": bool(str(d.get("host_id", "")).strip()
|
|
or str(d.get("steward_host", "")).strip()),
|
|
} for d in matched]
|
|
return await render_template("snmp/host_panel.html", devices=devices)
|
|
|
|
|
|
@snmp_bp.get("/widget")
|
|
@require_role(UserRole.viewer)
|
|
async def widget():
|
|
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
|
|
device_names = [d.get("name") or d.get("host", "?") for d in devices_cfg]
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
latest = await _latest_readings(db, device_names[:10])
|
|
now = datetime.now(timezone.utc)
|
|
|
|
devices = []
|
|
for cfg in devices_cfg[:10]:
|
|
name = cfg.get("name") or cfg.get("host", "?")
|
|
devices.append({
|
|
"name": name,
|
|
"oids": cfg.get("oids", []),
|
|
**_device_state(latest.get(name), _poll_seconds(), now),
|
|
})
|
|
|
|
return await render_template("snmp/widget.html",
|
|
devices=devices,
|
|
total=len(devices_cfg))
|
|
|
|
|
|
@snmp_bp.get("/device/<device_name>")
|
|
@require_role(UserRole.viewer)
|
|
async def device_detail(device_name: str):
|
|
devices_cfg: list[dict] = current_app.config["PLUGINS"]["snmp"].get("devices", [])
|
|
device_cfg = next(
|
|
(d for d in devices_cfg if (d.get("name") or d.get("host")) == device_name),
|
|
None,
|
|
)
|
|
if device_cfg is None:
|
|
from quart import abort
|
|
abort(404)
|
|
|
|
hours = int(request.args.get("hours", 24))
|
|
async with current_app.db_sessionmaker() as db:
|
|
hist = await _history(db, device_name, hours=hours)
|
|
|
|
import json
|
|
history_json = json.dumps(hist)
|
|
oid_labels = [o.get("label") or o.get("oid") for o in device_cfg.get("oids", [])]
|
|
|
|
return await render_template(
|
|
"snmp/device.html",
|
|
device=device_cfg,
|
|
device_name=device_name,
|
|
oid_labels=oid_labels,
|
|
history_json=history_json,
|
|
hours=hours,
|
|
)
|