diff --git a/plugins/host_agent/templates/panel.html b/plugins/host_agent/templates/panel.html index d106e52..5e1d615 100644 --- a/plugins/host_agent/templates/panel.html +++ b/plugins/host_agent/templates/panel.html @@ -4,9 +4,9 @@

Agent

{% if reporting %} - + {{ 'stale' if stale else 'reporting' }}{% if reg.agent_version %} · v{{ reg.agent_version }}{% endif %} - · last seen {{ reg.last_seen_at.strftime('%Y-%m-%d %H:%M') }} + · last seen {{ reg.last_seen_at|ago }} {% elif reg %} pending — no check-in yet diff --git a/plugins/host_agent/templates/widget_table.html b/plugins/host_agent/templates/widget_table.html index d0e5e50..31c892e 100644 --- a/plugins/host_agent/templates/widget_table.html +++ b/plugins/host_agent/templates/widget_table.html @@ -39,7 +39,7 @@ {{ r.host.name }} {% endif %} - {{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }} + {% if r.reg.last_seen_at %}seen {{ r.reg.last_seen_at|ago }}{% else %}never seen{% endif %}
diff --git a/plugins/snmp/routes.py b/plugins/snmp/routes.py index 41d2823..b11abc1 100644 --- a/plugins/snmp/routes.py +++ b/plugins/snmp/routes.py @@ -13,10 +13,32 @@ from steward.models.metrics import PluginMetric snmp_bp = Blueprint("snmp", __name__, template_folder="templates") -async def _latest_readings(db, device_names: list[str]) -> dict[str, dict[str, float]]: - """Return {device_name: {label: latest_value}} for all configured devices.""" +async def _latest_readings(db, device_names: list[str]) -> dict[str, dict]: + """{device_name: {"readings": {label: latest_value}, "at": newest reading's + time}} for the devices that have any readings.""" latest = await latest_by_resource(db, "snmp", device_names) - return {name: {m: r.value for m, r in metrics.items()} for name, metrics in latest.items()} + return {name: {"readings": {m: r.value for m, r in metrics.items()}, + "at": max(r.recorded_at for r in metrics.values())} + for name, metrics in latest.items() if metrics} + + +# A device whose newest reading is older than this many poll intervals is not +# shown as reachable: the poller has stopped getting answers from it. +STALE_POLLS = 3 + + +def _device_state(entry: dict | None, poll_seconds: int, now: datetime) -> dict: + """What the page says about a device's readings: {"readings", "at", + "reach"}, reach being "up" (answered within STALE_POLLS polls), "stale" + (has readings, but none recent) or "pending" (never answered).""" + if not entry: + return {"readings": {}, "at": None, "reach": "pending"} + fresh = (now - entry["at"]).total_seconds() <= STALE_POLLS * max(int(poll_seconds), 1) + return {**entry, "reach": "up" if fresh else "stale"} + + +def _poll_seconds() -> int: + return int(current_app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60)) async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]: @@ -49,6 +71,7 @@ async def index(): async with current_app.db_sessionmaker() as db: latest = await _latest_readings(db, device_names) + now = datetime.now(timezone.utc) # Merge config + latest readings for the template devices = [] @@ -58,7 +81,7 @@ async def index(): "name": name, "host": cfg.get("host", ""), "oids": cfg.get("oids", []), - "readings": latest.get(name, {}), + **_device_state(latest.get(name), _poll_seconds(), now), }) poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) @@ -124,12 +147,13 @@ async def host_panel(host_id: str): return "" names = [d.get("name") or d.get("host", "?") for d in matched] latest = await _latest_readings(db, names) + now = datetime.now(timezone.utc) devices = [{ "name": d.get("name") or d.get("host", "?"), "host": d.get("host", ""), "oids": d.get("oids", []), - "readings": latest.get(d.get("name") or d.get("host", "?"), {}), + **_device_state(latest.get(d.get("name") or d.get("host", "?")), _poll_seconds(), now), "bound": bool(str(d.get("host_id", "")).strip() or str(d.get("steward_host", "")).strip()), } for d in matched] @@ -144,6 +168,7 @@ async def widget(): async with current_app.db_sessionmaker() as db: latest = await _latest_readings(db, device_names[:10]) + now = datetime.now(timezone.utc) devices = [] for cfg in devices_cfg[:10]: @@ -151,7 +176,7 @@ async def widget(): devices.append({ "name": name, "oids": cfg.get("oids", []), - "readings": latest.get(name, {}), + **_device_state(latest.get(name), _poll_seconds(), now), }) return await render_template("snmp/widget.html", diff --git a/plugins/snmp/templates/snmp/host_panel.html b/plugins/snmp/templates/snmp/host_panel.html index de75144..fb944f3 100644 --- a/plugins/snmp/templates/snmp/host_panel.html +++ b/plugins/snmp/templates/snmp/host_panel.html @@ -15,8 +15,10 @@ bound {% endif %} - {% if device.readings %} - ● reachable + {% if device.reach == 'up' %} + ● reachable · read {{ device.at|ago }} + {% elif device.reach == 'stale' %} + ● no answer since {{ device.at|ago }} {% else %} ○ no data yet {% endif %} diff --git a/plugins/snmp/templates/snmp/index.html b/plugins/snmp/templates/snmp/index.html index 0ad1d43..8b551f2 100644 --- a/plugins/snmp/templates/snmp/index.html +++ b/plugins/snmp/templates/snmp/index.html @@ -12,9 +12,14 @@ {% for device in devices %}
- {{ status_badge('up', 'Reachable') if device.readings else status_badge('pending', 'No data yet') }} + {#- reach: answered within the last few polls (up), only long ago (stale), + or never (pending); routes._device_state. #} + {% if device.reach == 'up' %}{{ status_badge('up', 'Reachable') }} + {% elif device.reach == 'stale' %}{{ status_badge('stale', 'No answer', title="The last reading is from " ~ device.at.strftime('%Y-%m-%d %H:%M UTC') ~ "; the values below are from then") }} + {% else %}{{ status_badge('pending', 'No data yet') }}{% endif %}

{{ device.name }}

{{ device.host }} + {% if device.at %}read {{ device.at|ago }}{% endif %} History
diff --git a/plugins/snmp/templates/snmp/widget.html b/plugins/snmp/templates/snmp/widget.html index 2fd0204..687da20 100644 --- a/plugins/snmp/templates/snmp/widget.html +++ b/plugins/snmp/templates/snmp/widget.html @@ -6,8 +6,10 @@ {{ device.name }} - {% if device.readings %} - ● + {% if device.reach == 'up' %} + ● + {% elif device.reach == 'stale' %} + no answer since {{ device.at|ago }} {% else %} ○ {% endif %} diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py index 5da6f9f..850c178 100644 --- a/plugins/traefik/routes.py +++ b/plugins/traefik/routes.py @@ -1,7 +1,7 @@ # plugins/traefik/routes.py from __future__ import annotations from collections import defaultdict -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from quart import Blueprint, current_app, render_template, request from sqlalchemy import Integer, cast, func, select import json @@ -58,6 +58,24 @@ def router_state(latest, thresholds: dict) -> tuple[str, str | None]: return "ok", None +# A router missing from this many scrapes in a row is no longer reported: +# Traefik dropped it (a removed service), so its last state is history. +GONE_AFTER_SCRAPES = 3 + + +def gone_routers(routers: list[dict], scrape_seconds: int) -> tuple[list[dict], list[dict]]: + """(current, gone): a router whose newest row trails the newest scrape of + any router by GONE_AFTER_SCRAPES intervals is gone, newest-seen first.""" + if not routers: + return [], [] + newest = max(r["latest"].scraped_at for r in routers) + cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1)) + current = [r for r in routers if r["latest"].scraped_at >= cutoff] + gone = sorted((r for r in routers if r["latest"].scraped_at < cutoff), + key=lambda r: r["latest"].scraped_at, reverse=True) + return current, gone + + def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]: """(problems, healthy): failing routers before slow or noisy ones, each group by name; healthy routers by name.""" @@ -171,11 +189,14 @@ async def rows(): "days_remaining": days, }) - problems, healthy = split_routers(router_data) + current, gone = gone_routers( + router_data, current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"]) + problems, healthy = split_routers(current) return await render_template( "traefik/rows.html", problems=problems, healthy=healthy, + gone=gone, error_4xx=ERROR_4XX_PCT, error_5xx=ERROR_5XX_PCT, global_stat=global_stat, diff --git a/plugins/traefik/templates/traefik/rows.html b/plugins/traefik/templates/traefik/rows.html index 88ae3bf..55cc343 100644 --- a/plugins/traefik/templates/traefik/rows.html +++ b/plugins/traefik/templates/traefik/rows.html @@ -37,7 +37,7 @@ {% endmacro %} -{% if problems or healthy or certs %} +{% if problems or healthy or gone or certs %} {# ── Routers that need attention ─────────────────────────────────────────── #} {% if problems %}
@@ -67,6 +67,22 @@
+{% endif %} + +{# ── Routers Traefik no longer reports: their last numbers are history, not a + state, so they are listed by when they were last seen and nothing else. ── #} +{% if gone %} +
+ {{ gone|length }} router{{ 's' if gone|length != 1 }} no longer reported +

Traefik stopped reporting these, usually because the service behind them was removed.

+ +
+{% endif %} +{% if healthy %}