diff --git a/plugins/host_agent/templates/panel.html b/plugins/host_agent/templates/panel.html
index d106e52..5e1d615 100644
--- a/plugins/host_agent/templates/panel.html
+++ b/plugins/host_agent/templates/panel.html
@@ -4,9 +4,9 @@
Agent
{% if reporting %}
-
+
{{ 'stale' if stale else 'reporting' }}{% if reg.agent_version %} · v{{ reg.agent_version }}{% endif %}
- · last seen {{ reg.last_seen_at.strftime('%Y-%m-%d %H:%M') }}
+ · last seen {{ reg.last_seen_at|ago }}
{% elif reg %}
pending — no check-in yet
diff --git a/plugins/host_agent/templates/widget_table.html b/plugins/host_agent/templates/widget_table.html
index d0e5e50..31c892e 100644
--- a/plugins/host_agent/templates/widget_table.html
+++ b/plugins/host_agent/templates/widget_table.html
@@ -39,7 +39,7 @@
{{ r.host.name }}
{% endif %}
- {{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }}
+ {% if r.reg.last_seen_at %}seen {{ r.reg.last_seen_at|ago }}{% else %}never seen{% endif %}
diff --git a/plugins/snmp/routes.py b/plugins/snmp/routes.py
index 41d2823..b11abc1 100644
--- a/plugins/snmp/routes.py
+++ b/plugins/snmp/routes.py
@@ -13,10 +13,32 @@ from steward.models.metrics import PluginMetric
snmp_bp = Blueprint("snmp", __name__, template_folder="templates")
-async def _latest_readings(db, device_names: list[str]) -> dict[str, dict[str, float]]:
- """Return {device_name: {label: latest_value}} for all configured devices."""
+async def _latest_readings(db, device_names: list[str]) -> dict[str, dict]:
+ """{device_name: {"readings": {label: latest_value}, "at": newest reading's
+ time}} for the devices that have any readings."""
latest = await latest_by_resource(db, "snmp", device_names)
- return {name: {m: r.value for m, r in metrics.items()} for name, metrics in latest.items()}
+ return {name: {"readings": {m: r.value for m, r in metrics.items()},
+ "at": max(r.recorded_at for r in metrics.values())}
+ for name, metrics in latest.items() if metrics}
+
+
+# A device whose newest reading is older than this many poll intervals is not
+# shown as reachable: the poller has stopped getting answers from it.
+STALE_POLLS = 3
+
+
+def _device_state(entry: dict | None, poll_seconds: int, now: datetime) -> dict:
+ """What the page says about a device's readings: {"readings", "at",
+ "reach"}, reach being "up" (answered within STALE_POLLS polls), "stale"
+ (has readings, but none recent) or "pending" (never answered)."""
+ if not entry:
+ return {"readings": {}, "at": None, "reach": "pending"}
+ fresh = (now - entry["at"]).total_seconds() <= STALE_POLLS * max(int(poll_seconds), 1)
+ return {**entry, "reach": "up" if fresh else "stale"}
+
+
+def _poll_seconds() -> int:
+ return int(current_app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60))
async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]:
@@ -49,6 +71,7 @@ async def index():
async with current_app.db_sessionmaker() as db:
latest = await _latest_readings(db, device_names)
+ now = datetime.now(timezone.utc)
# Merge config + latest readings for the template
devices = []
@@ -58,7 +81,7 @@ async def index():
"name": name,
"host": cfg.get("host", ""),
"oids": cfg.get("oids", []),
- "readings": latest.get(name, {}),
+ **_device_state(latest.get(name), _poll_seconds(), now),
})
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
@@ -124,12 +147,13 @@ async def host_panel(host_id: str):
return ""
names = [d.get("name") or d.get("host", "?") for d in matched]
latest = await _latest_readings(db, names)
+ now = datetime.now(timezone.utc)
devices = [{
"name": d.get("name") or d.get("host", "?"),
"host": d.get("host", ""),
"oids": d.get("oids", []),
- "readings": latest.get(d.get("name") or d.get("host", "?"), {}),
+ **_device_state(latest.get(d.get("name") or d.get("host", "?")), _poll_seconds(), now),
"bound": bool(str(d.get("host_id", "")).strip()
or str(d.get("steward_host", "")).strip()),
} for d in matched]
@@ -144,6 +168,7 @@ async def widget():
async with current_app.db_sessionmaker() as db:
latest = await _latest_readings(db, device_names[:10])
+ now = datetime.now(timezone.utc)
devices = []
for cfg in devices_cfg[:10]:
@@ -151,7 +176,7 @@ async def widget():
devices.append({
"name": name,
"oids": cfg.get("oids", []),
- "readings": latest.get(name, {}),
+ **_device_state(latest.get(name), _poll_seconds(), now),
})
return await render_template("snmp/widget.html",
diff --git a/plugins/snmp/templates/snmp/host_panel.html b/plugins/snmp/templates/snmp/host_panel.html
index de75144..fb944f3 100644
--- a/plugins/snmp/templates/snmp/host_panel.html
+++ b/plugins/snmp/templates/snmp/host_panel.html
@@ -15,8 +15,10 @@
bound
{% endif %}
- {% if device.readings %}
- ● reachable
+ {% if device.reach == 'up' %}
+ ● reachable · read {{ device.at|ago }}
+ {% elif device.reach == 'stale' %}
+ ● no answer since {{ device.at|ago }}
{% else %}
○ no data yet
{% endif %}
diff --git a/plugins/snmp/templates/snmp/index.html b/plugins/snmp/templates/snmp/index.html
index 0ad1d43..8b551f2 100644
--- a/plugins/snmp/templates/snmp/index.html
+++ b/plugins/snmp/templates/snmp/index.html
@@ -12,9 +12,14 @@
{% for device in devices %}
- {{ status_badge('up', 'Reachable') if device.readings else status_badge('pending', 'No data yet') }}
+ {#- reach: answered within the last few polls (up), only long ago (stale),
+ or never (pending); routes._device_state. #}
+ {% if device.reach == 'up' %}{{ status_badge('up', 'Reachable') }}
+ {% elif device.reach == 'stale' %}{{ status_badge('stale', 'No answer', title="The last reading is from " ~ device.at.strftime('%Y-%m-%d %H:%M UTC') ~ "; the values below are from then") }}
+ {% else %}{{ status_badge('pending', 'No data yet') }}{% endif %}
{{ device.name }}
{{ device.host }}
+ {% if device.at %}
read {{ device.at|ago }}{% endif %}
History
diff --git a/plugins/snmp/templates/snmp/widget.html b/plugins/snmp/templates/snmp/widget.html
index 2fd0204..687da20 100644
--- a/plugins/snmp/templates/snmp/widget.html
+++ b/plugins/snmp/templates/snmp/widget.html
@@ -6,8 +6,10 @@
{{ device.name }}
- {% if device.readings %}
- ●
+ {% if device.reach == 'up' %}
+ ●
+ {% elif device.reach == 'stale' %}
+ no answer since {{ device.at|ago }}
{% else %}
○
{% endif %}
diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py
index 5da6f9f..850c178 100644
--- a/plugins/traefik/routes.py
+++ b/plugins/traefik/routes.py
@@ -1,7 +1,7 @@
# plugins/traefik/routes.py
from __future__ import annotations
from collections import defaultdict
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
from quart import Blueprint, current_app, render_template, request
from sqlalchemy import Integer, cast, func, select
import json
@@ -58,6 +58,24 @@ def router_state(latest, thresholds: dict) -> tuple[str, str | None]:
return "ok", None
+# A router missing from this many scrapes in a row is no longer reported:
+# Traefik dropped it (a removed service), so its last state is history.
+GONE_AFTER_SCRAPES = 3
+
+
+def gone_routers(routers: list[dict], scrape_seconds: int) -> tuple[list[dict], list[dict]]:
+ """(current, gone): a router whose newest row trails the newest scrape of
+ any router by GONE_AFTER_SCRAPES intervals is gone, newest-seen first."""
+ if not routers:
+ return [], []
+ newest = max(r["latest"].scraped_at for r in routers)
+ cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
+ current = [r for r in routers if r["latest"].scraped_at >= cutoff]
+ gone = sorted((r for r in routers if r["latest"].scraped_at < cutoff),
+ key=lambda r: r["latest"].scraped_at, reverse=True)
+ return current, gone
+
+
def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]:
"""(problems, healthy): failing routers before slow or noisy ones, each
group by name; healthy routers by name."""
@@ -171,11 +189,14 @@ async def rows():
"days_remaining": days,
})
- problems, healthy = split_routers(router_data)
+ current, gone = gone_routers(
+ router_data, current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"])
+ problems, healthy = split_routers(current)
return await render_template(
"traefik/rows.html",
problems=problems,
healthy=healthy,
+ gone=gone,
error_4xx=ERROR_4XX_PCT,
error_5xx=ERROR_5XX_PCT,
global_stat=global_stat,
diff --git a/plugins/traefik/templates/traefik/rows.html b/plugins/traefik/templates/traefik/rows.html
index 88ae3bf..55cc343 100644
--- a/plugins/traefik/templates/traefik/rows.html
+++ b/plugins/traefik/templates/traefik/rows.html
@@ -37,7 +37,7 @@
{% endmacro %}
-{% if problems or healthy or certs %}
+{% if problems or healthy or gone or certs %}
{# ── Routers that need attention ─────────────────────────────────────────── #}
{% if problems %}
@@ -67,6 +67,22 @@
+{% endif %}
+
+{# ── Routers Traefik no longer reports: their last numbers are history, not a
+ state, so they are listed by when they were last seen and nothing else. ── #}
+{% if gone %}
+
+ {{ gone|length }} router{{ 's' if gone|length != 1 }} no longer reported
+ Traefik stopped reporting these, usually because the service behind them was removed.
+
+ {% for r in gone %}
+ - {{ r.name }} last seen {{ r.latest.scraped_at|ago }}
+ {% endfor %}
+
+
+{% endif %}
+{% if healthy %}