feat(ui): host page leads with its alerts; old readings say how old (#5210)

#5210 "Problems first, with since-time, across every list" (milestone 477),
second batch.

- Host page: an "Alerts on this host" strip first (rules on its agent
  series, its containers or its monitors: alert_svc.watches_host); its
  monitors problems first with "Down 12m · reason"; Docker above Ansible.
- Uptime page and widget: worst 24h uptime first, then 7d and 30d; hosts
  with no uptime yet last.
- Agent panel and fleet widget: last seen as "3m ago", not a clock time.
- SNMP: a device is "Reachable" only while it answered within the last 3
  polls; older readings say "No answer" with their age, and every device
  says when it was last read (was: reachable whenever any reading existed).
- Traefik: a router missing from 3 scrapes in a row is listed apart as no
  longer reported, by when it was last seen, instead of keeping its last
  state (a removed router stayed Failing).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/templates/panel.html

@@ -4,9 +4,9 @@
   <div style="display:flex;align-items:baseline;justify-content:space-between;margin-bottom:0.75rem;gap:1rem;flex-wrap:wrap;">
     <h3 class="section-title" style="margin-bottom:0;">Agent</h3>
     {% if reporting %}
-    <span style="font-size:0.78rem;color:{{ 'var(--yellow)' if stale else 'var(--green)' }};">
+    <span style="font-size:0.78rem;" class="tone-{{ 'warn' if stale else 'ok' }}">
       {{ 'stale' if stale else 'reporting' }}{% if reg.agent_version %} · v{{ reg.agent_version }}{% endif %}
-      · last seen {{ reg.last_seen_at.strftime('%Y-%m-%d %H:%M') }}
+      · last seen {{ reg.last_seen_at|ago }}
     </span>
     {% elif reg %}
     <span style="font-size:0.78rem;color:var(--text-muted);">pending — no check-in yet</span>

plugins/host_agent/templates/widget_table.html

@@ -39,7 +39,7 @@
         <span style="font-weight:500;font-size:0.85rem;overflow-wrap:anywhere;">{{ r.host.name }}</span>
         {% endif %}
         <span style="display:block;font-size:0.7rem;color:var(--text-dim);">
-          {{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }}
+          {% if r.reg.last_seen_at %}seen {{ r.reg.last_seen_at|ago }}{% else %}never seen{% endif %}
         </span>
       </span>
     </div>

plugins/snmp/routes.py

@@ -13,10 +13,32 @@ from steward.models.metrics import PluginMetric
 snmp_bp = Blueprint("snmp", __name__, template_folder="templates")
 
 
-async def _latest_readings(db, device_names: list[str]) -> dict[str, dict[str, float]]:
-    """Return {device_name: {label: latest_value}} for all configured devices."""
+async def _latest_readings(db, device_names: list[str]) -> dict[str, dict]:
+    """{device_name: {"readings": {label: latest_value}, "at": newest reading's
+    time}} for the devices that have any readings."""
     latest = await latest_by_resource(db, "snmp", device_names)
-    return {name: {m: r.value for m, r in metrics.items()} for name, metrics in latest.items()}
+    return {name: {"readings": {m: r.value for m, r in metrics.items()},
+                   "at": max(r.recorded_at for r in metrics.values())}
+            for name, metrics in latest.items() if metrics}
+
+
+# A device whose newest reading is older than this many poll intervals is not
+# shown as reachable: the poller has stopped getting answers from it.
+STALE_POLLS = 3
+
+
+def _device_state(entry: dict | None, poll_seconds: int, now: datetime) -> dict:
+    """What the page says about a device's readings: {"readings", "at",
+    "reach"}, reach being "up" (answered within STALE_POLLS polls), "stale"
+    (has readings, but none recent) or "pending" (never answered)."""
+    if not entry:
+        return {"readings": {}, "at": None, "reach": "pending"}
+    fresh = (now - entry["at"]).total_seconds() <= STALE_POLLS * max(int(poll_seconds), 1)
+    return {**entry, "reach": "up" if fresh else "stale"}
+
+
+def _poll_seconds() -> int:
+    return int(current_app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60))
 
 
 async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]:
@@ -49,6 +71,7 @@ async def index():
 
     async with current_app.db_sessionmaker() as db:
         latest = await _latest_readings(db, device_names)
+    now = datetime.now(timezone.utc)
 
     # Merge config + latest readings for the template
     devices = []
@@ -58,7 +81,7 @@ async def index():
             "name": name,
             "host": cfg.get("host", ""),
             "oids": cfg.get("oids", []),
-            "readings": latest.get(name, {}),
+            **_device_state(latest.get(name), _poll_seconds(), now),
         })
 
     poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
@@ -124,12 +147,13 @@ async def host_panel(host_id: str):
             return ""
         names = [d.get("name") or d.get("host", "?") for d in matched]
         latest = await _latest_readings(db, names)
+    now = datetime.now(timezone.utc)
 
     devices = [{
         "name": d.get("name") or d.get("host", "?"),
         "host": d.get("host", ""),
         "oids": d.get("oids", []),
-        "readings": latest.get(d.get("name") or d.get("host", "?"), {}),
+        **_device_state(latest.get(d.get("name") or d.get("host", "?")), _poll_seconds(), now),
         "bound": bool(str(d.get("host_id", "")).strip()
                       or str(d.get("steward_host", "")).strip()),
     } for d in matched]
@@ -144,6 +168,7 @@ async def widget():
 
     async with current_app.db_sessionmaker() as db:
         latest = await _latest_readings(db, device_names[:10])
+    now = datetime.now(timezone.utc)
 
     devices = []
     for cfg in devices_cfg[:10]:
@@ -151,7 +176,7 @@ async def widget():
         devices.append({
             "name": name,
             "oids": cfg.get("oids", []),
-            "readings": latest.get(name, {}),
+            **_device_state(latest.get(name), _poll_seconds(), now),
         })
 
     return await render_template("snmp/widget.html",

plugins/snmp/templates/snmp/host_panel.html

@@ -15,8 +15,10 @@
       <span title="Explicitly bound to this host (host_id / steward_host)"
             style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:3px;padding:0.05rem 0.4rem;">bound</span>
       {% endif %}
-      {% if device.readings %}
-      <span style="font-size:0.74rem;color:var(--green);">&#9679; reachable</span>
+      {% if device.reach == 'up' %}
+      <span style="font-size:0.74rem;" class="tone-ok">&#9679; reachable · read {{ device.at|ago }}</span>
+      {% elif device.reach == 'stale' %}
+      <span style="font-size:0.74rem;" class="tone-warn" title="The values below are from then">&#9679; no answer since {{ device.at|ago }}</span>
       {% else %}
       <span style="font-size:0.74rem;color:var(--text-dim);">&#9675; no data yet</span>
       {% endif %}

plugins/snmp/templates/snmp/index.html

@@ -12,9 +12,14 @@
 {% for device in devices %}
 <section class="card" aria-labelledby="snmp-{{ loop.index }}">
   <div class="host-health-head">
-    {{ status_badge('up', 'Reachable') if device.readings else status_badge('pending', 'No data yet') }}
+    {#- reach: answered within the last few polls (up), only long ago (stale),
+        or never (pending); routes._device_state. #}
+    {% if device.reach == 'up' %}{{ status_badge('up', 'Reachable') }}
+    {% elif device.reach == 'stale' %}{{ status_badge('stale', 'No answer', title="The last reading is from " ~ device.at.strftime('%Y-%m-%d %H:%M UTC') ~ "; the values below are from then") }}
+    {% else %}{{ status_badge('pending', 'No data yet') }}{% endif %}
     <h2 id="snmp-{{ loop.index }}" class="section-title" style="margin:0;">{{ device.name }}</h2>
     <span class="field-hint mono">{{ device.host }}</span>
+    {% if device.at %}<span class="field-hint">read {{ device.at|ago }}</span>{% endif %}
     <a href="/plugins/snmp/device/{{ device.name }}" class="btn btn-sm btn-ghost" style="margin-left:auto;">History</a>
   </div>
 

plugins/snmp/templates/snmp/widget.html

@@ -6,8 +6,10 @@
     <span style="font-size:0.82rem;font-weight:500;flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">
       {{ device.name }}
     </span>
-    {% if device.readings %}
-    <span style="font-size:0.7rem;color:var(--green);">&#9679;</span>
+    {% if device.reach == 'up' %}
+    <span style="font-size:0.7rem;" class="tone-ok" title="Reachable · read {{ device.at.strftime('%H:%M UTC') }}">&#9679;</span>
+    {% elif device.reach == 'stale' %}
+    <span style="font-size:0.7rem;" class="tone-warn">no answer since {{ device.at|ago }}</span>
     {% else %}
     <span style="font-size:0.7rem;color:var(--text-dim);">&#9675;</span>
     {% endif %}

plugins/traefik/routes.py

@@ -1,7 +1,7 @@
 # plugins/traefik/routes.py
 from __future__ import annotations
 from collections import defaultdict
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from quart import Blueprint, current_app, render_template, request
 from sqlalchemy import Integer, cast, func, select
 import json
@@ -58,6 +58,24 @@ def router_state(latest, thresholds: dict) -> tuple[str, str | None]:
     return "ok", None
 
 
+# A router missing from this many scrapes in a row is no longer reported:
+# Traefik dropped it (a removed service), so its last state is history.
+GONE_AFTER_SCRAPES = 3
+
+
+def gone_routers(routers: list[dict], scrape_seconds: int) -> tuple[list[dict], list[dict]]:
+    """(current, gone): a router whose newest row trails the newest scrape of
+    any router by GONE_AFTER_SCRAPES intervals is gone, newest-seen first."""
+    if not routers:
+        return [], []
+    newest = max(r["latest"].scraped_at for r in routers)
+    cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
+    current = [r for r in routers if r["latest"].scraped_at >= cutoff]
+    gone = sorted((r for r in routers if r["latest"].scraped_at < cutoff),
+                  key=lambda r: r["latest"].scraped_at, reverse=True)
+    return current, gone
+
+
 def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]:
     """(problems, healthy): failing routers before slow or noisy ones, each
     group by name; healthy routers by name."""
@@ -171,11 +189,14 @@ async def rows():
             "days_remaining": days,
         })
 
-    problems, healthy = split_routers(router_data)
+    current, gone = gone_routers(
+        router_data, current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"])
+    problems, healthy = split_routers(current)
     return await render_template(
         "traefik/rows.html",
         problems=problems,
         healthy=healthy,
+        gone=gone,
         error_4xx=ERROR_4XX_PCT,
         error_5xx=ERROR_5XX_PCT,
         global_stat=global_stat,

plugins/traefik/templates/traefik/rows.html

@@ -37,7 +37,7 @@
 </div>
 {% endmacro %}
 
-{% if problems or healthy or certs %}
+{% if problems or healthy or gone or certs %}
 {# ── Routers that need attention ─────────────────────────────────────────── #}
 {% if problems %}
 <div style="column-width:340px;column-gap:1rem;">
@@ -67,6 +67,22 @@
   </table>
   </div>
 </details>
+{% endif %}
+
+{# ── Routers Traefik no longer reports: their last numbers are history, not a
+   state, so they are listed by when they were last seen and nothing else. ── #}
+{% if gone %}
+<details class="card" style="margin-bottom:1rem;">
+  <summary class="section-title" style="cursor:pointer;">{{ gone|length }} router{{ 's' if gone|length != 1 }} no longer reported</summary>
+  <p class="field-hint" style="margin:0.5rem 0;">Traefik stopped reporting these, usually because the service behind them was removed.</p>
+  <ul class="problem-list">
+    {% for r in gone %}
+    <li><span style="word-break:break-all;">{{ r.name }}</span> <span class="problem-detail">last seen {{ r.latest.scraped_at|ago }}</span></li>
+    {% endfor %}
+  </ul>
+</details>
+{% endif %}
+{% if healthy %}
 <script>
   // The fragment is re-polled; keep the healthy list as the reader left it.
   (function () {