feat(ui): host page leads with its alerts; old readings say how old (#5210)

#5210 "Problems first, with since-time, across every list" (milestone 477),
second batch.

- Host page: an "Alerts on this host" strip first (rules on its agent
  series, its containers or its monitors: alert_svc.watches_host); its
  monitors problems first with "Down 12m · reason"; Docker above Ansible.
- Uptime page and widget: worst 24h uptime first, then 7d and 30d; hosts
  with no uptime yet last.
- Agent panel and fleet widget: last seen as "3m ago", not a clock time.
- SNMP: a device is "Reachable" only while it answered within the last 3
  polls; older readings say "No answer" with their age, and every device
  says when it was last read (was: reachable whenever any reading existed).
- Traefik: a router missing from 3 scrapes in a row is listed apart as no
  longer reported, by when it was last seen, instead of keeping its last
  state (a removed router stayed Failing).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/snmp/routes.py

@@ -13,10 +13,32 @@ from steward.models.metrics import PluginMetric
 snmp_bp = Blueprint("snmp", __name__, template_folder="templates")
 
 
-async def _latest_readings(db, device_names: list[str]) -> dict[str, dict[str, float]]:
-    """Return {device_name: {label: latest_value}} for all configured devices."""
+async def _latest_readings(db, device_names: list[str]) -> dict[str, dict]:
+    """{device_name: {"readings": {label: latest_value}, "at": newest reading's
+    time}} for the devices that have any readings."""
     latest = await latest_by_resource(db, "snmp", device_names)
-    return {name: {m: r.value for m, r in metrics.items()} for name, metrics in latest.items()}
+    return {name: {"readings": {m: r.value for m, r in metrics.items()},
+                   "at": max(r.recorded_at for r in metrics.values())}
+            for name, metrics in latest.items() if metrics}
+
+
+# A device whose newest reading is older than this many poll intervals is not
+# shown as reachable: the poller has stopped getting answers from it.
+STALE_POLLS = 3
+
+
+def _device_state(entry: dict | None, poll_seconds: int, now: datetime) -> dict:
+    """What the page says about a device's readings: {"readings", "at",
+    "reach"}, reach being "up" (answered within STALE_POLLS polls), "stale"
+    (has readings, but none recent) or "pending" (never answered)."""
+    if not entry:
+        return {"readings": {}, "at": None, "reach": "pending"}
+    fresh = (now - entry["at"]).total_seconds() <= STALE_POLLS * max(int(poll_seconds), 1)
+    return {**entry, "reach": "up" if fresh else "stale"}
+
+
+def _poll_seconds() -> int:
+    return int(current_app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60))
 
 
 async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]:
@@ -49,6 +71,7 @@ async def index():
 
     async with current_app.db_sessionmaker() as db:
         latest = await _latest_readings(db, device_names)
+    now = datetime.now(timezone.utc)
 
     # Merge config + latest readings for the template
     devices = []
@@ -58,7 +81,7 @@ async def index():
             "name": name,
             "host": cfg.get("host", ""),
             "oids": cfg.get("oids", []),
-            "readings": latest.get(name, {}),
+            **_device_state(latest.get(name), _poll_seconds(), now),
         })
 
     poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
@@ -124,12 +147,13 @@ async def host_panel(host_id: str):
             return ""
         names = [d.get("name") or d.get("host", "?") for d in matched]
         latest = await _latest_readings(db, names)
+    now = datetime.now(timezone.utc)
 
     devices = [{
         "name": d.get("name") or d.get("host", "?"),
         "host": d.get("host", ""),
         "oids": d.get("oids", []),
-        "readings": latest.get(d.get("name") or d.get("host", "?"), {}),
+        **_device_state(latest.get(d.get("name") or d.get("host", "?")), _poll_seconds(), now),
         "bound": bool(str(d.get("host_id", "")).strip()
                       or str(d.get("steward_host", "")).strip()),
     } for d in matched]
@@ -144,6 +168,7 @@ async def widget():
 
     async with current_app.db_sessionmaker() as db:
         latest = await _latest_readings(db, device_names[:10])
+    now = datetime.now(timezone.utc)
 
     devices = []
     for cfg in devices_cfg[:10]:
@@ -151,7 +176,7 @@ async def widget():
         devices.append({
             "name": name,
             "oids": cfg.get("oids", []),
-            "readings": latest.get(name, {}),
+            **_device_state(latest.get(name), _poll_seconds(), now),
         })
 
     return await render_template("snmp/widget.html",

plugins/snmp/templates/snmp/host_panel.html

@@ -15,8 +15,10 @@
       <span title="Explicitly bound to this host (host_id / steward_host)"
             style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:3px;padding:0.05rem 0.4rem;">bound</span>
       {% endif %}
-      {% if device.readings %}
-      <span style="font-size:0.74rem;color:var(--green);">&#9679; reachable</span>
+      {% if device.reach == 'up' %}
+      <span style="font-size:0.74rem;" class="tone-ok">&#9679; reachable · read {{ device.at|ago }}</span>
+      {% elif device.reach == 'stale' %}
+      <span style="font-size:0.74rem;" class="tone-warn" title="The values below are from then">&#9679; no answer since {{ device.at|ago }}</span>
       {% else %}
       <span style="font-size:0.74rem;color:var(--text-dim);">&#9675; no data yet</span>
       {% endif %}

plugins/snmp/templates/snmp/index.html

@@ -12,9 +12,14 @@
 {% for device in devices %}
 <section class="card" aria-labelledby="snmp-{{ loop.index }}">
   <div class="host-health-head">
-    {{ status_badge('up', 'Reachable') if device.readings else status_badge('pending', 'No data yet') }}
+    {#- reach: answered within the last few polls (up), only long ago (stale),
+        or never (pending); routes._device_state. #}
+    {% if device.reach == 'up' %}{{ status_badge('up', 'Reachable') }}
+    {% elif device.reach == 'stale' %}{{ status_badge('stale', 'No answer', title="The last reading is from " ~ device.at.strftime('%Y-%m-%d %H:%M UTC') ~ "; the values below are from then") }}
+    {% else %}{{ status_badge('pending', 'No data yet') }}{% endif %}
     <h2 id="snmp-{{ loop.index }}" class="section-title" style="margin:0;">{{ device.name }}</h2>
     <span class="field-hint mono">{{ device.host }}</span>
+    {% if device.at %}<span class="field-hint">read {{ device.at|ago }}</span>{% endif %}
     <a href="/plugins/snmp/device/{{ device.name }}" class="btn btn-sm btn-ghost" style="margin-left:auto;">History</a>
   </div>
 

plugins/snmp/templates/snmp/widget.html

@@ -6,8 +6,10 @@
     <span style="font-size:0.82rem;font-weight:500;flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">
       {{ device.name }}
     </span>
-    {% if device.readings %}
-    <span style="font-size:0.7rem;color:var(--green);">&#9679;</span>
+    {% if device.reach == 'up' %}
+    <span style="font-size:0.7rem;" class="tone-ok" title="Reachable · read {{ device.at.strftime('%H:%M UTC') }}">&#9679;</span>
+    {% elif device.reach == 'stale' %}
+    <span style="font-size:0.7rem;" class="tone-warn">no answer since {{ device.at|ago }}</span>
     {% else %}
     <span style="font-size:0.7rem;color:var(--text-dim);">&#9675;</span>
     {% endif %}