feat(ui): host page leads with its alerts; old readings say how old (#5210)

#5210 "Problems first, with since-time, across every list" (milestone 477),
second batch.

- Host page: an "Alerts on this host" strip first (rules on its agent
  series, its containers or its monitors: alert_svc.watches_host); its
  monitors problems first with "Down 12m · reason"; Docker above Ansible.
- Uptime page and widget: worst 24h uptime first, then 7d and 30d; hosts
  with no uptime yet last.
- Agent panel and fleet widget: last seen as "3m ago", not a clock time.
- SNMP: a device is "Reachable" only while it answered within the last 3
  polls; older readings say "No answer" with their age, and every device
  says when it was last read (was: reachable whenever any reading existed).
- Traefik: a router missing from 3 scrapes in a row is listed apart as no
  longer reported, by when it was last seen, instead of keeping its last
  state (a removed router stayed Failing).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/templates/panel.html

@@ -4,9 +4,9 @@
   <div style="display:flex;align-items:baseline;justify-content:space-between;margin-bottom:0.75rem;gap:1rem;flex-wrap:wrap;">
     <h3 class="section-title" style="margin-bottom:0;">Agent</h3>
     {% if reporting %}
-    <span style="font-size:0.78rem;color:{{ 'var(--yellow)' if stale else 'var(--green)' }};">
+    <span style="font-size:0.78rem;" class="tone-{{ 'warn' if stale else 'ok' }}">
       {{ 'stale' if stale else 'reporting' }}{% if reg.agent_version %} · v{{ reg.agent_version }}{% endif %}
-      · last seen {{ reg.last_seen_at.strftime('%Y-%m-%d %H:%M') }}
+      · last seen {{ reg.last_seen_at|ago }}
     </span>
     {% elif reg %}
     <span style="font-size:0.78rem;color:var(--text-muted);">pending — no check-in yet</span>

plugins/host_agent/templates/widget_table.html

@@ -39,7 +39,7 @@
         <span style="font-weight:500;font-size:0.85rem;overflow-wrap:anywhere;">{{ r.host.name }}</span>
         {% endif %}
         <span style="display:block;font-size:0.7rem;color:var(--text-dim);">
-          {{ r.reg.last_seen_at.strftime("%H:%M:%S") if r.reg.last_seen_at else "never" }}
+          {% if r.reg.last_seen_at %}seen {{ r.reg.last_seen_at|ago }}{% else %}never seen{% endif %}
         </span>
       </span>
     </div>

plugins/snmp/routes.py

@@ -13,10 +13,32 @@ from steward.models.metrics import PluginMetric
 snmp_bp = Blueprint("snmp", __name__, template_folder="templates")
 
 
-async def _latest_readings(db, device_names: list[str]) -> dict[str, dict[str, float]]:
-    """Return {device_name: {label: latest_value}} for all configured devices."""
+async def _latest_readings(db, device_names: list[str]) -> dict[str, dict]:
+    """{device_name: {"readings": {label: latest_value}, "at": newest reading's
+    time}} for the devices that have any readings."""
     latest = await latest_by_resource(db, "snmp", device_names)
-    return {name: {m: r.value for m, r in metrics.items()} for name, metrics in latest.items()}
+    return {name: {"readings": {m: r.value for m, r in metrics.items()},
+                   "at": max(r.recorded_at for r in metrics.values())}
+            for name, metrics in latest.items() if metrics}
+
+
+# A device whose newest reading is older than this many poll intervals is not
+# shown as reachable: the poller has stopped getting answers from it.
+STALE_POLLS = 3
+
+
+def _device_state(entry: dict | None, poll_seconds: int, now: datetime) -> dict:
+    """What the page says about a device's readings: {"readings", "at",
+    "reach"}, reach being "up" (answered within STALE_POLLS polls), "stale"
+    (has readings, but none recent) or "pending" (never answered)."""
+    if not entry:
+        return {"readings": {}, "at": None, "reach": "pending"}
+    fresh = (now - entry["at"]).total_seconds() <= STALE_POLLS * max(int(poll_seconds), 1)
+    return {**entry, "reach": "up" if fresh else "stale"}
+
+
+def _poll_seconds() -> int:
+    return int(current_app.config["PLUGINS"]["snmp"].get("poll_interval_seconds", 60))
 
 
 async def _history(db, device_name: str, hours: int = 24) -> dict[str, list]:
@@ -49,6 +71,7 @@ async def index():
 
     async with current_app.db_sessionmaker() as db:
         latest = await _latest_readings(db, device_names)
+    now = datetime.now(timezone.utc)
 
     # Merge config + latest readings for the template
     devices = []
@@ -58,7 +81,7 @@ async def index():
             "name": name,
             "host": cfg.get("host", ""),
             "oids": cfg.get("oids", []),
-            "readings": latest.get(name, {}),
+            **_device_state(latest.get(name), _poll_seconds(), now),
         })
 
     poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
@@ -124,12 +147,13 @@ async def host_panel(host_id: str):
             return ""
         names = [d.get("name") or d.get("host", "?") for d in matched]
         latest = await _latest_readings(db, names)
+    now = datetime.now(timezone.utc)
 
     devices = [{
         "name": d.get("name") or d.get("host", "?"),
         "host": d.get("host", ""),
         "oids": d.get("oids", []),
-        "readings": latest.get(d.get("name") or d.get("host", "?"), {}),
+        **_device_state(latest.get(d.get("name") or d.get("host", "?")), _poll_seconds(), now),
         "bound": bool(str(d.get("host_id", "")).strip()
                       or str(d.get("steward_host", "")).strip()),
     } for d in matched]
@@ -144,6 +168,7 @@ async def widget():
 
     async with current_app.db_sessionmaker() as db:
         latest = await _latest_readings(db, device_names[:10])
+    now = datetime.now(timezone.utc)
 
     devices = []
     for cfg in devices_cfg[:10]:
@@ -151,7 +176,7 @@ async def widget():
         devices.append({
             "name": name,
             "oids": cfg.get("oids", []),
-            "readings": latest.get(name, {}),
+            **_device_state(latest.get(name), _poll_seconds(), now),
         })
 
     return await render_template("snmp/widget.html",

plugins/snmp/templates/snmp/host_panel.html

@@ -15,8 +15,10 @@
       <span title="Explicitly bound to this host (host_id / steward_host)"
             style="font-size:0.68rem;color:var(--text-muted);background:var(--bg-elevated);border:1px solid var(--border);border-radius:3px;padding:0.05rem 0.4rem;">bound</span>
       {% endif %}
-      {% if device.readings %}
-      <span style="font-size:0.74rem;color:var(--green);">&#9679; reachable</span>
+      {% if device.reach == 'up' %}
+      <span style="font-size:0.74rem;" class="tone-ok">&#9679; reachable · read {{ device.at|ago }}</span>
+      {% elif device.reach == 'stale' %}
+      <span style="font-size:0.74rem;" class="tone-warn" title="The values below are from then">&#9679; no answer since {{ device.at|ago }}</span>
       {% else %}
       <span style="font-size:0.74rem;color:var(--text-dim);">&#9675; no data yet</span>
       {% endif %}

plugins/snmp/templates/snmp/index.html

@@ -12,9 +12,14 @@
 {% for device in devices %}
 <section class="card" aria-labelledby="snmp-{{ loop.index }}">
   <div class="host-health-head">
-    {{ status_badge('up', 'Reachable') if device.readings else status_badge('pending', 'No data yet') }}
+    {#- reach: answered within the last few polls (up), only long ago (stale),
+        or never (pending); routes._device_state. #}
+    {% if device.reach == 'up' %}{{ status_badge('up', 'Reachable') }}
+    {% elif device.reach == 'stale' %}{{ status_badge('stale', 'No answer', title="The last reading is from " ~ device.at.strftime('%Y-%m-%d %H:%M UTC') ~ "; the values below are from then") }}
+    {% else %}{{ status_badge('pending', 'No data yet') }}{% endif %}
     <h2 id="snmp-{{ loop.index }}" class="section-title" style="margin:0;">{{ device.name }}</h2>
     <span class="field-hint mono">{{ device.host }}</span>
+    {% if device.at %}<span class="field-hint">read {{ device.at|ago }}</span>{% endif %}
     <a href="/plugins/snmp/device/{{ device.name }}" class="btn btn-sm btn-ghost" style="margin-left:auto;">History</a>
   </div>
 

plugins/snmp/templates/snmp/widget.html

@@ -6,8 +6,10 @@
     <span style="font-size:0.82rem;font-weight:500;flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">
       {{ device.name }}
     </span>
-    {% if device.readings %}
-    <span style="font-size:0.7rem;color:var(--green);">&#9679;</span>
+    {% if device.reach == 'up' %}
+    <span style="font-size:0.7rem;" class="tone-ok" title="Reachable · read {{ device.at.strftime('%H:%M UTC') }}">&#9679;</span>
+    {% elif device.reach == 'stale' %}
+    <span style="font-size:0.7rem;" class="tone-warn">no answer since {{ device.at|ago }}</span>
     {% else %}
     <span style="font-size:0.7rem;color:var(--text-dim);">&#9675;</span>
     {% endif %}

plugins/traefik/routes.py

@@ -1,7 +1,7 @@
 # plugins/traefik/routes.py
 from __future__ import annotations
 from collections import defaultdict
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from quart import Blueprint, current_app, render_template, request
 from sqlalchemy import Integer, cast, func, select
 import json
@@ -58,6 +58,24 @@ def router_state(latest, thresholds: dict) -> tuple[str, str | None]:
     return "ok", None
 
 
+# A router missing from this many scrapes in a row is no longer reported:
+# Traefik dropped it (a removed service), so its last state is history.
+GONE_AFTER_SCRAPES = 3
+
+
+def gone_routers(routers: list[dict], scrape_seconds: int) -> tuple[list[dict], list[dict]]:
+    """(current, gone): a router whose newest row trails the newest scrape of
+    any router by GONE_AFTER_SCRAPES intervals is gone, newest-seen first."""
+    if not routers:
+        return [], []
+    newest = max(r["latest"].scraped_at for r in routers)
+    cutoff = newest - timedelta(seconds=GONE_AFTER_SCRAPES * max(int(scrape_seconds), 1))
+    current = [r for r in routers if r["latest"].scraped_at >= cutoff]
+    gone = sorted((r for r in routers if r["latest"].scraped_at < cutoff),
+                  key=lambda r: r["latest"].scraped_at, reverse=True)
+    return current, gone
+
+
 def split_routers(routers: list[dict]) -> tuple[list[dict], list[dict]]:
     """(problems, healthy): failing routers before slow or noisy ones, each
     group by name; healthy routers by name."""
@@ -171,11 +189,14 @@ async def rows():
             "days_remaining": days,
         })
 
-    problems, healthy = split_routers(router_data)
+    current, gone = gone_routers(
+        router_data, current_app.config["PLUGINS"]["traefik"]["scrape_interval_seconds"])
+    problems, healthy = split_routers(current)
     return await render_template(
         "traefik/rows.html",
         problems=problems,
         healthy=healthy,
+        gone=gone,
         error_4xx=ERROR_4XX_PCT,
         error_5xx=ERROR_5XX_PCT,
         global_stat=global_stat,

plugins/traefik/templates/traefik/rows.html

@@ -37,7 +37,7 @@
 </div>
 {% endmacro %}
 
-{% if problems or healthy or certs %}
+{% if problems or healthy or gone or certs %}
 {# ── Routers that need attention ─────────────────────────────────────────── #}
 {% if problems %}
 <div style="column-width:340px;column-gap:1rem;">
@@ -67,6 +67,22 @@
   </table>
   </div>
 </details>
+{% endif %}
+
+{# ── Routers Traefik no longer reports: their last numbers are history, not a
+   state, so they are listed by when they were last seen and nothing else. ── #}
+{% if gone %}
+<details class="card" style="margin-bottom:1rem;">
+  <summary class="section-title" style="cursor:pointer;">{{ gone|length }} router{{ 's' if gone|length != 1 }} no longer reported</summary>
+  <p class="field-hint" style="margin:0.5rem 0;">Traefik stopped reporting these, usually because the service behind them was removed.</p>
+  <ul class="problem-list">
+    {% for r in gone %}
+    <li><span style="word-break:break-all;">{{ r.name }}</span> <span class="problem-detail">last seen {{ r.latest.scraped_at|ago }}</span></li>
+    {% endfor %}
+  </ul>
+</details>
+{% endif %}
+{% if healthy %}
 <script>
   // The fragment is re-polled; keep the healthy list as the reader left it.
   (function () {

steward/alerts/service.py

@@ -215,6 +215,18 @@ async def active_rows(db, now: datetime) -> list[dict]:
     return rows
 
 
+def watches_host(rule: AlertRule, host_name: str, monitor_keys: set) -> bool:
+    """Whether a rule watches something on this host: its agent series
+    ("<host>" or "<host>:<part>"), one of its containers ("<host>/<name>"),
+    or one of its monitors ((type, name) in monitor_keys)."""
+    source, resource = rule.source_module, rule.resource_name
+    if source == "host_agent":
+        return resource == host_name or resource.startswith(host_name + ":")
+    if source == "docker":
+        return resource.startswith(host_name + "/")
+    return (source, resource) in monitor_keys
+
+
 async def active_count(db) -> int:
     """How many alerts need attention: the sidebar's number."""
     from sqlalchemy import func

steward/hosts/routes.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
 import asyncio
 import uuid
+from datetime import datetime, timezone
 from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
 from sqlalchemy import select
 from steward.ansible import executor, sources as ansible_src
+from steward.alerts import service as alert_svc
 from steward.auth.middleware import require_role
 from steward.core import renames
 from steward.core.audit import audit_request, log_audit
@@ -78,6 +80,10 @@ async def host_detail(host_id: str):
             return "Not found", 404
         monitors = await host_svc.host_monitors(db, host_id)
         uptime = (await host_svc.compute_uptime(db)).get(host_id)
+        monitor_keys = {(getattr(d["monitor"].type, "value", d["monitor"].type), d["monitor"].name)
+                        for d in monitors}
+        alerts = [r for r in await alert_svc.active_rows(db, datetime.now(timezone.utc))
+                  if alert_svc.watches_host(r["rule"], host.name, monitor_keys)]
         linked_target = (await db.execute(
             select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
             .options(selectinload(AnsibleTarget.groups)))).scalar_one_or_none()
@@ -88,7 +94,7 @@ async def host_detail(host_id: str):
     from steward.models.monitors import MonitorType
     return await render_template(
         "hosts/detail.html",
-        host=host, monitors=monitors, uptime=uptime,
+        host=host, monitors=monitors, uptime=uptime, alerts=alerts,
         monitor_types=list(MonitorType),
         linked_target=linked_target, linkable_targets=linkable_targets,
         ansible_sources=_ansible_source_names(),
@@ -305,8 +311,8 @@ async def delete_host(host_id: str):
 async def uptime_page():
     async with current_app.db_sessionmaker() as db:
         result = await db.execute(select(Host).order_by(Host.name))
-        hosts = result.scalars().all()
         uptime = await host_svc.compute_uptime(db)
+        hosts = host_svc.worst_uptime_first(result.scalars().all(), uptime)
     return await render_template("hosts/uptime.html", hosts=hosts, uptime=uptime)
 
 
@@ -315,8 +321,8 @@ async def uptime_page():
 async def uptime_widget():
     async with current_app.db_sessionmaker() as db:
         result = await db.execute(select(Host).order_by(Host.name))
-        hosts = result.scalars().all()
         uptime = await host_svc.compute_uptime(db)
+        hosts = host_svc.worst_uptime_first(result.scalars().all(), uptime)
     return await render_template(
         "hosts/uptime_widget.html",
         hosts=hosts,

steward/hosts/service.py

@@ -113,6 +113,17 @@ async def host_status(db, host_ids: list[str]) -> dict[str, dict]:
     return out
 
 
+def worst_uptime_first(hosts, uptime: dict) -> list:
+    """Hosts by their 24h uptime, worst first, then 7d and 30d to break ties;
+    hosts with no uptime yet (no monitors) last, by name."""
+    def key(h):
+        u = uptime.get(h.id) or {}
+        windows = [u.get(w) for w in ("24h", "7d", "30d")]
+        known = [w for w in windows if w is not None]
+        return (not known, *[101.0 if w is None else w for w in windows], h.name.lower())
+    return sorted(hosts, key=key)
+
+
 def host_rank(status: dict | None, agent: dict | None) -> int:
     """Where a host sorts in the Hosts list: a down monitor first, then an
     agent gone quiet, then monitors not yet checked, then the healthy; a host
@@ -137,16 +148,23 @@ async def host_monitors(db, host_id: str) -> list[dict]:
     recent = await _last_n_results(db, ids)
     uptime = await _uptime_by_monitor(db, ids)
     data = []
+    from steward.monitors.service import down_reason, down_since
     for m in monitors:
         mrows = recent.get(m.id, [])
         latest = mrows[-1] if mrows else None
+        state = ("paused" if not m.enabled else "pending" if latest is None
+                 else "up" if latest.is_up else "down")
         data.append({
-            "monitor": m, "latest": latest,
+            "monitor": m, "latest": latest, "state": state,
             "heartbeat": [{"state": "up" if r.is_up else "down",
                            "title": _heartbeat_title(m.type, r)} for r in mrows],
             "uptime_pct": uptime.get(m.id, {}).get("24h"),
+            "down_since": await down_since(db, m.id) if state == "down" else None,
+            "down_reason": down_reason(latest) if state == "down" else None,
         })
-    return data
+    # Problems first: down, then not yet checked, then up, paused last.
+    order = {"down": 0, "pending": 1, "up": 2, "paused": 3}
+    return sorted(data, key=lambda d: (order[d["state"]], d["monitor"].name.lower()))
 
 
 async def agent_overview_by_host(

steward/templates/hosts/detail.html

@@ -24,6 +24,27 @@
 </div>
 
 
+{# ── Alerts on this host: what is breaching, before anything else ─────────── #}
+{% if alerts %}
+<section class="card problem-card" aria-labelledby="host-alerts" style="margin-bottom:1rem;">
+  <h2 id="host-alerts" class="section-title" style="margin-bottom:0.6rem;">Alerts on this host</h2>
+  <ul class="problem-list">
+    {% for row in alerts %}
+    <li>
+      {{ status_badge(row.label) }}
+      <span class="problem-detail">
+        {{- row.says.lead -}}
+        {%- if row.url %}<a href="{{ row.url }}">{{ row.says.resource }}</a>{% else %}<strong>{{ row.says.resource }}</strong>{% endif -%}
+        {{- row.says.tail -}}
+        {%- if row.label == 'acknowledged' and row.acked_by %} · acknowledged by {{ row.acked_by }}{% endif -%}
+      </span>
+      <a href="/alerts/history?rule={{ row.rule.id }}" class="field-hint" style="margin-left:0.4rem;">{{ row.rule.name }}</a>
+    </li>
+    {% endfor %}
+  </ul>
+</section>
+{% endif %}
+
 {# ── Live vitals strip (host_agent fragment) — full width on top, polled ────── #}
 <div id="hv-strip"
      hx-get="/plugins/host_agent/vitals/{{ host.id }}"
@@ -52,10 +73,15 @@
   <div class="ping-row{% if not m.enabled %} ping-row-paused{% endif %}">
     <div class="ping-meta" style="min-width:150px;max-width:230px;">
       <div style="display:flex;align-items:baseline;gap:0.4rem;">
-        {{ status_dot('paused' if not m.enabled else ('pending' if latest is none else ('up' if latest.is_up else 'down'))) }}
+        {{ status_dot(d.state) }}
         <a class="ping-name" href="/monitors/{{ m.id }}">{{ m.name }}</a>
       </div>
       <span class="ping-addr" title="{{ m.target }}">{{ m.type|monitor_type }} · {{ m.target }}</span>
+      {% if d.state == 'down' %}
+      <span class="tone-crit" style="font-size:0.75rem;overflow-wrap:anywhere;">
+        Down{% if d.down_since %} {{ d.down_since|ago|replace(' ago', '') }}{% endif %}{% if d.down_reason %} · {{ d.down_reason }}{% endif %}
+      </span>
+      {% endif %}
     </div>
     <div class="ping-pills">
       {% for _ in range([30 - d.heartbeat|length, 0]|max) %}<span class="pill pill-empty" title="No data"></span>{% endfor %}
@@ -124,6 +150,12 @@
 </div>
 </div>{# Monitors + Agent grid #}
 
+{# ── Docker (docker plugin fragment; renders nothing if the host has none);
+   above Ansible: containers are state to read, Ansible is something to do ──── #}
+{% if "docker" in enabled_plugins %}
+<div hx-get="/plugins/docker/host/{{ host.id }}" hx-trigger="load" hx-swap="innerHTML"></div>
+{% endif %}
+
 {# ── Ansible (full width) ─────────────────────────────────────────────────── #}
 <div class="card">
   <h3 class="section-title">Ansible</h3>
@@ -211,11 +243,6 @@
   {% endif %}
 </div>{# Ansible card #}
 
-{# ── Docker (docker plugin fragment; renders nothing if the host has none) ──── #}
-{% if "docker" in enabled_plugins %}
-<div hx-get="/plugins/docker/host/{{ host.id }}" hx-trigger="load" hx-swap="innerHTML"></div>
-{% endif %}
-
 {# ── SNMP (renders nothing unless a configured SNMP device maps to this host) ── #}
 {% if "snmp" in enabled_plugins %}
 <div hx-get="/plugins/snmp/host/{{ host.id }}" hx-trigger="load" hx-swap="innerHTML"></div>

tests/core/test_problems_first.py

@@ -29,3 +29,27 @@ def test_schedules_sort_failing_first_and_paused_last():
             _sched("busy", last_error="the last run is still going", status="success")]
     assert [r["s"].name for r in sorted(rows, key=_schedule_order)] == [
         "backup", "broken", "cut", "deploy", "busy", "nightly", "paused"]
+
+
+def test_a_host_page_shows_only_the_alerts_on_that_host():
+    from steward.alerts.service import watches_host
+
+    def rule(source, resource):
+        return NS(source_module=source, resource_name=resource)
+    mine = {("http", "nas web")}
+    assert watches_host(rule("host_agent", "nas"), "nas", mine)
+    assert watches_host(rule("host_agent", "nas:/"), "nas", mine)
+    assert not watches_host(rule("host_agent", "nas2"), "nas", mine)       # a name that merely starts alike
+    assert watches_host(rule("docker", "nas/plex"), "nas", mine)
+    assert not watches_host(rule("docker", "nas2/plex"), "nas", mine)
+    assert watches_host(rule("http", "nas web"), "nas", mine)
+    assert not watches_host(rule("http", "other web"), "nas", mine)
+
+
+def test_uptime_lists_the_worst_first_and_the_unmeasured_last():
+    from steward.hosts.service import worst_uptime_first
+    hosts = [NS(id=i, name=n) for i, n in [("a", "alpha"), ("b", "bravo"), ("c", "charlie"), ("d", "delta")]]
+    uptime = {"a": {"24h": 100.0, "7d": 99.0, "30d": 99.5},
+              "b": {"24h": 92.0, "7d": 98.0, "30d": 99.0},
+              "c": {"24h": 100.0, "7d": 95.0, "30d": 97.0}}
+    assert [h.name for h in worst_uptime_first(hosts, uptime)] == ["bravo", "charlie", "alpha", "delta"]

tests/plugins/snmp/test_snmp_page.py

@@ -1,5 +1,8 @@
 """The SNMP list says each device's state with the shared badge and shows its
 readings as stat tiles (#5126)."""
+from datetime import datetime, timedelta, timezone
+
+from plugins.snmp.routes import _device_state
 from tests._templates import render_plugin_template
 from tests.test_style_tokens import hex_colours
 
@@ -8,15 +11,29 @@ async def test_devices_wear_a_status_badge_and_readings_are_tiles(app):
     devices = [
         {"name": "ups", "host": "10.0.0.9", "oids": [{"oid": "1.3.6", "label": "battery_charge_pct"},
                                                        {"oid": "1.3.7", "label": "in_octets"}],
-         "readings": {"battery_charge_pct": 97.0, "in_octets": 2_500_000.0}},
-        {"name": "printer", "host": "10.0.0.10", "oids": [], "readings": {}},
+         "readings": {"battery_charge_pct": 97.0, "in_octets": 2_500_000.0}, "reach": "up",
+         "at": datetime.now(timezone.utc) - timedelta(seconds=30)},
+        {"name": "switch", "host": "10.0.0.11", "oids": [], "readings": {"x": 1.0}, "reach": "stale",
+         "at": datetime.now(timezone.utc) - timedelta(hours=3)},
+        {"name": "printer", "host": "10.0.0.10", "oids": [], "readings": {}, "reach": "pending", "at": None},
     ]
     async with app.test_request_context("/plugins/snmp/"):
         page = await render_plugin_template(app, "snmp", "snmp/index.html", devices=devices, poll_interval=60)
 
     assert '<span class="badge badge-ok">Reachable</span>' in page
     assert '<span class="badge badge-warn">No data yet</span>' in page
+    # Old readings are not "reachable": they say how old they are (#5210).
+    assert 'badge badge-warn" title="The last reading is from' in page and ">No answer</span>" in page
+    assert ">3h ago</time>" in page
     assert '<span class="stat-tile-value">97</span>' in page and '<span class="stat-tile-value">2.50M</span>' in page
     assert 'href="/plugins/snmp/device/ups"' in page
     main = page[page.index("<main"):]
     assert "font-weight:600" not in main and not hex_colours(main), hex_colours(main)
+
+
+def test_a_device_is_reachable_only_while_its_readings_are_recent():
+    now = datetime.now(timezone.utc)
+    entry = {"readings": {"x": 1.0}, "at": now - timedelta(seconds=150)}
+    assert _device_state(entry, 60, now)["reach"] == "up"           # within 3 polls
+    assert _device_state(entry, 30, now)["reach"] == "stale"        # 5 polls without an answer
+    assert _device_state(None, 60, now) == {"readings": {}, "at": None, "reach": "pending"}

tests/plugins/traefik/test_traefik_page.py

@@ -5,7 +5,7 @@ from types import SimpleNamespace as NS
 
 import pytest
 
-from plugins.traefik.routes import router_state, split_routers
+from plugins.traefik.routes import gone_routers, router_state, split_routers
 from steward.core.settings import to_thresholds_cfg
 from tests._templates import render_plugin_template
 from tests.test_style_tokens import hex_colours
@@ -63,3 +63,28 @@ async def test_the_page_leads_with_problems_and_folds_the_healthy(app):
     assert page.index(">docs</td>") < page.index(">web</td>")
     assert 'class="tone-crit">2 days</td>' in page and 'class="tone-ok">60 days</td>' in page
     assert "font-weight:600" not in page and not hex_colours(page), hex_colours(page)
+
+
+def test_a_router_traefik_stopped_reporting_is_set_apart_not_failing():
+    live = _router("api", _latest())
+    removed = _router("old", _latest(e5=9.0))
+    removed["latest"].scraped_at = live["latest"].scraped_at - timedelta(minutes=10)
+    recent = _router("blip", _latest())
+    recent["latest"].scraped_at = live["latest"].scraped_at - timedelta(seconds=60)
+    current, gone = gone_routers([live, removed, recent], 30)     # 3 scrapes = 90 s
+    assert [r["name"] for r in current] == ["api", "blip"]
+    assert [r["name"] for r in gone] == ["old"]
+    assert gone_routers([], 30) == ([], [])
+
+
+async def test_gone_routers_are_listed_by_when_they_were_last_seen(app):
+    removed = _router("old", _latest(e5=9.0))
+    removed["latest"].scraped_at = datetime.now(timezone.utc) - timedelta(hours=2)
+    async with app.test_request_context("/plugins/traefik/rows"):
+        page = await render_plugin_template(app, "traefik", "traefik/rows.html", problems=[], healthy=[],
+                                            gone=[removed], global_stat=None, certs=[], range_key="24h",
+                                            error_4xx=1.0, error_5xx=0.1)
+    page = " ".join(page.split())
+    assert "1 router no longer reported</summary>" in page
+    assert "old</span> <span class=\"problem-detail\">last seen <time" in page and ">2h ago</time>" in page
+    assert "Failing" not in page