feat(ui): host page leads with its alerts; old readings say how old (#5210)

#5210 "Problems first, with since-time, across every list" (milestone 477),
second batch.

- Host page: an "Alerts on this host" strip first (rules on its agent
  series, its containers or its monitors: alert_svc.watches_host); its
  monitors problems first with "Down 12m · reason"; Docker above Ansible.
- Uptime page and widget: worst 24h uptime first, then 7d and 30d; hosts
  with no uptime yet last.
- Agent panel and fleet widget: last seen as "3m ago", not a clock time.
- SNMP: a device is "Reachable" only while it answered within the last 3
  polls; older readings say "No answer" with their age, and every device
  says when it was last read (was: reachable whenever any reading existed).
- Traefik: a router missing from 3 scrapes in a row is listed apart as no
  longer reported, by when it was last seen, instead of keeping its last
  state (a removed router stayed Failing).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_problems_first.py

@@ -29,3 +29,27 @@ def test_schedules_sort_failing_first_and_paused_last():
             _sched("busy", last_error="the last run is still going", status="success")]
     assert [r["s"].name for r in sorted(rows, key=_schedule_order)] == [
         "backup", "broken", "cut", "deploy", "busy", "nightly", "paused"]
+
+
+def test_a_host_page_shows_only_the_alerts_on_that_host():
+    from steward.alerts.service import watches_host
+
+    def rule(source, resource):
+        return NS(source_module=source, resource_name=resource)
+    mine = {("http", "nas web")}
+    assert watches_host(rule("host_agent", "nas"), "nas", mine)
+    assert watches_host(rule("host_agent", "nas:/"), "nas", mine)
+    assert not watches_host(rule("host_agent", "nas2"), "nas", mine)       # a name that merely starts alike
+    assert watches_host(rule("docker", "nas/plex"), "nas", mine)
+    assert not watches_host(rule("docker", "nas2/plex"), "nas", mine)
+    assert watches_host(rule("http", "nas web"), "nas", mine)
+    assert not watches_host(rule("http", "other web"), "nas", mine)
+
+
+def test_uptime_lists_the_worst_first_and_the_unmeasured_last():
+    from steward.hosts.service import worst_uptime_first
+    hosts = [NS(id=i, name=n) for i, n in [("a", "alpha"), ("b", "bravo"), ("c", "charlie"), ("d", "delta")]]
+    uptime = {"a": {"24h": 100.0, "7d": 99.0, "30d": 99.5},
+              "b": {"24h": 92.0, "7d": 98.0, "30d": 99.0},
+              "c": {"24h": 100.0, "7d": 95.0, "30d": 97.0}}
+    assert [h.name for h in worst_uptime_first(hosts, uptime)] == ["bravo", "charlie", "alpha", "delta"]

tests/plugins/snmp/test_snmp_page.py

@@ -1,5 +1,8 @@
 """The SNMP list says each device's state with the shared badge and shows its
 readings as stat tiles (#5126)."""
+from datetime import datetime, timedelta, timezone
+
+from plugins.snmp.routes import _device_state
 from tests._templates import render_plugin_template
 from tests.test_style_tokens import hex_colours
 
@@ -8,15 +11,29 @@ async def test_devices_wear_a_status_badge_and_readings_are_tiles(app):
     devices = [
         {"name": "ups", "host": "10.0.0.9", "oids": [{"oid": "1.3.6", "label": "battery_charge_pct"},
                                                        {"oid": "1.3.7", "label": "in_octets"}],
-         "readings": {"battery_charge_pct": 97.0, "in_octets": 2_500_000.0}},
-        {"name": "printer", "host": "10.0.0.10", "oids": [], "readings": {}},
+         "readings": {"battery_charge_pct": 97.0, "in_octets": 2_500_000.0}, "reach": "up",
+         "at": datetime.now(timezone.utc) - timedelta(seconds=30)},
+        {"name": "switch", "host": "10.0.0.11", "oids": [], "readings": {"x": 1.0}, "reach": "stale",
+         "at": datetime.now(timezone.utc) - timedelta(hours=3)},
+        {"name": "printer", "host": "10.0.0.10", "oids": [], "readings": {}, "reach": "pending", "at": None},
     ]
     async with app.test_request_context("/plugins/snmp/"):
         page = await render_plugin_template(app, "snmp", "snmp/index.html", devices=devices, poll_interval=60)
 
     assert '<span class="badge badge-ok">Reachable</span>' in page
     assert '<span class="badge badge-warn">No data yet</span>' in page
+    # Old readings are not "reachable": they say how old they are (#5210).
+    assert 'badge badge-warn" title="The last reading is from' in page and ">No answer</span>" in page
+    assert ">3h ago</time>" in page
     assert '<span class="stat-tile-value">97</span>' in page and '<span class="stat-tile-value">2.50M</span>' in page
     assert 'href="/plugins/snmp/device/ups"' in page
     main = page[page.index("<main"):]
     assert "font-weight:600" not in main and not hex_colours(main), hex_colours(main)
+
+
+def test_a_device_is_reachable_only_while_its_readings_are_recent():
+    now = datetime.now(timezone.utc)
+    entry = {"readings": {"x": 1.0}, "at": now - timedelta(seconds=150)}
+    assert _device_state(entry, 60, now)["reach"] == "up"           # within 3 polls
+    assert _device_state(entry, 30, now)["reach"] == "stale"        # 5 polls without an answer
+    assert _device_state(None, 60, now) == {"readings": {}, "at": None, "reach": "pending"}

tests/plugins/traefik/test_traefik_page.py

@@ -5,7 +5,7 @@ from types import SimpleNamespace as NS
 
 import pytest
 
-from plugins.traefik.routes import router_state, split_routers
+from plugins.traefik.routes import gone_routers, router_state, split_routers
 from steward.core.settings import to_thresholds_cfg
 from tests._templates import render_plugin_template
 from tests.test_style_tokens import hex_colours
@@ -63,3 +63,28 @@ async def test_the_page_leads_with_problems_and_folds_the_healthy(app):
     assert page.index(">docs</td>") < page.index(">web</td>")
     assert 'class="tone-crit">2 days</td>' in page and 'class="tone-ok">60 days</td>' in page
     assert "font-weight:600" not in page and not hex_colours(page), hex_colours(page)
+
+
+def test_a_router_traefik_stopped_reporting_is_set_apart_not_failing():
+    live = _router("api", _latest())
+    removed = _router("old", _latest(e5=9.0))
+    removed["latest"].scraped_at = live["latest"].scraped_at - timedelta(minutes=10)
+    recent = _router("blip", _latest())
+    recent["latest"].scraped_at = live["latest"].scraped_at - timedelta(seconds=60)
+    current, gone = gone_routers([live, removed, recent], 30)     # 3 scrapes = 90 s
+    assert [r["name"] for r in current] == ["api", "blip"]
+    assert [r["name"] for r in gone] == ["old"]
+    assert gone_routers([], 30) == ([], [])
+
+
+async def test_gone_routers_are_listed_by_when_they_were_last_seen(app):
+    removed = _router("old", _latest(e5=9.0))
+    removed["latest"].scraped_at = datetime.now(timezone.utc) - timedelta(hours=2)
+    async with app.test_request_context("/plugins/traefik/rows"):
+        page = await render_plugin_template(app, "traefik", "traefik/rows.html", problems=[], healthy=[],
+                                            gone=[removed], global_stat=None, certs=[], range_key="24h",
+                                            error_4xx=1.0, error_5xx=0.1)
+    page = " ".join(page.split())
+    assert "1 router no longer reported</summary>" in page
+    assert "old</span> <span class=\"problem-detail\">last seen <time" in page and ">2h ago</time>" in page
+    assert "Failing" not in page