feat(ui): alerts, monitors, hosts, schedules and maintenance lists put problems first, with since-times (#5210)

#5210 "Problems first, with since-time, across every list" (milestone 477),
first batch.

- Alerts: the Active list reads firing, no data, acknowledged, pending, the
  longest-running first, and re-polls itself. An acknowledged alert says how
  long it has been breaching (from the alert's events; acknowledging moves
  last_transition_at) and, apart from that, who acknowledged it and when.
  Rules show what each is doing now; disabled rules sort last. The current
  value reads the latest-value table (#5208) instead of plugin_metrics.
- Monitors: a down monitor says "Down 12m · connection refused" in the list.
- Hosts: sorted down, agent gone quiet, pending, healthy. The agent column
  shows "—" for a missing reading (was 0%) and a Stale badge with its age.
  agent_metrics_by_host is gone; the list reads agent_overview_by_host.
- Schedules: failing first (failed or interrupted run, or a firing that could
  not start one), then running, then the rest, paused last; status badge
  from the shared states.
- Maintenance windows: active (ending soonest), then upcoming (starting
  soonest), with expired folded away; each says when it ends/starts/ended.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_alerts_page.py

@@ -51,7 +51,8 @@ def test_an_up_or_down_metric_says_the_word():
     now = datetime.now(timezone.utc)
     says = alert_sentence(_rule("is_up", AlertOperator.lt, 1.0, "nas"), "acknowledged", 0.0,
                           now - timedelta(hours=2, minutes=5), now)
-    assert _said(says) == "nas is down, acknowledged for 2h 5m"
+    # Acknowledged is still breaching: the sentence says for how long (#5210).
+    assert _said(says) == "nas is down, breaching for 2h 5m"
 
 
 def test_no_reading_yet_is_said_not_shown_as_a_number():
@@ -132,3 +133,35 @@ async def test_no_count_for_a_fragment_or_a_logged_out_page(app, monkeypatch):
     async with app.test_request_context("/alerts/"):
         logged_out = await render_template("alerts/list.html", active=[], rules=[], operators=[])
     assert "count-badge" not in fragment and "count-badge" not in logged_out
+
+
+async def test_an_acknowledged_alert_says_who_and_when_apart_from_the_breach(app):
+    rule = _rule("is_up", AlertOperator.lt, 1.0, "nas")
+    now = datetime.now(timezone.utc)
+    st = SimpleNamespace(state=AlertStateEnum.acknowledged, no_data_since=None, last_sample_at=None,
+                         last_transition_at=now - timedelta(minutes=3),
+                         acknowledged_at=now - timedelta(minutes=3))
+    row = {"state": st, "rule": rule, "label": "acknowledged", "url": None, "acked_by": "bryan",
+           "says": alert_sentence(rule, "acknowledged", 0.0, now - timedelta(hours=1), now)}
+    async with app.test_request_context("/alerts/active"):
+        session["user_role"] = "operator"
+        page = await render_template("alerts/_active.html", active=[row])
+    page = " ".join(page.split())
+    assert "<strong>nas</strong> is down, breaching for 1h" in page
+    assert "Acknowledged by bryan <time" in page and ">3m ago</time>" in page
+    assert "/acknowledge" not in page                      # already taken
+
+
+async def test_rules_say_what_each_is_doing_now(app):
+    firing, quiet, off = _rule(rid="r1"), _rule(rid="r2", resource="db"), _rule(rid="r3", resource="old")
+    off.enabled = False
+    async with app.test_request_context("/alerts/"):
+        session["user_role"] = "viewer"
+        page = await render_template("alerts/list.html", active=[], rules=[firing, quiet, off],
+                                     rule_state={"r1": "firing", "r2": "inactive"}, operators=[])
+    page = " ".join(page.split())
+    rules = page[page.index(">Rules</h2>"):]
+    assert '<span class="badge badge-crit">Firing</span>' in rules
+    assert 'title="Within its threshold">quiet</span>' in rules
+    assert '<span class="tone-dim" style="font-size:0.8rem;">off</span>' in rules
+    assert 'hx-get="/alerts/active"' in page

tests/core/test_monitors_page.py

@@ -1,5 +1,6 @@
 """The Monitors page leads with what needs looking at (#5121): a count strip,
 problems first, each monitor on a page of its own, adding behind a button."""
+import re
 from datetime import datetime, timedelta, timezone
 from types import SimpleNamespace
 
@@ -115,3 +116,24 @@ async def test_adding_is_its_own_page_and_keeps_what_was_typed(app):
     assert '<a href="/monitors/">Monitors</a>' in page and 'action="/monitors/add"' in page
     assert '<p class="field-error" role="alert">port must be between 1 and 65535, got 0.</p>' in page
     assert 'value="db"' in page and 'value="10.0.0.9"' in page and 'value="30"' in page
+
+
+def test_down_reason_says_why_in_a_few_words():
+    from steward.monitors.service import down_reason
+    assert down_reason(_check(False)) == "503 Service Unavailable"       # the probe's own error first
+    assert down_reason(SimpleNamespace(error_msg=None, status_code=502, content_matched=None)) == "HTTP 502"
+    assert down_reason(SimpleNamespace(error_msg=None, status_code=None, content_matched=False)) == \
+        "expected content not found"
+    assert down_reason(None) is None
+
+
+async def test_a_down_monitor_says_since_when_and_why_in_the_list(app):
+    row = _row(_mon("api", mid="m2"), "down", _check(False), 90.0)
+    row.update(down_since=datetime.now(timezone.utc) - timedelta(minutes=12),
+               down_reason="connection refused")
+    async with app.test_request_context("/monitors/rows"):
+        page = await render_template("monitors/rows.html", monitor_data=[row],
+                                     counts={"up": 0, "degraded": 0, "down": 1, "pending": 0, "paused": 0},
+                                     range_key="24h")
+    page = " ".join(page.split())
+    assert re.search(r'Down <time [^>]*>12m</time> · connection refused', page), page

tests/core/test_problems_first.py

@@ -0,0 +1,31 @@
+"""Lists put what needs looking at first (#5210): hosts by their worst
+signal, schedules failing first, maintenance windows active then upcoming."""
+from types import SimpleNamespace as NS
+
+from steward.ansible.routes import _schedule_order
+from steward.hosts.service import host_rank
+
+
+def test_hosts_sort_down_then_quiet_agent_then_pending_then_healthy():
+    down, up, pending = {"state": "down"}, {"state": "up"}, {"state": "pending"}
+    fresh, quiet = {"fresh": True}, {"fresh": False}
+    assert host_rank(down, fresh) == 0
+    assert host_rank(up, quiet) == 1            # monitors fine, but the agent went silent
+    assert host_rank(None, quiet) == 1
+    assert host_rank(pending, fresh) == 2
+    assert host_rank(up, fresh) == 3 and host_rank(None, fresh) == 3
+    assert host_rank(None, None) == 4           # nothing watches it
+
+
+def _sched(name, *, enabled=True, last_error=None, status=None):
+    run = NS(status=NS(value=status)) if status else None
+    return {"s": NS(name=name, enabled=enabled, last_error=last_error), "last_run": run}
+
+
+def test_schedules_sort_failing_first_and_paused_last():
+    rows = [_sched("nightly", status="success"), _sched("paused", enabled=False, status="failed"),
+            _sched("backup", status="failed"), _sched("deploy", status="running"),
+            _sched("broken", last_error="source not found"), _sched("cut", status="interrupted"),
+            _sched("busy", last_error="the last run is still going", status="success")]
+    assert [r["s"].name for r in sorted(rows, key=_schedule_order)] == [
+        "backup", "broken", "cut", "deploy", "busy", "nightly", "paused"]

tests/integration/test_metric_latest.py

@@ -72,7 +72,7 @@ def test_the_newest_sample_wins_and_a_late_backlog_does_not_move_it_back(app):
 
 def test_a_quiet_host_keeps_its_last_values_and_reads_stale_not_missing(app):
     from steward.core.alerts import record_metrics
-    from steward.hosts.service import agent_metrics_by_host, agent_overview_by_host
+    from steward.hosts.service import agent_overview_by_host
 
     now = datetime.now(timezone.utc)
     quiet, live = ("quiet-" + uuid.uuid4().hex[:8]), ("live-" + uuid.uuid4().hex[:8])
@@ -87,18 +87,15 @@ def test_a_quiet_host_keeps_its_last_values_and_reads_stale_not_missing(app):
                     _sample(quiet + ":/", "disk_used_pct", 56.0, two_hours_ago),
                     _sample(live, "cpu_pct", 7.0, now - timedelta(seconds=20)),
                 ])
-            overview = await agent_overview_by_host(s, [quiet, live, "never-" + quiet], 180)
-            metrics = await agent_metrics_by_host(s, [quiet, live])
-            return overview, metrics
+            return await agent_overview_by_host(s, [quiet, live, "never-" + quiet], 180)
 
-    overview, metrics = asyncio.run(go())
+    overview = asyncio.run(go())
     q = overview[quiet]
     assert (q["cpu_pct"], q["mem_used_pct"], q["disk_root"]) == (12.0, 34.0, 56.0)
     assert q["fresh"] is False
     assert abs((q["last_at"] - two_hours_ago).total_seconds()) < 1
     assert overview[live]["fresh"] is True
     assert "never-" + quiet not in overview          # no samples at all: absent
-    assert metrics[quiet] == {"cpu_pct": 12.0, "mem_used_pct": 34.0}
 
 
 def test_retention_drops_a_latest_value_with_its_raw_history(app):