feat(ui): alerts, monitors, hosts, schedules and maintenance lists put problems first, with since-times (#5210)

#5210 "Problems first, with since-time, across every list" (milestone 477),
first batch.

- Alerts: the Active list reads firing, no data, acknowledged, pending, the
  longest-running first, and re-polls itself. An acknowledged alert says how
  long it has been breaching (from the alert's events; acknowledging moves
  last_transition_at) and, apart from that, who acknowledged it and when.
  Rules show what each is doing now; disabled rules sort last. The current
  value reads the latest-value table (#5208) instead of plugin_metrics.
- Monitors: a down monitor says "Down 12m · connection refused" in the list.
- Hosts: sorted down, agent gone quiet, pending, healthy. The agent column
  shows "—" for a missing reading (was 0%) and a Stale badge with its age.
  agent_metrics_by_host is gone; the list reads agent_overview_by_host.
- Schedules: failing first (failed or interrupted run, or a firing that could
  not start one), then running, then the rest, paused last; status badge
  from the shared states.
- Maintenance windows: active (ending soonest), then upcoming (starting
  soonest), with expired folded away; each says when it ends/starts/ended.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/routes.py

@@ -7,7 +7,7 @@ from steward.auth.middleware import require_role
 from steward.alerts import service as alert_svc
 from steward.core.time_range import parse_range
 from steward.ansible import sources as src_module
-from steward.models.alerts import AlertRule, AlertOperator, MaintenanceWindow
+from steward.models.alerts import AlertRule, AlertOperator, AlertState, MaintenanceWindow
 from steward.models.users import UserRole
 
 alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")
@@ -98,8 +98,23 @@ async def list_alerts():
     async with current_app.db_sessionmaker() as db:
         active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
         rules = await alert_svc.list_rules(db)
+        states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
 
-    return await render_template("alerts/list.html", active=active, rules=rules, operators=_OPERATORS)
+    # Every rule with what it is doing now; disabled rules last.
+    rules.sort(key=lambda r: (not r.enabled, r.name.lower()))
+    rule_state = {r.id: alert_svc.alert_label(states[r.id]) for r in rules if r.id in states}
+    return await render_template(
+        "alerts/list.html", active=active, rules=rules, rule_state=rule_state,
+        operators=_OPERATORS, poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60))
+
+
+@alerts_bp.get("/active")
+@require_role(UserRole.viewer)
+async def active_fragment():
+    """The Active section alone, which the alerts page polls."""
+    async with current_app.db_sessionmaker() as db:
+        active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
+    return await render_template("alerts/_active.html", active=active)
 
 
 HISTORY_LIMIT = 200
@@ -298,9 +313,14 @@ async def list_maintenance():
     now = datetime.now(timezone.utc)
     async with current_app.db_sessionmaker() as db:
         windows = await alert_svc.list_maintenance_windows(db)
+    # Active first (ending soonest first), then upcoming (starting soonest
+    # first); expired ones are history, newest first, folded away.
+    active = sorted((w for w in windows if w.start_at <= now <= w.end_at), key=lambda w: w.end_at)
+    upcoming = sorted((w for w in windows if w.start_at > now), key=lambda w: w.start_at)
+    expired = sorted((w for w in windows if w.end_at < now), key=lambda w: w.end_at, reverse=True)
     return await render_template(
         "alerts/maintenance.html",
-        windows=windows,
+        windows=windows, active=active, upcoming=upcoming, expired=expired,
         now=now,
         source_modules=alert_svc.source_modules(),
     )

steward/alerts/service.py

@@ -9,12 +9,12 @@ from __future__ import annotations
 import logging
 from datetime import datetime, timezone
 
-from sqlalchemy import or_, select
+from sqlalchemy import func, or_, select
 
 from steward.models.alerts import (
     AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
 )
-from steward.models.metrics import PluginMetric
+from steward.models.metrics import PluginMetric, PluginMetricLatest
 from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
 
 logger = logging.getLogger(__name__)
@@ -86,13 +86,13 @@ def alert_label(state: AlertState) -> str:
 
 
 async def latest_value(db, rule: AlertRule) -> float | None:
-    """The newest sample of the series a rule watches, or None before the first."""
+    """The newest sample of the series a rule watches, or None before the first
+    (a primary-key read of the latest-value table, #5208)."""
     return (await db.execute(
-        select(PluginMetric.value)
-        .where(PluginMetric.source_module == rule.source_module,
-               PluginMetric.resource_name == rule.resource_name,
-               PluginMetric.metric_name == rule.metric_name)
-        .order_by(PluginMetric.recorded_at.desc()).limit(1)
+        select(PluginMetricLatest.value)
+        .where(PluginMetricLatest.source_module == rule.source_module,
+               PluginMetricLatest.resource_name == rule.resource_name,
+               PluginMetricLatest.metric_name == rule.metric_name)
     )).scalar_one_or_none()
 
 
@@ -143,7 +143,10 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None,
     if label == "no_data":
         return {"lead": f"No {metric} from ", "resource": rule.resource_name,
                 "tail": f" for {held}" if held else ""}
-    for_how_long = f", {label} for {held}" if held else f", {label}"
+    # An acknowledged alert is still breaching: say for how long it has been,
+    # not how long ago someone acknowledged it (the page says that apart).
+    word = "breaching" if label == "acknowledged" else label
+    for_how_long = f", {word} for {held}" if held else f", {word}"
     if value is None:
         reading = " has no reading yet"
     elif metric in _BINARY:
@@ -156,18 +159,59 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None,
             "tail": f"{reading}{condition}{for_how_long}"}
 
 
+# The order the active list reads in: what is breaching and unhandled, then
+# what has gone silent, then what someone has taken, then what may yet clear.
+ACTIVE_ORDER = {"firing": 0, "no_data": 1, "acknowledged": 2, "pending": 3}
+
+_BREACHING = (AlertStateEnum.pending.value, AlertStateEnum.firing.value,
+              AlertStateEnum.acknowledged.value)
+
+
+async def breach_started(db, rule_ids) -> dict[str, datetime]:
+    """{rule_id: when its current breach began}: the newest transition into
+    pending or firing from a quiet state. Acknowledging moves an alert's
+    last_transition_at, so that column cannot say this."""
+    if not rule_ids:
+        return {}
+    return dict((await db.execute(
+        select(AlertEvent.rule_id, func.max(AlertEvent.transitioned_at))
+        .where(AlertEvent.rule_id.in_(list(rule_ids)),
+               AlertEvent.from_state.not_in(_BREACHING),
+               AlertEvent.to_state.in_((AlertStateEnum.pending.value, AlertStateEnum.firing.value)))
+        .group_by(AlertEvent.rule_id)
+    )).all())
+
+
 async def active_rows(db, now: datetime) -> list[dict]:
-    """The alerts page's active list: each alert's state, rule, sentence and
-    the link to what it watches."""
+    """The alerts page's active list, firing first, then no data, then
+    acknowledged, then pending; within each the longest-running first. Each
+    row: its state, rule, label, sentence, the link to what it watches, when
+    the breach began, and who acknowledged it."""
+    from steward.models.users import User
+    alerts = await active_alerts(db)
+    started = await breach_started(db, {rule.id for state, rule in alerts
+                                        if state.state.value in _BREACHING})
+    ackers = {s.acknowledged_by for s, _ in alerts if s.acknowledged_by}
+    names = dict((await db.execute(
+        select(User.id, User.username).where(User.id.in_(ackers)))).all()) if ackers else {}
     rows = []
-    for state, rule in await active_alerts(db):
+    for state, rule in alerts:
         label = alert_label(state)
-        since = state.no_data_since if label == "no_data" else state.last_transition_at
+        if label == "no_data":
+            since = state.no_data_since
+        elif label == "acknowledged":
+            since = started.get(rule.id, state.last_transition_at)
+        else:
+            since = state.last_transition_at
         rows.append({
-            "state": state, "rule": rule, "label": label,
+            "state": state, "rule": rule, "label": label, "since": since,
+            "breach_since": started.get(rule.id) if label != "no_data" else None,
+            "acked_by": names.get(state.acknowledged_by, state.acknowledged_by),
             "says": alert_sentence(rule, label, await latest_value(db, rule), since, now),
             "url": await resource_url(db, rule),
         })
+    epoch = datetime.max.replace(tzinfo=timezone.utc)
+    rows.sort(key=lambda r: (ACTIVE_ORDER.get(r["label"], 9), r["since"] or epoch))
     return rows