feat(ui): alerts, monitors, hosts, schedules and maintenance lists put problems first, with since-times (#5210)

#5210 "Problems first, with since-time, across every list" (milestone 477),
first batch.

- Alerts: the Active list reads firing, no data, acknowledged, pending, the
  longest-running first, and re-polls itself. An acknowledged alert says how
  long it has been breaching (from the alert's events; acknowledging moves
  last_transition_at) and, apart from that, who acknowledged it and when.
  Rules show what each is doing now; disabled rules sort last. The current
  value reads the latest-value table (#5208) instead of plugin_metrics.
- Monitors: a down monitor says "Down 12m · connection refused" in the list.
- Hosts: sorted down, agent gone quiet, pending, healthy. The agent column
  shows "—" for a missing reading (was 0%) and a Stale badge with its age.
  agent_metrics_by_host is gone; the list reads agent_overview_by_host.
- Schedules: failing first (failed or interrupted run, or a firing that could
  not start one), then running, then the rest, paused last; status badge
  from the shared states.
- Maintenance windows: active (ending soonest), then upcoming (starting
  soonest), with expired folded away; each says when it ends/starts/ended.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/routes.py

@@ -7,7 +7,7 @@ from steward.auth.middleware import require_role
 from steward.alerts import service as alert_svc
 from steward.core.time_range import parse_range
 from steward.ansible import sources as src_module
-from steward.models.alerts import AlertRule, AlertOperator, MaintenanceWindow
+from steward.models.alerts import AlertRule, AlertOperator, AlertState, MaintenanceWindow
 from steward.models.users import UserRole
 
 alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")
@@ -98,8 +98,23 @@ async def list_alerts():
     async with current_app.db_sessionmaker() as db:
         active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
         rules = await alert_svc.list_rules(db)
+        states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
 
-    return await render_template("alerts/list.html", active=active, rules=rules, operators=_OPERATORS)
+    # Every rule with what it is doing now; disabled rules last.
+    rules.sort(key=lambda r: (not r.enabled, r.name.lower()))
+    rule_state = {r.id: alert_svc.alert_label(states[r.id]) for r in rules if r.id in states}
+    return await render_template(
+        "alerts/list.html", active=active, rules=rules, rule_state=rule_state,
+        operators=_OPERATORS, poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60))
+
+
+@alerts_bp.get("/active")
+@require_role(UserRole.viewer)
+async def active_fragment():
+    """The Active section alone, which the alerts page polls."""
+    async with current_app.db_sessionmaker() as db:
+        active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
+    return await render_template("alerts/_active.html", active=active)
 
 
 HISTORY_LIMIT = 200
@@ -298,9 +313,14 @@ async def list_maintenance():
     now = datetime.now(timezone.utc)
     async with current_app.db_sessionmaker() as db:
         windows = await alert_svc.list_maintenance_windows(db)
+    # Active first (ending soonest first), then upcoming (starting soonest
+    # first); expired ones are history, newest first, folded away.
+    active = sorted((w for w in windows if w.start_at <= now <= w.end_at), key=lambda w: w.end_at)
+    upcoming = sorted((w for w in windows if w.start_at > now), key=lambda w: w.start_at)
+    expired = sorted((w for w in windows if w.end_at < now), key=lambda w: w.end_at, reverse=True)
     return await render_template(
         "alerts/maintenance.html",
-        windows=windows,
+        windows=windows, active=active, upcoming=upcoming, expired=expired,
         now=now,
         source_modules=alert_svc.source_modules(),
     )

steward/alerts/service.py

@@ -9,12 +9,12 @@ from __future__ import annotations
 import logging
 from datetime import datetime, timezone
 
-from sqlalchemy import or_, select
+from sqlalchemy import func, or_, select
 
 from steward.models.alerts import (
     AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
 )
-from steward.models.metrics import PluginMetric
+from steward.models.metrics import PluginMetric, PluginMetricLatest
 from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
 
 logger = logging.getLogger(__name__)
@@ -86,13 +86,13 @@ def alert_label(state: AlertState) -> str:
 
 
 async def latest_value(db, rule: AlertRule) -> float | None:
-    """The newest sample of the series a rule watches, or None before the first."""
+    """The newest sample of the series a rule watches, or None before the first
+    (a primary-key read of the latest-value table, #5208)."""
     return (await db.execute(
-        select(PluginMetric.value)
+        select(PluginMetricLatest.value)
-        .where(PluginMetric.source_module == rule.source_module,
+        .where(PluginMetricLatest.source_module == rule.source_module,
-               PluginMetric.resource_name == rule.resource_name,
+               PluginMetricLatest.resource_name == rule.resource_name,
-               PluginMetric.metric_name == rule.metric_name)
+               PluginMetricLatest.metric_name == rule.metric_name)
-        .order_by(PluginMetric.recorded_at.desc()).limit(1)
     )).scalar_one_or_none()
 
 
@@ -143,7 +143,10 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None,
     if label == "no_data":
         return {"lead": f"No {metric} from ", "resource": rule.resource_name,
                 "tail": f" for {held}" if held else ""}
-    for_how_long = f", {label} for {held}" if held else f", {label}"
+    # An acknowledged alert is still breaching: say for how long it has been,
+    # not how long ago someone acknowledged it (the page says that apart).
+    word = "breaching" if label == "acknowledged" else label
+    for_how_long = f", {word} for {held}" if held else f", {word}"
     if value is None:
         reading = " has no reading yet"
     elif metric in _BINARY:
@@ -156,18 +159,59 @@ def alert_sentence(rule: AlertRule, label: str, value: float | None,
             "tail": f"{reading}{condition}{for_how_long}"}
 
 
+# The order the active list reads in: what is breaching and unhandled, then
+# what has gone silent, then what someone has taken, then what may yet clear.
+ACTIVE_ORDER = {"firing": 0, "no_data": 1, "acknowledged": 2, "pending": 3}
+
+_BREACHING = (AlertStateEnum.pending.value, AlertStateEnum.firing.value,
+              AlertStateEnum.acknowledged.value)
+
+
+async def breach_started(db, rule_ids) -> dict[str, datetime]:
+    """{rule_id: when its current breach began}: the newest transition into
+    pending or firing from a quiet state. Acknowledging moves an alert's
+    last_transition_at, so that column cannot say this."""
+    if not rule_ids:
+        return {}
+    return dict((await db.execute(
+        select(AlertEvent.rule_id, func.max(AlertEvent.transitioned_at))
+        .where(AlertEvent.rule_id.in_(list(rule_ids)),
+               AlertEvent.from_state.not_in(_BREACHING),
+               AlertEvent.to_state.in_((AlertStateEnum.pending.value, AlertStateEnum.firing.value)))
+        .group_by(AlertEvent.rule_id)
+    )).all())
+
+
 async def active_rows(db, now: datetime) -> list[dict]:
-    """The alerts page's active list: each alert's state, rule, sentence and
+    """The alerts page's active list, firing first, then no data, then
-    the link to what it watches."""
+    acknowledged, then pending; within each the longest-running first. Each
+    row: its state, rule, label, sentence, the link to what it watches, when
+    the breach began, and who acknowledged it."""
+    from steward.models.users import User
+    alerts = await active_alerts(db)
+    started = await breach_started(db, {rule.id for state, rule in alerts
+                                        if state.state.value in _BREACHING})
+    ackers = {s.acknowledged_by for s, _ in alerts if s.acknowledged_by}
+    names = dict((await db.execute(
+        select(User.id, User.username).where(User.id.in_(ackers)))).all()) if ackers else {}
     rows = []
-    for state, rule in await active_alerts(db):
+    for state, rule in alerts:
         label = alert_label(state)
-        since = state.no_data_since if label == "no_data" else state.last_transition_at
+        if label == "no_data":
+            since = state.no_data_since
+        elif label == "acknowledged":
+            since = started.get(rule.id, state.last_transition_at)
+        else:
+            since = state.last_transition_at
         rows.append({
-            "state": state, "rule": rule, "label": label,
+            "state": state, "rule": rule, "label": label, "since": since,
+            "breach_since": started.get(rule.id) if label != "no_data" else None,
+            "acked_by": names.get(state.acknowledged_by, state.acknowledged_by),
             "says": alert_sentence(rule, label, await latest_value(db, rule), since, now),
             "url": await resource_url(db, rule),
         })
+    epoch = datetime.max.replace(tzinfo=timezone.utc)
+    rows.sort(key=lambda r: (ACTIVE_ORDER.get(r["label"], 9), r["since"] or epoch))
     return rows
 
 

steward/ansible/routes.py

@@ -443,6 +443,7 @@ async def schedules():
         })
         if editing_id and s.id == editing_id:
             editing = s
+    rows.sort(key=_schedule_order)
 
     # Default the source dropdown to the first source (or the edited schedule's
     # source), and pre-render that source's playbooks so the playbook dropdown
@@ -472,6 +473,23 @@ async def schedules():
     )
 
 
+def _schedule_order(row: dict) -> tuple:
+    """Schedules list order: failing first (its last run failed or was cut
+    short, or its last firing could not start one), then running, then the
+    rest, paused last; by name within each."""
+    s, run = row["s"], row["last_run"]
+    status = run.status.value if run else None
+    if not s.enabled:
+        rank = 3
+    elif status in ("failed", "interrupted") or (s.last_error and run is None):
+        rank = 0
+    elif status in ("queued", "running"):
+        rank = 1
+    else:
+        rank = 2
+    return rank, s.name.lower()
+
+
 def _schedule_form_fields(form) -> tuple[dict | None, str | None]:
     """Validate + extract schedule fields from a form. Returns (fields, error)."""
     name = (form.get("name", "") or "").strip()

steward/hosts/routes.py

@@ -50,14 +50,17 @@ async def list_hosts():
         hosts = result.scalars().all()
         host_status = await host_svc.host_status(db, [h.id for h in hosts])
         uptime = await host_svc.compute_uptime(db)
-        agent_metrics = await host_svc.agent_metrics_by_host(db, [h.name for h in hosts])
+        agent = await host_svc.agent_overview_by_host(
+            db, [h.name for h in hosts], host_svc.agent_stale_after_seconds(current_app.config))
 
+    hosts = sorted(hosts, key=lambda h: (
+        host_svc.host_rank(host_status.get(h.id), agent.get(h.name)), h.name.lower()))
     return await render_template(
         "hosts/list.html",
         hosts=hosts,
         host_status=host_status,
         uptime=uptime,
-        agent_metrics=agent_metrics,
+        agent=agent,
     )
 
 

steward/hosts/service.py

@@ -113,6 +113,21 @@ async def host_status(db, host_ids: list[str]) -> dict[str, dict]:
     return out
 
 
+def host_rank(status: dict | None, agent: dict | None) -> int:
+    """Where a host sorts in the Hosts list: a down monitor first, then an
+    agent gone quiet, then monitors not yet checked, then the healthy; a host
+    with neither monitors nor agent last."""
+    if status and status["state"] == "down":
+        return 0
+    if agent and not agent.get("fresh"):
+        return 1
+    if status and status["state"] == "pending":
+        return 2
+    if status or agent:
+        return 3
+    return 4
+
+
 async def host_monitors(db, host_id: str) -> list[dict]:
     """Display rows for the monitors linked to one host (host hub section)."""
     monitors = (await db.execute(
@@ -134,17 +149,6 @@ async def host_monitors(db, host_id: str) -> list[dict]:
     return data
 
 
-async def agent_metrics_by_host(db, host_names: list[str]) -> dict[str, dict]:
-    """Latest agent cpu/mem per host name, from plugin_metric_latest.
-
-    Core-safe (no host_agent import), so the Hosts list can show an
-    at-a-glance agent column without importing the host_agent plugin.
-    """
-    from steward.core.metric_series import latest_by_resource
-    latest = await latest_by_resource(db, "host_agent", host_names, ("cpu_pct", "mem_used_pct"))
-    return {name: {m: r.value for m, r in metrics.items()} for name, metrics in latest.items()}
-
-
 async def agent_overview_by_host(
     db, host_names: list[str], stale_after_seconds: int = 180,
 ) -> dict[str, dict]:

steward/monitors/service.py

@@ -152,6 +152,20 @@ def row_state(monitor: Monitor, latest, uptime_pct, thresholds: dict) -> str:
     return "up"
 
 
+def down_reason(latest) -> str | None:
+    """Why a failed check failed, in a few words: the probe's error, else the
+    HTTP status, else a content mismatch; None when the result says nothing."""
+    if latest is None:
+        return None
+    if latest.error_msg:
+        return latest.error_msg
+    if latest.status_code:
+        return f"HTTP {latest.status_code}"
+    if latest.content_matched is False:
+        return "expected content not found"
+    return None
+
+
 def problems_first(rows: list[dict]) -> list[dict]:
     """Down, then degraded, then pending, up and paused; by name within each."""
     return sorted(rows, key=lambda d: (STATE_ORDER[d["state"]], d["monitor"].name.lower()))
@@ -191,7 +205,13 @@ async def monitor_rows(db, range_key: str, thresholds: dict):
                            "title": _heartbeat_title(m.type, r)} for r in rows],
             "uptime_pct": uptime_pct,
             "tls_days": tls_days,
+            "down_since": None,
+            "down_reason": down_reason(latest) if latest is not None and not latest.is_up else None,
         })
+    # When each down monitor went down (a query per down monitor: few, and indexed).
+    for d in data:
+        if d["state"] == "down":
+            d["down_since"] = await down_since(db, d["monitor"].id)
     counts = {k: sum(d["state"] == k for d in data) for k in STATE_ORDER}
     counts["up"] += counts["degraded"]
     return problems_first(data), counts, range_key

steward/templates/alerts/_active.html

@@ -0,0 +1,51 @@
+{# alerts/_active.html — the Alerts page's Active section, polled on its own:
+   firing first, then no data, then acknowledged, then pending
+   (alert_svc.active_rows). An acknowledged alert says how long it has been
+   breaching, and apart from that who acknowledged it and when. #}
+{% from "_macros.html" import status_badge, empty_state %}
+{% set can_edit = session.user_role in ('operator', 'admin') %}
+{% if active %}
+<h2 class="section-title" style="margin-bottom:1rem;">Active</h2>
+<div class="card-flush" style="margin-bottom:2rem;">
+    <table class="table">
+        <thead>
+            <tr>
+                <th>State</th>
+                <th>What is wrong</th>
+                <th>Rule</th>
+                <th></th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for row in active %}
+            <tr>
+                <td>
+                    {{ status_badge(row.label, title=("No sample since " ~ (row.state.last_sample_at.strftime('%Y-%m-%d %H:%M UTC') if row.state.last_sample_at else 'the rule started watching')) if row.label == 'no_data' else none) }}
+                    {% if row.state.no_data_since and row.label != 'no_data' %}
+                    {{ status_badge('no_data', title="Silent since " ~ row.state.no_data_since.strftime('%Y-%m-%d %H:%M UTC')) }}
+                    {% endif %}
+                </td>
+                <td>
+                    {{- row.says.lead -}}
+                    {%- if row.url %}<a href="{{ row.url }}">{{ row.says.resource }}</a>{% else %}<strong>{{ row.says.resource }}</strong>{% endif -%}
+                    {{- row.says.tail -}}
+                    {%- if row.label == 'acknowledged' and row.state.acknowledged_at %}
+                    <div class="field-hint" style="margin-top:0.15rem;">Acknowledged{% if row.acked_by %} by {{ row.acked_by }}{% endif %} {{ row.state.acknowledged_at|ago }}</div>
+                    {%- endif %}
+                </td>
+                <td style="color:var(--text-muted);"><a href="/alerts/history?rule={{ row.rule.id }}">{{ row.rule.name }}</a></td>
+                <td class="td-actions">
+                    {% if can_edit and row.state.state.value == 'firing' %}
+                    <form method="post" action="/alerts/{{ row.rule.id }}/acknowledge" style="display:inline;">{{ csrf_field() }}
+                        <button type="submit" class="btn btn-sm btn-warn">Acknowledge</button>
+                    </form>
+                    {% endif %}
+                </td>
+            </tr>
+            {% endfor %}
+        </tbody>
+    </table>
+</div>
+{% else %}
+<div class="card" style="margin-bottom:2rem;">{{ empty_state("No active alerts. Every rule is quiet.") }}</div>
+{% endif %}

steward/templates/alerts/list.html

@@ -9,48 +9,11 @@
     {% if can_edit %}<a class="btn" href="/alerts/rules/new">New rule</a>{% endif %}
 {% endcall %}
 
-{% if active %}
+{# The Active section re-reads itself, so an alert that clears or starts shows
-<h2 class="section-title" style="margin-bottom:1rem;">Active</h2>
+   without a reload. #}
-<div class="card-flush" style="margin-bottom:2rem;">
+<div id="alerts-active" hx-get="/alerts/active" hx-trigger="every {{ poll_interval or 60 }}s" hx-swap="innerHTML">
-    <table class="table">
+{% include "alerts/_active.html" %}
-        <thead>
-            <tr>
-                <th>State</th>
-                <th>What is wrong</th>
-                <th>Rule</th>
-                <th></th>
-            </tr>
-        </thead>
-        <tbody>
-            {% for row in active %}
-            <tr>
-                <td>
-                    {{ status_badge(row.label, title=("No sample since " ~ (row.state.last_sample_at.strftime('%Y-%m-%d %H:%M UTC') if row.state.last_sample_at else 'the rule started watching')) if row.label == 'no_data' else none) }}
-                    {% if row.state.no_data_since and row.label != 'no_data' %}
-                    {{ status_badge('no_data', title="Silent since " ~ row.state.no_data_since.strftime('%Y-%m-%d %H:%M UTC')) }}
-                    {% endif %}
-                </td>
-                <td>
-                    {{- row.says.lead -}}
-                    {%- if row.url %}<a href="{{ row.url }}">{{ row.says.resource }}</a>{% else %}<strong>{{ row.says.resource }}</strong>{% endif -%}
-                    {{- row.says.tail -}}
-                </td>
-                <td style="color:var(--text-muted);"><a href="/alerts/history?rule={{ row.rule.id }}">{{ row.rule.name }}</a></td>
-                <td class="td-actions">
-                    {% if can_edit and row.state.state.value == 'firing' %}
-                    <form method="post" action="/alerts/{{ row.rule.id }}/acknowledge" style="display:inline;">{{ csrf_field() }}
-                        <button type="submit" class="btn btn-sm btn-warn">Acknowledge</button>
-                    </form>
-                    {% endif %}
-                </td>
-            </tr>
-            {% endfor %}
-        </tbody>
-    </table>
 </div>
-{% else %}
-<div class="card" style="margin-bottom:2rem;">{{ empty_state("No active alerts. Every rule is quiet.") }}</div>
-{% endif %}
 
 <h2 class="section-title" style="margin-bottom:1rem;">Rules</h2>
 {% if rules %}
@@ -59,6 +22,7 @@
         <thead>
             <tr>
                 <th>Name</th>
+                <th>Now</th>
                 <th>Condition</th>
                 <th style="text-align:center;" title="Checks in a row to fire / to clear">Fire / clear</th>
                 <th></th>
@@ -73,6 +37,12 @@
                   <span style="font-size:0.72rem;color:var(--text-dim);margin-left:0.35rem;">disabled</span>
                   {% endif %}
                 </td>
+                <td>
+                  {%- set now_label = (rule_state or {}).get(rule.id) %}
+                  {%- if not rule.enabled %}<span class="tone-dim" style="font-size:0.8rem;">off</span>
+                  {%- elif now_label and now_label not in ('inactive', 'resolved') %}{{ status_badge(now_label) }}
+                  {%- else %}<span class="tone-ok" style="font-size:0.8rem;" title="Within its threshold">quiet</span>{% endif -%}
+                </td>
                 <td style="color:var(--text-muted);font-size:0.85rem;">
                   <span style="color:var(--text-dim);">{{ rule.source_module }}/</span>{{ rule.resource_name }}
                   <span style="font-family:ui-monospace,monospace;font-size:0.8rem;">

steward/templates/alerts/maintenance.html

@@ -7,8 +7,8 @@
     <a class="btn" href="/alerts/maintenance/new">New window</a>
 {% endcall %}
 
-{% if windows %}
+{% macro window_table(rows, kind) %}
-<div class="card-flush">
+<div class="card-flush"{% if kind != 'expired' %} style="margin-bottom:1.5rem;"{% endif %}>
   <table class="table">
     <thead>
       <tr>
@@ -21,9 +21,7 @@
       </tr>
     </thead>
     <tbody>
-      {% for w in windows %}
+      {% for w in rows %}
-      {% set is_active = w.start_at <= now and w.end_at >= now %}
-      {% set is_upcoming = w.start_at > now %}
       <tr>
         <td style="font-weight:500;">{{ w.name }}</td>
         <td style="font-size:0.85rem;color:var(--text-muted);">
@@ -41,10 +39,10 @@
         <td style="font-size:0.85rem;font-family:ui-monospace,monospace;">
           {{ w.end_at.strftime("%Y-%m-%d %H:%M") }} UTC
         </td>
-        <td style="text-align:center;">
+        <td style="text-align:center;font-size:0.8rem;">
-          {% if is_active %}{{ status_badge('warn', 'Active') }}
+          {% if kind == 'active' %}{{ status_badge('warn', 'Active') }}<div class="field-hint">ends {{ w.end_at|ago }}</div>
-          {% elif is_upcoming %}{{ status_badge('pending', 'Upcoming') }}
+          {% elif kind == 'upcoming' %}{{ status_badge('pending', 'Upcoming') }}<div class="field-hint">starts {{ w.start_at|ago }}</div>
-          {% else %}{{ status_badge('inactive', 'Expired') }}{% endif %}
+          {% else %}{{ status_badge('inactive', 'Expired') }}<div class="field-hint">ended {{ w.end_at|ago }}</div>{% endif %}
         </td>
         <td class="td-actions">
           <form method="post" action="/alerts/maintenance/{{ w.id }}/delete" style="display:inline;">{{ csrf_field() }}
@@ -57,6 +55,26 @@
     </tbody>
   </table>
 </div>
+{% endmacro %}
+
+{% if windows %}
+{% if active %}
+<h2 class="section-title" style="margin-bottom:0.75rem;">Active now</h2>
+{{ window_table(active, 'active') }}
+{% endif %}
+{% if upcoming %}
+<h2 class="section-title" style="margin-bottom:0.75rem;">Upcoming</h2>
+{{ window_table(upcoming, 'upcoming') }}
+{% endif %}
+{% if not active and not upcoming %}
+<div class="card" style="margin-bottom:1.5rem;">{{ empty_state("No window is active or scheduled. Every alert notifies as usual.") }}</div>
+{% endif %}
+{% if expired %}
+<details>
+  <summary class="section-title" style="cursor:pointer;margin-bottom:0.75rem;">Expired ({{ expired|length }})</summary>
+  {{ window_table(expired, 'expired') }}
+</details>
+{% endif %}
 {% else %}
 <div class="card">{{ empty_state("No maintenance windows yet.", "New window", "/alerts/maintenance/new") }}</div>
 {% endif %}

steward/templates/ansible/schedules.html

@@ -1,5 +1,5 @@
 {% extends "base.html" %}
-{% from "_macros.html" import crumbs %}
+{% from "_macros.html" import crumbs, status_badge %}
 {% block title %}Ansible Schedules — Steward{% endblock %}
 {% block breadcrumb %}{{ crumbs([("Ansible", "/ansible/"), ("Schedules", "")]) }}{% endblock %}
 
@@ -39,9 +39,8 @@
     <td style="font-size:0.85rem;">{{ fmt_interval(s.interval_seconds) }}</td>
     <td style="font-size:0.82rem;">
       {% if row.last_run %}
-        {% set sc = {"queued":"var(--text-dim)","running":"var(--yellow)","success":"var(--green)","failed":"var(--red)","interrupted":"var(--orange)","cancelled":"var(--orange)"}.get(row.last_run.status.value, "var(--text-muted)") %}
+        <a href="/ansible/runs/{{ row.last_run.id }}" style="text-decoration:none;">{{ status_badge(row.last_run.status) }}</a>
-        <a href="/ansible/runs/{{ row.last_run.id }}" style="color:{{ sc }};font-weight:500;">{{ row.last_run.status.value }}</a>
+        <span style="color:var(--text-dim);">· {{ row.last_run.started_at|ago }}</span>
-        <span style="color:var(--text-dim);">· {{ row.last_run.started_at.strftime("%m-%d %H:%M") }}</span>
       {% endif %}
       {% if s.last_error %}
         {# With a run beside it, the last firing was skipped because that run was

steward/templates/hosts/list.html

@@ -1,4 +1,8 @@
 {% extends "base.html" %}
+{% from "_macros.html" import status_badge, status_dot %}
+{# Hosts, problems first (host_svc.host_rank): a down monitor, then an agent
+   gone quiet, then monitors not yet checked, then the healthy. #}
+{% macro pct(v) %}{% if v is none %}—{% else %}{{ '%.0f'|format(v) }}%{% endif %}{% endmacro %}
 {% block title %}Hosts — Steward{% endblock %}
 {% block content %}
 <div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1.5rem;">
@@ -38,13 +42,7 @@
                 <td style="text-align:center;">
                     {% if not hs %}
                         <span style="color:var(--text-dim);font-size:0.8rem;" title="No monitors">—</span>
-                    {% elif hs.state == 'up' %}
+                    {% else %}{{ status_dot(hs.state) }}{% endif %}
-                        <span class="dot dot-up" title="Up"></span>
-                    {% elif hs.state == 'down' %}
-                        <span class="dot dot-down" title="Down"></span>
-                    {% else %}
-                        <span class="dot dot-dim" title="Pending"></span>
-                    {% endif %}
                 </td>
                 <td style="text-align:center;font-size:0.9rem;">
                     {% if hs and hs.latency_ms is not none %}
@@ -66,14 +64,13 @@
                     {% endif %}
                 </td>
                 {% endfor %}
-                {% set am = agent_metrics.get(host.name) %}
+                {% set am = agent.get(host.name) %}
                 <td style="text-align:center;font-size:0.8rem;font-family:ui-monospace,monospace;">
                     {% if am %}
-                        <span style="color:var(--text-muted);" title="CPU / memory">
+                        <span style="color:var(--text-muted);" title="CPU / memory, latest">{{ pct(am.cpu_pct) }} / {{ pct(am.mem_used_pct) }}</span>
-                            {{ '%.0f'|format(am.get('cpu_pct', 0)) }}% / {{ '%.0f'|format(am.get('mem_used_pct', 0)) }}%
+                        {% if not am.fresh %}<div>{{ status_badge('stale', title="No agent report since " ~ (am.last_at.strftime('%Y-%m-%d %H:%M UTC') if am.last_at else 'it started')) }} <span class="tone-dim" style="font-family:inherit;">{{ am.last_at|ago }}</span></div>{% endif %}
-                        </span>
                     {% else %}
-                        <span style="color:var(--text-dim);">—</span>
+                        <span style="color:var(--text-dim);" title="No agent on this host">—</span>
                     {% endif %}
                 </td>
                 <td class="td-actions">

steward/templates/monitors/rows.html

@@ -22,6 +22,11 @@
       <a class="ping-name" href="/monitors/{{ m.id }}">{{ m.name }}</a>
     </div>
     <span class="ping-addr" title="{{ m.target }}">{{ m.type|monitor_type }} · {{ m.target }}</span>
+    {% if d.state == 'down' %}
+    <span class="tone-crit" style="font-size:0.75rem;overflow-wrap:anywhere;">
+      Down{% if d.down_since %} {{ d.down_since|ago|replace(' ago', '') }}{% endif %}{% if d.down_reason %} · {{ d.down_reason }}{% endif %}
+    </span>
+    {% endif %}
   </div>
 
   <div class="ping-pills">

tests/core/test_alerts_page.py

@@ -51,7 +51,8 @@ def test_an_up_or_down_metric_says_the_word():
     now = datetime.now(timezone.utc)
     says = alert_sentence(_rule("is_up", AlertOperator.lt, 1.0, "nas"), "acknowledged", 0.0,
                           now - timedelta(hours=2, minutes=5), now)
-    assert _said(says) == "nas is down, acknowledged for 2h 5m"
+    # Acknowledged is still breaching: the sentence says for how long (#5210).
+    assert _said(says) == "nas is down, breaching for 2h 5m"
 
 
 def test_no_reading_yet_is_said_not_shown_as_a_number():
@@ -132,3 +133,35 @@ async def test_no_count_for_a_fragment_or_a_logged_out_page(app, monkeypatch):
     async with app.test_request_context("/alerts/"):
         logged_out = await render_template("alerts/list.html", active=[], rules=[], operators=[])
     assert "count-badge" not in fragment and "count-badge" not in logged_out
+
+
+async def test_an_acknowledged_alert_says_who_and_when_apart_from_the_breach(app):
+    rule = _rule("is_up", AlertOperator.lt, 1.0, "nas")
+    now = datetime.now(timezone.utc)
+    st = SimpleNamespace(state=AlertStateEnum.acknowledged, no_data_since=None, last_sample_at=None,
+                         last_transition_at=now - timedelta(minutes=3),
+                         acknowledged_at=now - timedelta(minutes=3))
+    row = {"state": st, "rule": rule, "label": "acknowledged", "url": None, "acked_by": "bryan",
+           "says": alert_sentence(rule, "acknowledged", 0.0, now - timedelta(hours=1), now)}
+    async with app.test_request_context("/alerts/active"):
+        session["user_role"] = "operator"
+        page = await render_template("alerts/_active.html", active=[row])
+    page = " ".join(page.split())
+    assert "<strong>nas</strong> is down, breaching for 1h" in page
+    assert "Acknowledged by bryan <time" in page and ">3m ago</time>" in page
+    assert "/acknowledge" not in page                      # already taken
+
+
+async def test_rules_say_what_each_is_doing_now(app):
+    firing, quiet, off = _rule(rid="r1"), _rule(rid="r2", resource="db"), _rule(rid="r3", resource="old")
+    off.enabled = False
+    async with app.test_request_context("/alerts/"):
+        session["user_role"] = "viewer"
+        page = await render_template("alerts/list.html", active=[], rules=[firing, quiet, off],
+                                     rule_state={"r1": "firing", "r2": "inactive"}, operators=[])
+    page = " ".join(page.split())
+    rules = page[page.index(">Rules</h2>"):]
+    assert '<span class="badge badge-crit">Firing</span>' in rules
+    assert 'title="Within its threshold">quiet</span>' in rules
+    assert '<span class="tone-dim" style="font-size:0.8rem;">off</span>' in rules
+    assert 'hx-get="/alerts/active"' in page

tests/core/test_monitors_page.py

@@ -1,5 +1,6 @@
 """The Monitors page leads with what needs looking at (#5121): a count strip,
 problems first, each monitor on a page of its own, adding behind a button."""
+import re
 from datetime import datetime, timedelta, timezone
 from types import SimpleNamespace
 
@@ -115,3 +116,24 @@ async def test_adding_is_its_own_page_and_keeps_what_was_typed(app):
     assert '<a href="/monitors/">Monitors</a>' in page and 'action="/monitors/add"' in page
     assert '<p class="field-error" role="alert">port must be between 1 and 65535, got 0.</p>' in page
     assert 'value="db"' in page and 'value="10.0.0.9"' in page and 'value="30"' in page
+
+
+def test_down_reason_says_why_in_a_few_words():
+    from steward.monitors.service import down_reason
+    assert down_reason(_check(False)) == "503 Service Unavailable"       # the probe's own error first
+    assert down_reason(SimpleNamespace(error_msg=None, status_code=502, content_matched=None)) == "HTTP 502"
+    assert down_reason(SimpleNamespace(error_msg=None, status_code=None, content_matched=False)) == \
+        "expected content not found"
+    assert down_reason(None) is None
+
+
+async def test_a_down_monitor_says_since_when_and_why_in_the_list(app):
+    row = _row(_mon("api", mid="m2"), "down", _check(False), 90.0)
+    row.update(down_since=datetime.now(timezone.utc) - timedelta(minutes=12),
+               down_reason="connection refused")
+    async with app.test_request_context("/monitors/rows"):
+        page = await render_template("monitors/rows.html", monitor_data=[row],
+                                     counts={"up": 0, "degraded": 0, "down": 1, "pending": 0, "paused": 0},
+                                     range_key="24h")
+    page = " ".join(page.split())
+    assert re.search(r'Down <time [^>]*>12m</time> · connection refused', page), page

tests/core/test_problems_first.py

@@ -0,0 +1,31 @@
+"""Lists put what needs looking at first (#5210): hosts by their worst
+signal, schedules failing first, maintenance windows active then upcoming."""
+from types import SimpleNamespace as NS
+
+from steward.ansible.routes import _schedule_order
+from steward.hosts.service import host_rank
+
+
+def test_hosts_sort_down_then_quiet_agent_then_pending_then_healthy():
+    down, up, pending = {"state": "down"}, {"state": "up"}, {"state": "pending"}
+    fresh, quiet = {"fresh": True}, {"fresh": False}
+    assert host_rank(down, fresh) == 0
+    assert host_rank(up, quiet) == 1            # monitors fine, but the agent went silent
+    assert host_rank(None, quiet) == 1
+    assert host_rank(pending, fresh) == 2
+    assert host_rank(up, fresh) == 3 and host_rank(None, fresh) == 3
+    assert host_rank(None, None) == 4           # nothing watches it
+
+
+def _sched(name, *, enabled=True, last_error=None, status=None):
+    run = NS(status=NS(value=status)) if status else None
+    return {"s": NS(name=name, enabled=enabled, last_error=last_error), "last_run": run}
+
+
+def test_schedules_sort_failing_first_and_paused_last():
+    rows = [_sched("nightly", status="success"), _sched("paused", enabled=False, status="failed"),
+            _sched("backup", status="failed"), _sched("deploy", status="running"),
+            _sched("broken", last_error="source not found"), _sched("cut", status="interrupted"),
+            _sched("busy", last_error="the last run is still going", status="success")]
+    assert [r["s"].name for r in sorted(rows, key=_schedule_order)] == [
+        "backup", "broken", "cut", "deploy", "busy", "nightly", "paused"]

tests/integration/test_metric_latest.py

@@ -72,7 +72,7 @@ def test_the_newest_sample_wins_and_a_late_backlog_does_not_move_it_back(app):
 
 def test_a_quiet_host_keeps_its_last_values_and_reads_stale_not_missing(app):
     from steward.core.alerts import record_metrics
-    from steward.hosts.service import agent_metrics_by_host, agent_overview_by_host
+    from steward.hosts.service import agent_overview_by_host
 
     now = datetime.now(timezone.utc)
     quiet, live = ("quiet-" + uuid.uuid4().hex[:8]), ("live-" + uuid.uuid4().hex[:8])
@@ -87,18 +87,15 @@ def test_a_quiet_host_keeps_its_last_values_and_reads_stale_not_missing(app):
                     _sample(quiet + ":/", "disk_used_pct", 56.0, two_hours_ago),
                     _sample(live, "cpu_pct", 7.0, now - timedelta(seconds=20)),
                 ])
-            overview = await agent_overview_by_host(s, [quiet, live, "never-" + quiet], 180)
+            return await agent_overview_by_host(s, [quiet, live, "never-" + quiet], 180)
-            metrics = await agent_metrics_by_host(s, [quiet, live])
-            return overview, metrics
 
-    overview, metrics = asyncio.run(go())
+    overview = asyncio.run(go())
     q = overview[quiet]
     assert (q["cpu_pct"], q["mem_used_pct"], q["disk_root"]) == (12.0, 34.0, 56.0)
     assert q["fresh"] is False
     assert abs((q["last_at"] - two_hours_ago).total_seconds()) < 1
     assert overview[live]["fresh"] is True
     assert "never-" + quiet not in overview          # no samples at all: absent
-    assert metrics[quiet] == {"cpu_pct": 12.0, "mem_used_pct": 34.0}
 
 
 def test_retention_drops_a_latest_value_with_its_raw_history(app):