feat(ui): alerts, monitors, hosts, schedules and maintenance lists put problems first, with since-times (#5210)

#5210 "Problems first, with since-time, across every list" (milestone 477),
first batch.

- Alerts: the Active list reads firing, no data, acknowledged, pending, the
  longest-running first, and re-polls itself. An acknowledged alert says how
  long it has been breaching (from the alert's events; acknowledging moves
  last_transition_at) and, apart from that, who acknowledged it and when.
  Rules show what each is doing now; disabled rules sort last. The current
  value reads the latest-value table (#5208) instead of plugin_metrics.
- Monitors: a down monitor says "Down 12m · connection refused" in the list.
- Hosts: sorted down, agent gone quiet, pending, healthy. The agent column
  shows "—" for a missing reading (was 0%) and a Stale badge with its age.
  agent_metrics_by_host is gone; the list reads agent_overview_by_host.
- Schedules: failing first (failed or interrupted run, or a firing that could
  not start one), then running, then the rest, paused last; status badge
  from the shared states.
- Maintenance windows: active (ending soonest), then upcoming (starting
  soonest), with expired folded away; each says when it ends/starts/ended.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/templates/alerts/_active.html

@@ -0,0 +1,51 @@
+{# alerts/_active.html — the Alerts page's Active section, polled on its own:
+   firing first, then no data, then acknowledged, then pending
+   (alert_svc.active_rows). An acknowledged alert says how long it has been
+   breaching, and apart from that who acknowledged it and when. #}
+{% from "_macros.html" import status_badge, empty_state %}
+{% set can_edit = session.user_role in ('operator', 'admin') %}
+{% if active %}
+<h2 class="section-title" style="margin-bottom:1rem;">Active</h2>
+<div class="card-flush" style="margin-bottom:2rem;">
+    <table class="table">
+        <thead>
+            <tr>
+                <th>State</th>
+                <th>What is wrong</th>
+                <th>Rule</th>
+                <th></th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for row in active %}
+            <tr>
+                <td>
+                    {{ status_badge(row.label, title=("No sample since " ~ (row.state.last_sample_at.strftime('%Y-%m-%d %H:%M UTC') if row.state.last_sample_at else 'the rule started watching')) if row.label == 'no_data' else none) }}
+                    {% if row.state.no_data_since and row.label != 'no_data' %}
+                    {{ status_badge('no_data', title="Silent since " ~ row.state.no_data_since.strftime('%Y-%m-%d %H:%M UTC')) }}
+                    {% endif %}
+                </td>
+                <td>
+                    {{- row.says.lead -}}
+                    {%- if row.url %}<a href="{{ row.url }}">{{ row.says.resource }}</a>{% else %}<strong>{{ row.says.resource }}</strong>{% endif -%}
+                    {{- row.says.tail -}}
+                    {%- if row.label == 'acknowledged' and row.state.acknowledged_at %}
+                    <div class="field-hint" style="margin-top:0.15rem;">Acknowledged{% if row.acked_by %} by {{ row.acked_by }}{% endif %} {{ row.state.acknowledged_at|ago }}</div>
+                    {%- endif %}
+                </td>
+                <td style="color:var(--text-muted);"><a href="/alerts/history?rule={{ row.rule.id }}">{{ row.rule.name }}</a></td>
+                <td class="td-actions">
+                    {% if can_edit and row.state.state.value == 'firing' %}
+                    <form method="post" action="/alerts/{{ row.rule.id }}/acknowledge" style="display:inline;">{{ csrf_field() }}
+                        <button type="submit" class="btn btn-sm btn-warn">Acknowledge</button>
+                    </form>
+                    {% endif %}
+                </td>
+            </tr>
+            {% endfor %}
+        </tbody>
+    </table>
+</div>
+{% else %}
+<div class="card" style="margin-bottom:2rem;">{{ empty_state("No active alerts. Every rule is quiet.") }}</div>
+{% endif %}

steward/templates/alerts/list.html

@@ -9,48 +9,11 @@
     {% if can_edit %}<a class="btn" href="/alerts/rules/new">New rule</a>{% endif %}
 {% endcall %}
 
-{% if active %}
-<h2 class="section-title" style="margin-bottom:1rem;">Active</h2>
-<div class="card-flush" style="margin-bottom:2rem;">
-    <table class="table">
-        <thead>
-            <tr>
-                <th>State</th>
-                <th>What is wrong</th>
-                <th>Rule</th>
-                <th></th>
-            </tr>
-        </thead>
-        <tbody>
-            {% for row in active %}
-            <tr>
-                <td>
-                    {{ status_badge(row.label, title=("No sample since " ~ (row.state.last_sample_at.strftime('%Y-%m-%d %H:%M UTC') if row.state.last_sample_at else 'the rule started watching')) if row.label == 'no_data' else none) }}
-                    {% if row.state.no_data_since and row.label != 'no_data' %}
-                    {{ status_badge('no_data', title="Silent since " ~ row.state.no_data_since.strftime('%Y-%m-%d %H:%M UTC')) }}
-                    {% endif %}
-                </td>
-                <td>
-                    {{- row.says.lead -}}
-                    {%- if row.url %}<a href="{{ row.url }}">{{ row.says.resource }}</a>{% else %}<strong>{{ row.says.resource }}</strong>{% endif -%}
-                    {{- row.says.tail -}}
-                </td>
-                <td style="color:var(--text-muted);"><a href="/alerts/history?rule={{ row.rule.id }}">{{ row.rule.name }}</a></td>
-                <td class="td-actions">
-                    {% if can_edit and row.state.state.value == 'firing' %}
-                    <form method="post" action="/alerts/{{ row.rule.id }}/acknowledge" style="display:inline;">{{ csrf_field() }}
-                        <button type="submit" class="btn btn-sm btn-warn">Acknowledge</button>
-                    </form>
-                    {% endif %}
-                </td>
-            </tr>
-            {% endfor %}
-        </tbody>
-    </table>
+{# The Active section re-reads itself, so an alert that clears or starts shows
+   without a reload. #}
+<div id="alerts-active" hx-get="/alerts/active" hx-trigger="every {{ poll_interval or 60 }}s" hx-swap="innerHTML">
+{% include "alerts/_active.html" %}
 </div>
-{% else %}
-<div class="card" style="margin-bottom:2rem;">{{ empty_state("No active alerts. Every rule is quiet.") }}</div>
-{% endif %}
 
 <h2 class="section-title" style="margin-bottom:1rem;">Rules</h2>
 {% if rules %}
@@ -59,6 +22,7 @@
         <thead>
             <tr>
                 <th>Name</th>
+                <th>Now</th>
                 <th>Condition</th>
                 <th style="text-align:center;" title="Checks in a row to fire / to clear">Fire / clear</th>
                 <th></th>
@@ -73,6 +37,12 @@
                   <span style="font-size:0.72rem;color:var(--text-dim);margin-left:0.35rem;">disabled</span>
                   {% endif %}
                 </td>
+                <td>
+                  {%- set now_label = (rule_state or {}).get(rule.id) %}
+                  {%- if not rule.enabled %}<span class="tone-dim" style="font-size:0.8rem;">off</span>
+                  {%- elif now_label and now_label not in ('inactive', 'resolved') %}{{ status_badge(now_label) }}
+                  {%- else %}<span class="tone-ok" style="font-size:0.8rem;" title="Within its threshold">quiet</span>{% endif -%}
+                </td>
                 <td style="color:var(--text-muted);font-size:0.85rem;">
                   <span style="color:var(--text-dim);">{{ rule.source_module }}/</span>{{ rule.resource_name }}
                   <span style="font-family:ui-monospace,monospace;font-size:0.8rem;">

steward/templates/alerts/maintenance.html

@@ -7,8 +7,8 @@
     <a class="btn" href="/alerts/maintenance/new">New window</a>
 {% endcall %}
 
-{% if windows %}
-<div class="card-flush">
+{% macro window_table(rows, kind) %}
+<div class="card-flush"{% if kind != 'expired' %} style="margin-bottom:1.5rem;"{% endif %}>
   <table class="table">
     <thead>
       <tr>
@@ -21,9 +21,7 @@
       </tr>
     </thead>
     <tbody>
-      {% for w in windows %}
-      {% set is_active = w.start_at <= now and w.end_at >= now %}
-      {% set is_upcoming = w.start_at > now %}
+      {% for w in rows %}
       <tr>
         <td style="font-weight:500;">{{ w.name }}</td>
         <td style="font-size:0.85rem;color:var(--text-muted);">
@@ -41,10 +39,10 @@
         <td style="font-size:0.85rem;font-family:ui-monospace,monospace;">
           {{ w.end_at.strftime("%Y-%m-%d %H:%M") }} UTC
         </td>
-        <td style="text-align:center;">
-          {% if is_active %}{{ status_badge('warn', 'Active') }}
-          {% elif is_upcoming %}{{ status_badge('pending', 'Upcoming') }}
-          {% else %}{{ status_badge('inactive', 'Expired') }}{% endif %}
+        <td style="text-align:center;font-size:0.8rem;">
+          {% if kind == 'active' %}{{ status_badge('warn', 'Active') }}<div class="field-hint">ends {{ w.end_at|ago }}</div>
+          {% elif kind == 'upcoming' %}{{ status_badge('pending', 'Upcoming') }}<div class="field-hint">starts {{ w.start_at|ago }}</div>
+          {% else %}{{ status_badge('inactive', 'Expired') }}<div class="field-hint">ended {{ w.end_at|ago }}</div>{% endif %}
         </td>
         <td class="td-actions">
           <form method="post" action="/alerts/maintenance/{{ w.id }}/delete" style="display:inline;">{{ csrf_field() }}
@@ -57,6 +55,26 @@
     </tbody>
   </table>
 </div>
+{% endmacro %}
+
+{% if windows %}
+{% if active %}
+<h2 class="section-title" style="margin-bottom:0.75rem;">Active now</h2>
+{{ window_table(active, 'active') }}
+{% endif %}
+{% if upcoming %}
+<h2 class="section-title" style="margin-bottom:0.75rem;">Upcoming</h2>
+{{ window_table(upcoming, 'upcoming') }}
+{% endif %}
+{% if not active and not upcoming %}
+<div class="card" style="margin-bottom:1.5rem;">{{ empty_state("No window is active or scheduled. Every alert notifies as usual.") }}</div>
+{% endif %}
+{% if expired %}
+<details>
+  <summary class="section-title" style="cursor:pointer;margin-bottom:0.75rem;">Expired ({{ expired|length }})</summary>
+  {{ window_table(expired, 'expired') }}
+</details>
+{% endif %}
 {% else %}
 <div class="card">{{ empty_state("No maintenance windows yet.", "New window", "/alerts/maintenance/new") }}</div>
 {% endif %}

steward/templates/ansible/schedules.html

@@ -1,5 +1,5 @@
 {% extends "base.html" %}
-{% from "_macros.html" import crumbs %}
+{% from "_macros.html" import crumbs, status_badge %}
 {% block title %}Ansible Schedules — Steward{% endblock %}
 {% block breadcrumb %}{{ crumbs([("Ansible", "/ansible/"), ("Schedules", "")]) }}{% endblock %}
 
@@ -39,9 +39,8 @@
     <td style="font-size:0.85rem;">{{ fmt_interval(s.interval_seconds) }}</td>
     <td style="font-size:0.82rem;">
       {% if row.last_run %}
-        {% set sc = {"queued":"var(--text-dim)","running":"var(--yellow)","success":"var(--green)","failed":"var(--red)","interrupted":"var(--orange)","cancelled":"var(--orange)"}.get(row.last_run.status.value, "var(--text-muted)") %}
-        <a href="/ansible/runs/{{ row.last_run.id }}" style="color:{{ sc }};font-weight:500;">{{ row.last_run.status.value }}</a>
-        <span style="color:var(--text-dim);">· {{ row.last_run.started_at.strftime("%m-%d %H:%M") }}</span>
+        <a href="/ansible/runs/{{ row.last_run.id }}" style="text-decoration:none;">{{ status_badge(row.last_run.status) }}</a>
+        <span style="color:var(--text-dim);">· {{ row.last_run.started_at|ago }}</span>
       {% endif %}
       {% if s.last_error %}
         {# With a run beside it, the last firing was skipped because that run was

steward/templates/hosts/list.html

@@ -1,4 +1,8 @@
 {% extends "base.html" %}
+{% from "_macros.html" import status_badge, status_dot %}
+{# Hosts, problems first (host_svc.host_rank): a down monitor, then an agent
+   gone quiet, then monitors not yet checked, then the healthy. #}
+{% macro pct(v) %}{% if v is none %}—{% else %}{{ '%.0f'|format(v) }}%{% endif %}{% endmacro %}
 {% block title %}Hosts — Steward{% endblock %}
 {% block content %}
 <div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1.5rem;">
@@ -38,13 +42,7 @@
                 <td style="text-align:center;">
                     {% if not hs %}
                         <span style="color:var(--text-dim);font-size:0.8rem;" title="No monitors">—</span>
-                    {% elif hs.state == 'up' %}
-                        <span class="dot dot-up" title="Up"></span>
-                    {% elif hs.state == 'down' %}
-                        <span class="dot dot-down" title="Down"></span>
-                    {% else %}
-                        <span class="dot dot-dim" title="Pending"></span>
-                    {% endif %}
+                    {% else %}{{ status_dot(hs.state) }}{% endif %}
                 </td>
                 <td style="text-align:center;font-size:0.9rem;">
                     {% if hs and hs.latency_ms is not none %}
@@ -66,14 +64,13 @@
                     {% endif %}
                 </td>
                 {% endfor %}
-                {% set am = agent_metrics.get(host.name) %}
+                {% set am = agent.get(host.name) %}
                 <td style="text-align:center;font-size:0.8rem;font-family:ui-monospace,monospace;">
                     {% if am %}
-                        <span style="color:var(--text-muted);" title="CPU / memory">
-                            {{ '%.0f'|format(am.get('cpu_pct', 0)) }}% / {{ '%.0f'|format(am.get('mem_used_pct', 0)) }}%
-                        </span>
+                        <span style="color:var(--text-muted);" title="CPU / memory, latest">{{ pct(am.cpu_pct) }} / {{ pct(am.mem_used_pct) }}</span>
+                        {% if not am.fresh %}<div>{{ status_badge('stale', title="No agent report since " ~ (am.last_at.strftime('%Y-%m-%d %H:%M UTC') if am.last_at else 'it started')) }} <span class="tone-dim" style="font-family:inherit;">{{ am.last_at|ago }}</span></div>{% endif %}
                     {% else %}
-                        <span style="color:var(--text-dim);">—</span>
+                        <span style="color:var(--text-dim);" title="No agent on this host">—</span>
                     {% endif %}
                 </td>
                 <td class="td-actions">

steward/templates/monitors/rows.html

@@ -22,6 +22,11 @@
       <a class="ping-name" href="/monitors/{{ m.id }}">{{ m.name }}</a>
     </div>
     <span class="ping-addr" title="{{ m.target }}">{{ m.type|monitor_type }} · {{ m.target }}</span>
+    {% if d.state == 'down' %}
+    <span class="tone-crit" style="font-size:0.75rem;overflow-wrap:anywhere;">
+      Down{% if d.down_since %} {{ d.down_since|ago|replace(' ago', '') }}{% endif %}{% if d.down_reason %} · {{ d.down_reason }}{% endif %}
+    </span>
+    {% endif %}
   </div>
 
   <div class="ping-pills">