feat(status): what is wrong first — down monitors and active alerts, and since when (#5120)

#5120 "Status page: what is wrong first".

- A "Needs attention" card heads the Status board: each down monitor with
  how long it has been down, and each active alert (firing, acknowledged,
  pending, no data) with its condition and how long, linking to its history.
- StatusEntry.since / since_at_least: when the current up/down run began,
  from the recent checks (streak_start); "for at least" when every check we
  have agrees.
- Counts as stat tiles (Down, Active alerts, Pending, Up); names wrap
  instead of being cut off; dots carry their label.
- Uptime and TLS are coloured by status_class against Settings → Thresholds
  instead of literal cutoffs; heartbeat pills and DOWN/UP use tokens and
  status_badge rather than hex and bold.
- tests/core/test_status_page.py: streak_start cases, and the rows render
  with the problem card before the list, each since-time, threshold tones
  and no hex colour.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/status.py

@@ -38,6 +38,10 @@ class StatusEntry:
     spark_svg: str = ""             # filled by the route via sparkline_svg()
     tls_days: float | None = None   # days until TLS expiry (http only)
     detail_url: str | None = None
+    # When the current up/down state began, as far as the recent checks show;
+    # since_at_least means every check we have agrees, so it began earlier.
+    since: datetime | None = None
+    since_at_least: bool = False
 
 
 StatusSource = Callable[[object], Awaitable[list[StatusEntry]]]
@@ -158,6 +162,17 @@ async def _uptime_by_monitor(db, monitor_ids: list[str]) -> dict[str, dict]:
     return out
 
 
+def streak_start(results) -> tuple[datetime | None, bool]:
+    """(checked_at of the first check in the latest run of the same result,
+    whether that run fills every check given). Results are oldest-first."""
+    if not results:
+        return None, False
+    i = len(results) - 1
+    while i > 0 and results[i - 1].is_up == results[-1].is_up:
+        i -= 1
+    return results[i].checked_at, i == 0
+
+
 def _heartbeat_title(mtype: str, r) -> str:
     """Type-aware tooltip for one heartbeat cell."""
     ts = f"{r.checked_at:%H:%M:%S} UTC"
@@ -191,6 +206,7 @@ async def monitor_status_source(db) -> list[StatusEntry]:
         rows = recent.get(m.id, [])
         latest = rows[-1] if rows else None
         status = "pending" if latest is None else ("up" if latest.is_up else "down")
+        since, since_at_least = streak_start(rows)
         tls_days = None
         if latest and latest.tls_expires_at:
             tls_days = (latest.tls_expires_at - now).total_seconds() / 86400.0
@@ -203,5 +219,6 @@ async def monitor_status_source(db) -> list[StatusEntry]:
             latency_ms=latest.response_ms if latest else None,
             spark=[r.response_ms for r in rows if r.response_ms is not None],
             tls_days=tls_days, detail_url="/monitors/",
+            since=since, since_at_least=since_at_least,
         ))
     return entries

steward/static/steward.css

@@ -228,6 +228,13 @@ textarea { resize: vertical; }
 .dot-crit { background: var(--red); }
 .dot-info { background: var(--blue); }
 .dot-label { margin-left: 0.35rem; font-size: 0.85rem; }
+/* Text in a tone: status_class() results ('ok' / 'warn' / 'crit' / 'unknown')
+   and the STATES tones. */
+.tone-ok { color: var(--green); }
+.tone-warn { color: var(--yellow); }
+.tone-crit { color: var(--red); }
+.tone-info { color: var(--blue); }
+.tone-dim, .tone-unknown { color: var(--text-dim); }
 .sr-only { position: absolute; width: 1px; height: 1px; padding: 0; margin: -1px; overflow: hidden; clip: rect(0, 0, 0, 0); white-space: nowrap; border: 0; }
 
 /* Empty state */
@@ -251,6 +258,13 @@ a.stat-tile:hover { border-color: var(--border-mid); background: var(--bg-elevat
 .stat-crit .stat-tile-value { color: var(--red); }
 .stat-info .stat-tile-value { color: var(--blue); }
 .stat-dim .stat-tile-value { color: var(--text-dim); }
+.stat-strip { display: grid; grid-template-columns: repeat(auto-fill, minmax(9rem, 1fr)); gap: 0.75rem; margin-bottom: 1rem; }
+
+/* "Needs attention": what is wrong, first on a page. */
+.problem-card { border-color: color-mix(in srgb, var(--fs-error) 45%, var(--border)); }
+.problem-list { list-style: none; display: flex; flex-direction: column; gap: 0.45rem; }
+.problem-list li { display: flex; flex-wrap: wrap; align-items: baseline; gap: 0.25rem 0.6rem; overflow-wrap: anywhere; }
+.problem-detail { color: var(--text-muted); font-size: 0.82rem; }
 
 /* Breadcrumbs */
 /* Breadcrumb renders as a kicker line tucked directly above the page header
@@ -307,6 +321,8 @@ body.dash-editing .widget-drawer { transform:translateY(0); }
 .pill { display:inline-block; width:7px; height:22px; border-radius:2px; cursor:default; transition:opacity .1s; }
 .pill:hover { opacity:.75; }
 .pill-empty { background:var(--bg-elevated); }
+.pill-up { background:var(--green); }
+.pill-down { background:var(--red); }
 .ping-cur { min-width:72px; text-align:right; font-size:0.85rem; font-variant-numeric:tabular-nums; }
 .ping-cur-good { color:var(--green); }
 .ping-cur-warn { color:var(--yellow); }

steward/status/routes.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 from quart import Blueprint, current_app, render_template, request
 from sqlalchemy import func, select
 
+from steward.alerts.service import active_alerts
 from steward.auth.middleware import require_role
 from steward.core.status import collect_status, sparkline_svg
 from steward.models.hosts import Host
@@ -27,15 +28,18 @@ async def index():
 @status_bp.get("/rows")
 @require_role(UserRole.viewer)
 async def rows():
-    """HTMX fragment — the full status list + summary, refreshed on poll."""
+    """HTMX fragment — what is wrong (down monitors, active alerts), then the
+    full status list, refreshed on poll."""
     async with current_app.db_sessionmaker() as db:
         entries = await collect_status(db)
+        alerts = await active_alerts(db)
     for e in entries:
         if e.spark:
             e.spark_svg = sparkline_svg(e.spark)
     up, down, pending = _summarize(entries)
     return await render_template(
-        "status/rows.html", entries=entries, up=up, down=down, pending=pending
+        "status/rows.html", entries=entries, up=up, down=down, pending=pending,
+        alerts=alerts, problems=[e for e in entries if e.status == "down"],
     )
 
 

steward/templates/status/rows.html

@@ -1,38 +1,52 @@
-{# status/rows.html — unified monitor list + summary, refreshed on poll #}
-{% macro uptime_cell(pct) %}
-{%- if pct is none -%}<span style="color:var(--text-dim);">—</span>
-{%- elif pct >= 99.9 -%}<span style="color:var(--green);">{{ "%.2f"|format(pct) }}%</span>
-{%- elif pct >= 99 -%}<span style="color:var(--yellow);">{{ "%.2f"|format(pct) }}%</span>
-{%- elif pct >= 95 -%}<span style="color:var(--orange);">{{ "%.2f"|format(pct) }}%</span>
-{%- else -%}<span style="color:var(--red);">{{ "%.2f"|format(pct) }}%</span>
-{%- endif -%}
-{% endmacro %}
+{# status/rows.html — what is wrong first (down monitors, active alerts), then
+   every monitor; refreshed on poll #}
+{% from "_macros.html" import status_badge, status_dot, stat_tile, empty_state %}
+{% macro uptime_cell(pct) -%}
+{%- if pct is none -%}<span class="tone-unknown">—</span>
+{%- else -%}<span class="tone-{{ status_class(pct, 'uptime') }}">{{ "%.2f"|format(pct) }}%</span>{%- endif -%}
+{%- endmacro %}
+{% macro since(e) -%}
+{%- if e.since -%}{{ "for at least " if e.since_at_least else "since " }}{{ e.since|ago }}{%- endif -%}
+{%- endmacro %}
 
-{# ── Summary ─────────────────────────────────────────────────────────────── #}
-<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(130px,1fr));gap:0.75rem;margin-bottom:1rem;">
-  <div class="card" style="margin-bottom:0;">
-    <div class="section-title" style="margin-bottom:0.3rem;">Up</div>
-    <span class="stat-val" style="color:var(--green);">{{ up }}</span>
-  </div>
-  <div class="card" style="margin-bottom:0;">
-    <div class="section-title" style="margin-bottom:0.3rem;">Down</div>
-    <span class="stat-val" style="color:{% if down %}var(--red){% else %}var(--text-dim){% endif %};">{{ down }}</span>
-  </div>
-  <div class="card" style="margin-bottom:0;">
-    <div class="section-title" style="margin-bottom:0.3rem;">Pending</div>
-    <span class="stat-val" style="color:{% if pending %}var(--yellow){% else %}var(--text-dim){% endif %};">{{ pending }}</span>
-  </div>
+{# ── What is wrong ───────────────────────────────────────────────────────── #}
+{% if problems or alerts %}
+<section class="card problem-card" aria-labelledby="status-problems">
+  <h2 id="status-problems" class="section-title" style="margin-bottom:0.6rem;">Needs attention</h2>
+  <ul class="problem-list">
+    {% for e in problems %}
+    <li>
+      {{ status_badge('down') }}
+      {% if e.detail_url %}<a href="{{ e.detail_url }}">{{ e.name }}</a>{% else %}<strong>{{ e.name }}</strong>{% endif %}
+      <span class="problem-detail">{{ e.kind|upper }} · {{ e.target }} · down {{ since(e) }}</span>
+    </li>
+    {% endfor %}
+    {% for state, rule in alerts %}
+    <li>
+      {{ status_badge('no_data' if state.no_data_since and state.state.value == 'inactive' else state.state) }}
+      <a href="/alerts/history?rule={{ rule.id }}">{{ rule.name }}</a>
+      <span class="problem-detail">{{ rule.resource_name }} · {{ rule.metric_name }} {{ rule.operator.value }} {{ rule.threshold }} · since {{ (state.no_data_since if state.state.value == 'inactive' else state.last_transition_at)|ago }}</span>
+    </li>
+    {% endfor %}
+  </ul>
+</section>
+{% endif %}
+
+{# ── Counts ──────────────────────────────────────────────────────────────── #}
+<div class="stat-strip">
+  {{ stat_tile("Down", down, state="down" if down else "inactive") }}
+  {{ stat_tile("Active alerts", alerts|length, state="firing" if alerts else "inactive", href="/alerts/") }}
+  {{ stat_tile("Pending", pending, state="pending" if pending else "inactive") }}
+  {{ stat_tile("Up", up, state="up") }}
 </div>
 
 {% if not entries %}
-<div class="card empty">
-  No monitors configured yet. Add <a href="/hosts/">hosts</a> (ping / DNS) or HTTP monitors to populate the board.
-</div>
+<div class="card">{{ empty_state("No monitors yet. Add a host (ping or DNS) or an HTTP monitor to fill the board.", "Add a monitor", "/monitors/") }}</div>
 {% else %}
 <div class="card-flush">
   {# header row #}
   <div style="display:flex;align-items:center;gap:1rem;padding:0.55rem 1rem;border-bottom:1px solid var(--border-mid);font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.04em;">
-    <span style="width:130px;flex-shrink:0;">Monitor</span>
+    <span style="width:11rem;flex-shrink:0;">Monitor</span>
     <span style="flex:1;min-width:0;">Recent checks</span>
     <span style="width:92px;text-align:right;flex-shrink:0;">Latest</span>
     <span style="width:54px;text-align:center;flex-shrink:0;">24h</span>
@@ -44,13 +58,13 @@
   {% for e in entries %}
   <div style="display:flex;align-items:center;gap:1rem;padding:0.55rem 1rem;border-bottom:1px solid var(--border);">
     {# name + kind #}
-    <div style="width:130px;flex-shrink:0;overflow:hidden;">
-      <div style="display:flex;align-items:center;gap:0.4rem;">
-        <span class="dot {% if e.status == 'up' %}dot-up{% elif e.status == 'down' %}dot-down{% else %}dot-dim{% endif %}"></span>
-        {% if e.detail_url %}<a href="{{ e.detail_url }}" style="font-size:0.85rem;font-weight:500;color:var(--text);white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ e.name }}</a>
-        {% else %}<span style="font-size:0.85rem;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ e.name }}</span>{% endif %}
+    <div style="width:11rem;flex-shrink:0;min-width:0;">
+      <div style="display:flex;align-items:baseline;gap:0.4rem;">
+        {{ status_dot(e.status) }}
+        {% if e.detail_url %}<a href="{{ e.detail_url }}" style="font-size:0.85rem;font-weight:500;color:var(--text);overflow-wrap:anywhere;">{{ e.name }}</a>
+        {% else %}<span style="font-size:0.85rem;font-weight:500;overflow-wrap:anywhere;">{{ e.name }}</span>{% endif %}
       </div>
-      <div style="color:var(--text-dim);font-size:0.7rem;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;padding-left:0.85rem;">
+      <div style="color:var(--text-dim);font-size:0.7rem;overflow-wrap:anywhere;padding-left:0.85rem;">
         <span style="text-transform:uppercase;letter-spacing:0.04em;">{{ e.kind }}</span> · {{ e.target }}
       </div>
     </div>
@@ -60,17 +74,17 @@
       {% set pad = 30 - e.heartbeat|length %}
       {% if pad > 0 %}{% for _ in range(pad) %}<span class="pill pill-empty" title="No data"></span>{% endfor %}{% endif %}
       {% for hb in e.heartbeat %}
-      <span class="pill" style="background:{% if hb.state == 'down' %}#6a1515{% else %}#1a6632{% endif %};" title="{{ hb.title }}"></span>
+      <span class="pill pill-{{ 'down' if hb.state == 'down' else 'up' }}" title="{{ hb.title }}"></span>
       {% endfor %}
     </div>
 
     {# latest latency + sparkline #}
     <div style="width:92px;text-align:right;flex-shrink:0;font-size:0.8rem;font-variant-numeric:tabular-nums;">
-      {% if e.status == 'down' %}<span style="color:var(--red);font-weight:bold;">DOWN</span>
+      {% if e.status == 'down' %}{{ status_badge('down') }}
       {% elif e.latency_ms is not none %}
         {% if e.spark_svg %}<span style="opacity:0.8;">{{ e.spark_svg|safe }}</span><br>{% endif %}
         <span style="color:var(--text-muted);">{{ "%.0f"|format(e.latency_ms) }} ms</span>
-      {% elif e.status == 'up' %}<span style="color:var(--green);">UP</span>
+      {% elif e.status == 'up' %}{{ status_badge('up') }}
       {% else %}<span style="color:var(--text-dim);">—</span>{% endif %}
     </div>
 
@@ -81,11 +95,9 @@
 
     {# TLS expiry countdown #}
     <span style="width:64px;text-align:right;flex-shrink:0;font-size:0.78rem;font-variant-numeric:tabular-nums;">
-      {% if e.tls_days is none %}<span style="color:var(--text-dim);">—</span>
-      {% elif e.tls_days < 0 %}<span style="color:var(--red);font-weight:bold;" title="Certificate expired">expired</span>
-      {% elif e.tls_days < 14 %}<span style="color:var(--red);" title="TLS certificate expiry">{{ e.tls_days|round|int }}d</span>
-      {% elif e.tls_days < 30 %}<span style="color:var(--yellow);" title="TLS certificate expiry">{{ e.tls_days|round|int }}d</span>
-      {% else %}<span style="color:var(--text-muted);" title="TLS certificate expiry">{{ e.tls_days|round|int }}d</span>{% endif %}
+      {% if e.tls_days is none %}<span class="tone-unknown">—</span>
+      {% elif e.tls_days < 0 %}<span class="tone-crit" title="Certificate expired">expired</span>
+      {% else %}<span class="tone-{{ status_class(e.tls_days, 'tls_days') }}" title="TLS certificate expires in {{ e.tls_days|round|int }} days">{{ e.tls_days|round|int }}d</span>{% endif %}
     </span>
   </div>
   {% endfor %}

tests/core/test_status_page.py

@@ -0,0 +1,70 @@
+"""The Status page says what is wrong, and since when, before anything else
+(#5120)."""
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+
+import pytest
+from quart import render_template
+
+from steward.core.status import StatusEntry, streak_start
+from tests.test_style_tokens import hex_colours
+
+T0 = datetime(2026, 10, 6, 18, 0, tzinfo=timezone.utc)
+
+
+def _checks(*ups):
+    return [SimpleNamespace(is_up=up, checked_at=T0 + timedelta(minutes=i)) for i, up in enumerate(ups)]
+
+
+@pytest.mark.parametrize("ups, since, at_least", [
+    ((), None, False),
+    ((True,), 0, True),
+    ((False, False, False), 0, True),         # down for every check we have
+    ((True, True, False, False), 2, False),   # went down at the third check
+    ((False, True, True, False), 3, False),
+])
+def test_streak_start(ups, since, at_least):
+    checks = _checks(*ups)
+    assert streak_start(checks) == ((checks[since].checked_at if since is not None else None), at_least)
+
+
+async def _rows(app, entries, alerts):
+    async with app.test_request_context("/status/rows"):
+        return await render_template(
+            "status/rows.html", entries=entries, alerts=alerts,
+            problems=[e for e in entries if e.status == "down"],
+            up=sum(e.status == "up" for e in entries), down=sum(e.status == "down" for e in entries), pending=0)
+
+
+async def test_the_problems_come_first_and_say_since_when(app):
+    now = datetime.now(timezone.utc)
+    entries = [
+        StatusEntry(kind="http", key="1", name="shop-api", target="https://shop/x", status="down",
+                    since=now - timedelta(minutes=12), uptime={"24h": 93.0}, tls_days=2.0),
+        StatusEntry(kind="icmp", key="2", name="nas", target="10.0.0.5", status="up",
+                    since=now - timedelta(days=3), since_at_least=True, uptime={"24h": 100.0}),
+    ]
+    state = SimpleNamespace(state=SimpleNamespace(value="firing"), no_data_since=None,
+                            last_transition_at=now - timedelta(minutes=4))
+    rule = SimpleNamespace(id="r1", name="CPU high", resource_name="web01", metric_name="cpu_pct",
+                           operator=SimpleNamespace(value=">"), threshold=90)
+    page = await _rows(app, entries, [(state, rule)])
+
+    attention = page.index("Needs attention")
+    assert attention < page.index("Recent checks")              # before the full list
+    problem_card = page[attention:page.index("</section>")]
+    assert "shop-api" in problem_card and "nas" not in problem_card
+    assert "down since <time" in problem_card and ">12m ago</time>" in problem_card
+    assert "CPU high" in problem_card and "cpu_pct &gt; 90" in problem_card and ">4m ago</time>" in problem_card
+    assert '<span class="badge badge-crit">Firing</span>' in problem_card
+    # Uptime and TLS coloured by Settings → Thresholds, not by literals.
+    assert '<span class="tone-crit">93.00%</span>' in page and '<span class="tone-ok">100.00%</span>' in page
+    assert 'class="tone-crit" title="TLS certificate expires in 2 days">2d' in page
+    assert not hex_colours(page), hex_colours(page)
+
+
+async def test_a_quiet_board_has_no_problem_card(app):
+    entries = [StatusEntry(kind="icmp", key="2", name="nas", target="10.0.0.5", status="up")]
+    page = await _rows(app, entries, [])
+    assert "Needs attention" not in page and "nas" in page
+    assert "No monitors yet" in await _rows(app, [], [])