feat(alerts): a sentence per alert, a link to what it watches, a count in the sidebar (#5124)

The Alerts page's active list now says each alert in one line around the
resource it watches, e.g. "cpu_pct on web-1 is 94% (> 90%), firing for 12m",
with web-1 linking to its host, monitor or container page (or the plugin's
page for Traefik, UniFi and SNMP). Values take their unit from the metric
name through formatting.metric_value. A silent series reads "No cpu_pct from
web-1 for 20m"; an up/down metric says the word.

Actions follow the routes' roles: acknowledge, edit, toggle and New rule for
operators and admins, Delete for admins only, nothing for viewers.

The sidebar shows how many alerts need attention beside Alerts, from an
async context processor that skips htmx fragments, logged-out pages and the
database-down page, and renders without the count if the query fails.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/routes.py

@@ -96,7 +96,7 @@ def _parse_ansible_action(form) -> tuple[dict | None, str | None]:
 @require_role(UserRole.viewer)
 async def list_alerts():
     async with current_app.db_sessionmaker() as db:
-        active = await alert_svc.active_alerts(db)
+        active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
         rules = await alert_svc.list_rules(db)
 
     return await render_template("alerts/list.html", active=active, rules=rules, operators=_OPERATORS)

steward/alerts/service.py

@@ -85,6 +85,101 @@ def alert_label(state: AlertState) -> str:
     return state.state.value
 
 
+async def latest_value(db, rule: AlertRule) -> float | None:
+    """The newest sample of the series a rule watches, or None before the first."""
+    return (await db.execute(
+        select(PluginMetric.value)
+        .where(PluginMetric.source_module == rule.source_module,
+               PluginMetric.resource_name == rule.resource_name,
+               PluginMetric.metric_name == rule.metric_name)
+        .order_by(PluginMetric.recorded_at.desc()).limit(1)
+    )).scalar_one_or_none()
+
+
+# Sources whose resources have no page of their own: the plugin's page lists them.
+PLUGIN_PAGES = {"traefik": "/plugins/traefik/", "unifi": "/plugins/unifi/",
+                "snmp": "/plugins/snmp/"}
+
+
+async def resource_url(db, rule: AlertRule) -> str | None:
+    """Where the thing a rule watches is shown, or None when nothing shows it.
+
+    host_agent resources are the host's name, or "<host>:<mount|nic|core>";
+    docker's are "<host>/<container>"; a monitor's is its name under its type.
+    """
+    from steward.models.hosts import Host
+    source, resource = rule.source_module, rule.resource_name
+    if source == "host_agent":
+        name = resource.split(":", 1)[0]
+        host_id = (await db.execute(
+            select(Host.id).where(Host.name == name).limit(1))).scalar_one_or_none()
+        return f"/hosts/{host_id}" if host_id else None
+    if source in MONITOR_TYPE_VALUES:
+        monitor_id = (await db.execute(
+            select(Monitor.id).where(Monitor.type == source, Monitor.name == resource).limit(1)
+        )).scalar_one_or_none()
+        return f"/monitors/{monitor_id}" if monitor_id else None
+    if source == "docker" and "/" in resource:
+        host_name, container = resource.split("/", 1)
+        host_id = (await db.execute(
+            select(Host.id).where(Host.name == host_name).limit(1))).scalar_one_or_none()
+        return f"/plugins/docker/container/{host_id}/{container}" if host_id else None
+    return PLUGIN_PAGES.get(source)
+
+
+# Metrics whose value is a word: the sentence says the word, not the condition.
+_BINARY = ("is_up", "is_healthy")
+
+
+def alert_sentence(rule: AlertRule, label: str, value: float | None,
+             since: datetime | None, now: datetime) -> dict:
+    """An alert as one sentence around its resource: {"lead", "resource",
+    "tail"}, read lead + resource + tail, e.g. "cpu_pct on " + "web-1" +
+    " is 94% (> 90%), firing for 12m". label is alert_label(state); since is
+    when that label began (no_data_since for no_data)."""
+    from steward.core.formatting import human_duration, metric_value
+    metric = rule.metric_name
+    held = human_duration((now - since).total_seconds()) if since else None
+    if label == "no_data":
+        return {"lead": f"No {metric} from ", "resource": rule.resource_name,
+                "tail": f" for {held}" if held else ""}
+    for_how_long = f", {label} for {held}" if held else f", {label}"
+    if value is None:
+        reading = " has no reading yet"
+    elif metric in _BINARY:
+        return {"lead": "", "resource": rule.resource_name,
+                "tail": f" is {metric_value(metric, value)}{for_how_long}"}
+    else:
+        reading = f" is {metric_value(metric, value)}"
+    condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})"
+    return {"lead": f"{metric} on ", "resource": rule.resource_name,
+            "tail": f"{reading}{condition}{for_how_long}"}
+
+
+async def active_rows(db, now: datetime) -> list[dict]:
+    """The alerts page's active list: each alert's state, rule, sentence and
+    the link to what it watches."""
+    rows = []
+    for state, rule in await active_alerts(db):
+        label = alert_label(state)
+        since = state.no_data_since if label == "no_data" else state.last_transition_at
+        rows.append({
+            "state": state, "rule": rule, "label": label,
+            "says": alert_sentence(rule, label, await latest_value(db, rule), since, now),
+            "url": await resource_url(db, rule),
+        })
+    return rows
+
+
+async def active_count(db) -> int:
+    """How many alerts need attention: the sidebar's number."""
+    from sqlalchemy import func
+    return (await db.execute(
+        select(func.count()).select_from(AlertState)
+        .where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
+    )).scalar_one()
+
+
 async def list_rules(db) -> list[AlertRule]:
     return list((await db.execute(select(AlertRule).order_by(AlertRule.name))).scalars())
 

steward/app.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import asyncio
 import logging
 from pathlib import Path
-from quart import Quart, render_template, request
+from quart import Quart, g, render_template, request
 from .config import load_bootstrap
 from .database import (
     init_db, ensure_database_reachable, DatabaseUnavailable,
@@ -269,6 +269,23 @@ def create_app(
             "undecryptable_secrets": get_undecryptable_secrets(),
         }
 
+    @app.context_processor
+    async def _inject_alert_count():
+        """The sidebar's count of alerts needing attention. Full pages only:
+        an htmx fragment has no sidebar, and a logged-out page has no nav."""
+        from quart import session
+        if (request.headers.get("HX-Request") or not session.get("user_id")
+                or g.get("database_down")):
+            return {}
+        try:
+            from .alerts.service import active_count
+            async with app.db_sessionmaker() as db:
+                return {"active_alert_count": await active_count(db)}
+        except Exception:
+            # A count is decoration; the page renders without it.
+            logging.getLogger(__name__).exception("Active alert count failed")
+            return {}
+
     # ── 11. Share-token middleware ─────────────────────────────────────────────
     @app.before_request
     async def _validate_share_token():
@@ -312,6 +329,7 @@ def create_app(
             except DatabaseUnavailable as exc:
                 logger.error("Database unreachable while serving %s: %s",
                              request.path, exc)
+                g.database_down = True
                 return await render_template(
                     "errors/database_unavailable.html"), 503
             return None

steward/core/formatting.py

@@ -149,3 +149,30 @@ def worst_reading(readings, thresholds: dict):
         if _SEVERITY.get(grade, 0) > rank:
             worst, rank = (label, value, grade), _SEVERITY[grade]
     return worst
+
+
+def metric_value(name: str, value: Any) -> str:
+    """A metric reading in its unit, from the name's suffix: _pct → 94%,
+    _ms → 120 ms, _bytes → 1.2 GiB, _bps → its rate, _secs / seconds_ → a
+    span, _c → 71°C, _days → 12 days; is_up / is_healthy read as yes or no.
+    None (no reading) is "—"."""
+    if value is None:
+        return "—"
+    n = name or ""
+    if n in ("is_up", "is_healthy"):
+        return {"is_up": ("up", "down"), "is_healthy": ("healthy", "unhealthy")}[n][0 if value >= 0.5 else 1]
+    if n.endswith("_pct") or "_pct_" in n:
+        return f"{value:.1f}%" if abs(value) < 10 else f"{value:.0f}%"
+    if n.endswith("_ms"):
+        return f"{value:.0f} ms"
+    if n.endswith("_bytes"):
+        return human_bytes(value)
+    if n.endswith(("_bps", "bytes_rate")):
+        return human_rate(value)
+    if n.endswith("_secs") or n.startswith("seconds_"):
+        return human_duration(value)
+    if n.endswith("_c"):
+        return f"{value:.0f}°C"
+    if n.endswith("_days"):
+        return f"{value:.0f} days"
+    return f"{value:g}"

steward/static/steward.css

@@ -86,6 +86,12 @@ code { font-family: var(--fs-font-mono); font-size: 0.875em; background: var(--b
   color: var(--gold); border-left-color: var(--gold);
   background: color-mix(in srgb, var(--gold) 8%, transparent);
 }
+/* How many alerts need attention, beside the Alerts link. */
+.nav-count {
+  margin-left: auto; padding: 0 0.45em; border-radius: var(--fs-radius-sm);
+  background: var(--red-dim); color: var(--red); font-size: 0.75rem; font-weight: 600;
+  font-variant-numeric: tabular-nums;
+}
 .side-user { border-top: 1px solid var(--border); padding: 0.7rem 1.25rem; }
 .side-user form { margin: 0; }
 .side-user button {

steward/templates/alerts/list.html

@@ -2,10 +2,11 @@
 {% block title %}Alerts — Steward{% endblock %}
 {% from "_macros.html" import page_header, status_badge, empty_state %}
 {% block content %}
-{% call page_header("Alerts") %}
+{% set can_edit = session.user_role in ('operator', 'admin') %}
+{% call page_header("Alerts", "What is breaching now, in a sentence each, then every rule.") %}
     <a class="btn btn-ghost" href="/alerts/history">History</a>
     <a class="btn btn-ghost" href="/alerts/maintenance">Maintenance</a>
-    <a class="btn" href="/alerts/rules/new">New rule</a>
+    {% if can_edit %}<a class="btn" href="/alerts/rules/new">New rule</a>{% endif %}
 {% endcall %}
 
 {% if active %}
@@ -15,30 +16,29 @@
         <thead>
             <tr>
                 <th>State</th>
+                <th>What is wrong</th>
                 <th>Rule</th>
-                <th>Resource</th>
-                <th>Metric</th>
                 <th></th>
             </tr>
         </thead>
         <tbody>
-            {% for state, rule in active %}
+            {% for row in active %}
             <tr>
                 <td>
-                    {{ status_badge(state.state) }}
-                    {% if state.no_data_since %}
-                    {{ status_badge('no_data', title="No sample since " ~ (state.last_sample_at.strftime('%Y-%m-%d %H:%M UTC') if state.last_sample_at else 'the rule started watching')) }}
-                    <div style="font-size:0.72rem;color:var(--text-dim);margin-top:0.2rem;">
-                      since {{ state.no_data_since|ago }}
-                    </div>
+                    {{ status_badge(row.label, title=("No sample since " ~ (row.state.last_sample_at.strftime('%Y-%m-%d %H:%M UTC') if row.state.last_sample_at else 'the rule started watching')) if row.label == 'no_data' else none) }}
+                    {% if row.state.no_data_since and row.label != 'no_data' %}
+                    {{ status_badge('no_data', title="Silent since " ~ row.state.no_data_since.strftime('%Y-%m-%d %H:%M UTC')) }}
                     {% endif %}
                 </td>
-                <td>{{ rule.name }}</td>
-                <td style="color:var(--text-muted);">{{ rule.resource_name }}</td>
-                <td style="color:var(--text-muted);">{{ rule.metric_name }} {{ rule.operator.value }} {{ rule.threshold }}</td>
+                <td>
+                    {{- row.says.lead -}}
+                    {%- if row.url %}<a href="{{ row.url }}">{{ row.says.resource }}</a>{% else %}<strong>{{ row.says.resource }}</strong>{% endif -%}
+                    {{- row.says.tail -}}
+                </td>
+                <td style="color:var(--text-muted);"><a href="/alerts/history?rule={{ row.rule.id }}">{{ row.rule.name }}</a></td>
                 <td class="td-actions">
-                    {% if state.state.value == 'firing' %}
-                    <form method="post" action="/alerts/{{ rule.id }}/acknowledge" style="display:inline;">{{ csrf_field() }}
+                    {% if can_edit and row.state.state.value == 'firing' %}
+                    <form method="post" action="/alerts/{{ row.rule.id }}/acknowledge" style="display:inline;">{{ csrf_field() }}
                         <button type="submit" class="btn btn-sm btn-warn">Acknowledge</button>
                     </form>
                     {% endif %}
@@ -82,16 +82,20 @@
                 <td style="color:var(--text-muted);text-align:center;">{{ rule.consecutive_failures_required }} / {{ rule.consecutive_ok_required }}</td>
                 <td class="td-actions">
                     <a href="/alerts/history?rule={{ rule.id }}" class="btn btn-sm btn-ghost">History</a>
+                    {% if can_edit %}
                     <a href="/alerts/rules/{{ rule.id }}/edit" class="btn btn-sm btn-ghost">Edit</a>
                     <form method="post" action="/alerts/rules/{{ rule.id }}/toggle" style="display:inline;">{{ csrf_field() }}
                         <button type="submit" class="btn btn-sm btn-ghost">
                           {% if rule.enabled %}Disable{% else %}Enable{% endif %}
                         </button>
                     </form>
+                    {% endif %}
+                    {% if session.user_role == 'admin' %}
                     <form method="post" action="/alerts/rules/{{ rule.id }}/delete" style="display:inline;">{{ csrf_field() }}
                         <button type="submit" class="btn btn-sm btn-danger"
                             data-confirm="Delete rule {{ rule.name }}?">Delete</button>
                     </form>
+                    {% endif %}
                 </td>
             </tr>
             {% endfor %}
@@ -99,6 +103,6 @@
     </table>
 </div>
 {% else %}
-<div class="card">{{ empty_state("No alert rules yet.", "New rule", "/alerts/rules/new") }}</div>
+<div class="card">{{ empty_state("No alert rules yet.", "New rule" if can_edit else none, "/alerts/rules/new" if can_edit else none) }}</div>
 {% endif %}
 {% endblock %}

steward/templates/base.html

@@ -62,7 +62,7 @@
       <div class="nav-group">
         <div class="nav-group-label">Monitoring</div>
         <a href="/monitors/" {% if _p.startswith('/monitors') %}class="active" aria-current="page"{% endif %}>Monitors</a>
-        <a href="/alerts/" {% if _p.startswith('/alerts') %}class="active" aria-current="page"{% endif %}>Alerts</a>
+        <a href="/alerts/" {% if _p.startswith('/alerts') %}class="active" aria-current="page"{% endif %}>Alerts{% if active_alert_count %} <span class="nav-count" title="{{ active_alert_count }} alert{{ 's' if active_alert_count != 1 }} need{{ 's' if active_alert_count == 1 }} attention">{{ active_alert_count }}</span>{% endif %}</a>
       </div>
       <div class="nav-group">
         <div class="nav-group-label">Automation</div>