From 57e1960dc1d9e2f15b0e0115d3587356b192b1c4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 9 Oct 2026 16:27:17 -0400 Subject: [PATCH] feat(dashboard): an Alerts widget beside the Overview; the Overview's overflow links to Alerts and Monitors (#5510) The Alerts widget (/alerts/widget) is shaped like every widget: a verdict ("2 firing, 1 silent." / "Nothing firing: all 12 rules are quiet."), then what is firing or silent with its reading now, what the rule fires on and since when, then what someone acknowledged and who; a line when alert email failed in the last 7 days or is retrying (outbox.alert_trouble), linking admins to Settings -> System; the quiet rules counted, a covering rule once. It sits beside the Overview in the recommended layout (6x4 each). The Overview's "+N more" line pointed at the removed /status/; alerts now link to /alerts/ and monitors to /monitors/. Co-Authored-By: Claude Opus 5.5 --- docs/core/alerting.md | 11 ++ steward/alerts/routes.py | 13 +++ steward/alerts/service.py | 17 ++- steward/core/outbox.py | 26 ++++- steward/core/widgets.py | 18 ++- steward/templates/alerts/widget.html | 58 ++++++++++ steward/templates/status/widget.html | 10 +- tests/core/test_alerts_widget.py | 104 ++++++++++++++++++ tests/core/test_dashboard_defaults.py | 31 +++++- .../integration/test_email_delivery_pages.py | 4 + 10 files changed, 279 insertions(+), 13 deletions(-) create mode 100644 steward/templates/alerts/widget.html create mode 100644 tests/core/test_alerts_widget.py diff --git a/docs/core/alerting.md b/docs/core/alerting.md index 77a718c..8efac52 100644 --- a/docs/core/alerting.md +++ b/docs/core/alerting.md @@ -232,6 +232,17 @@ host. Each channel's result shows under the button. Nothing fires, the rule's history doesn't change, and the test is recorded in the audit log as `alert_rule.tested`. +### On the dashboard + +The **Alerts** widget (`/alerts/widget`) sits beside the Overview in the +recommended layout. It opens with a verdict ("2 firing, 1 silent." or +"Nothing firing: all 12 rules are quiet."). Below that it lists what is +firing or silent: each rule, what it watches, its reading now against what +it fires on, and since when. Acknowledged alerts follow, with who +acknowledged them and when. A line appears when alert email has failed in +the last 7 days or is still retrying, linking admins to Settings → System. +The quiet rules are counted at the end. + --- ## Data Models diff --git a/steward/alerts/routes.py b/steward/alerts/routes.py index 760a009..4457968 100644 --- a/steward/alerts/routes.py +++ b/steward/alerts/routes.py @@ -178,6 +178,19 @@ async def active_fragment(): rules_off=sum(1 for r in rules if not r.enabled)) +@alerts_bp.get("/widget") +@require_role(UserRole.viewer) +async def widget(): + """The dashboard Alerts widget (#5510): a verdict, what is firing or + silent, then what someone has taken, a line when alert email is failing, + and the quiet rules counted.""" + async with current_app.db_sessionmaker() as db: + active = await alert_svc.active_rows(db, datetime.now(timezone.utc)) + view = alert_svc.widget_view(active, await alert_svc.list_rules(db)) + email = await outbox.alert_trouble(db) + return await render_template("alerts/widget.html", **view, email=email) + + def _delivery(label: str, sent: bool | None, error: str | None) -> dict | None: """A channel's outcome as recorded on the event; None when not tried.""" if sent is None: diff --git a/steward/alerts/service.py b/steward/alerts/service.py index 70f46ae..d98bb76 100644 --- a/steward/alerts/service.py +++ b/steward/alerts/service.py @@ -231,11 +231,13 @@ async def active_rows(db, now: datetime) -> list[dict]: since = started.get(rule.id, state.last_transition_at) else: since = state.last_transition_at + value = await latest_value(db, rule) rows.append({ "state": state, "rule": rule, "label": label, "since": since, "breach_since": started.get(rule.id) if label != "no_data" else None, "acked_by": names.get(state.acknowledged_by, state.acknowledged_by), - "says": alert_sentence(rule, label, await latest_value(db, rule), since, now), + "value": value, + "says": alert_sentence(rule, label, value, since, now), "url": await resource_url(db, rule), }) epoch = datetime.max.replace(tzinfo=timezone.utc) @@ -243,6 +245,19 @@ async def active_rows(db, now: datetime) -> list[dict]: return rows +def widget_view(active: list[dict], rules: list[AlertRule]) -> dict: + """The dashboard Alerts widget (#5510) from active_rows and every rule: + the alerts in that order, each with what its rule fires on, and how many + enabled rules, as people made them, are quiet (a covering rule counts + once, and is quiet only while every host under it is). + {"rows", "quiet", "total"}.""" + busy = {r["rule"].id for r in active} + busy |= {r["rule"].parent_id for r in active if r["rule"].parent_id} + made = [r for r in rules if r.enabled and not r.parent_id] + return {"rows": [{**r, "condition": rule_condition(r["rule"])} for r in active], + "quiet": sum(1 for r in made if r.id not in busy), "total": len(made)} + + def watches_host(rule: AlertRule, host_name: str, monitor_keys: set) -> bool: """Whether a rule watches something on this host: its agent series ("" or ":"), one of its containers ("/"), diff --git a/steward/core/outbox.py b/steward/core/outbox.py index 44b2646..ca87223 100644 --- a/steward/core/outbox.py +++ b/steward/core/outbox.py @@ -35,7 +35,7 @@ from datetime import datetime, timedelta, timezone from email.message import EmailMessage from typing import TYPE_CHECKING, Callable -from sqlalchemy import insert, or_, select, text, update +from sqlalchemy import and_, insert, or_, select, text, update from steward.models.outbox import EmailOutbox, email_outbox_events @@ -434,3 +434,27 @@ async def outgoing(db, now: datetime | None = None, limit: int = 20) -> dict: shown = sum(counts.get(s, 0) for s in open_states) return {"rows": rows, "more": max(shown - len(rows), 0), "sent": counts.get("sent", 0), "problems": problems} + + +async def alert_trouble(db, now: datetime | None = None) -> dict: + """The dashboard Alerts widget's delivery line (#5510): alert emails of + the last SHOWN_DAYS that failed, and those still retrying (tried, not yet + accepted), and the line that says so ("" when neither). + {"failed": n, "retrying": n, "said": str}.""" + from sqlalchemy import func + now = now or datetime.now(timezone.utc) + rows = dict((await db.execute( + select(EmailOutbox.state, func.count()) + .where(EmailOutbox.kind == "alert", EmailOutbox.created_at >= now - timedelta(days=SHOWN_DAYS), + or_(EmailOutbox.state == "failed", + and_(EmailOutbox.state.in_(("pending", "sending")), EmailOutbox.attempts > 0))) + .group_by(EmailOutbox.state))).all()) + failed = rows.get("failed", 0) + retrying = rows.get("pending", 0) + rows.get("sending", 0) + said = [] + if failed: + said.append(f"{failed} alert email{'s' if failed != 1 else ''} failed in the last {SHOWN_DAYS} days") + if retrying: + said.append(f"{retrying} still retrying" if failed + else f"{retrying} alert email{'s' if retrying != 1 else ''} still retrying") + return {"failed": failed, "retrying": retrying, "said": "; ".join(said)} diff --git a/steward/core/widgets.py b/steward/core/widgets.py index 6b0fbb4..d2cf2a5 100644 --- a/steward/core/widgets.py +++ b/steward/core/widgets.py @@ -70,6 +70,16 @@ WIDGET_REGISTRY: dict[str, dict] = { "poll": True, "params": [], }, + "alerts": { + "key": "alerts", + "label": "Alerts", + "description": "What is firing or silent, its reading and since when, then what someone has acknowledged; says when alert email is failing", + "hx_url": "/alerts/widget", + "detail_url": "/alerts/", + "plugin": None, + "poll": True, + "params": [], + }, "hosts_overview": { "key": "hosts_overview", "label": "Hosts — Overview", @@ -305,13 +315,15 @@ RETIRED = { "uptime_summary": "monitors", # Monitors shows 24h/7d/30d uptime per monitor (#5250) } -# The recommended dashboard (#5213): what needs attention first, then the +# The recommended dashboard (#5213): what needs attention first (the +# Overview, and beside it the Alerts, #5510), then the # fleet, then each integration. (key, width, height, config) on the 12-column # grid, one cell 70px tall; a widget whose plugin is off is skipped and the # rest close up. Seeded into the first dashboard and offered by # "Reset to the recommended layout" in edit mode. RECOMMENDED: list[tuple[str, int, int, dict]] = [ - ("status_overview", 12, 3, {}), + ("status_overview", 6, 4, {}), + ("alerts", 6, 4, {}), ("hosts_overview", 6, 7, {}), ("monitors", 6, 7, {"show_down_only": "no", "limit": 10}), ("docker_containers", 6, 6, {}), @@ -342,7 +354,7 @@ def recommended_layout(plugins_cfg: dict) -> list[dict]: # Group each widget for the picker (Core monitors / Monitoring capabilities / # Integrations), mirroring the Settings → Plugins taxonomy. Capability plugins # are host/monitoring facets; traefik/unifi are discrete vendor integrations. -_CORE_WIDGETS = {"monitors", "status_overview", "hosts_overview"} +_CORE_WIDGETS = {"monitors", "status_overview", "alerts", "hosts_overview"} _INTEGRATION_PLUGINS = {"traefik", "unifi"} GROUP_LABELS = { "core": "Core monitors", diff --git a/steward/templates/alerts/widget.html b/steward/templates/alerts/widget.html new file mode 100644 index 0000000..debafb6 --- /dev/null +++ b/steward/templates/alerts/widget.html @@ -0,0 +1,58 @@ +{# alerts/widget.html — the dashboard Alerts widget (#5510), in the widget + shape (snippet #5259): a verdict; what is firing or silent, its reading + now against what the rule fires on, and since when; then what someone has + acknowledged, by whom and when; a line when alert email is failing or + retrying; the quiet rules counted. Rows come from alert_svc.active_rows, + so they read in the Alerts page's order. #} +{% from "_macros.html" import status_dot %} +{% set firing = rows|selectattr("label", "equalto", "firing")|list|length %} +{% if not total and not rows %} +
+ No alert rules yet.{% if session.user_role in ('operator', 'admin') %} Add a rule →{% endif %} +
+{% else %} +
+ {%- if firing %}{{ firing }} firing{% else %}Nothing firing{% endif %} + {%- for key, word in (("no_data", "silent"), ("acknowledged", "acknowledged"), ("pending", "pending")) %} + {%- set n = rows|selectattr("label", "equalto", key)|list|length %} + {%- if n %}, {{ n }} {{ word }}{% endif %} + {%- endfor %} + {%- if not rows %}: {{ "the rule is" if total == 1 else "all " ~ total ~ " rules are" }} quiet{% endif %}. +
+{% if rows %} + + + + + {% for r in rows %} + {% set rule = r.rule %} + + + + + + {% endfor %} + +
AlertNowSince
+ {{- status_dot(r.label) -}} + {% if session.user_id %}{{ rule.name }}{% else %}{{ rule.name }}{% endif %} + {%- if r.label == "acknowledged" %} + {{ rule.resource_name }} · acknowledged{% if r.acked_by %} by {{ r.acked_by }}{% endif %}{% if r.state.acknowledged_at %} {{ r.state.acknowledged_at|ago }}{% endif %} + {%- elif r.label == "no_data" %} + {{ rule.resource_name }} · no reading + {%- else %} + {{ rule.resource_name }} · fires on {{ r.condition }} + {%- endif %} + + {%- if r.label == "no_data" %}—{% else %}{{ metric_value(rule.metric_name, r.value) }}{% endif -%} + {% if r.since %}{{ r.since|ago }}{% else %}—{% endif %}
+{% endif %} +{% if email.said %} +
+ {{- email.said }}{% if session.user_role == "admin" %} — Outgoing email{% endif %} +
+{% endif %} +{% if rows and quiet %} +
+{{ quiet }} rule{{ "s" if quiet != 1 }} quiet{% if session.user_id %} — Alerts{% endif %}
+{% endif %} +{% endif %} diff --git a/steward/templates/status/widget.html b/steward/templates/status/widget.html index dc60b95..4b0bba6 100644 --- a/steward/templates/status/widget.html +++ b/steward/templates/status/widget.html @@ -54,11 +54,13 @@ {% if e.since %}{% if e.since_at_least %}over {% endif %}{{ e.since|ago }}{% endif %} {% endfor %} - {% if alerts|length > 6 or problems|length > 8 %} + {# What didn't fit, each linking to its own list (#5510). #} + {% set more_alerts, more_monitors = alerts|length - 6, problems|length - 8 %} + {% if more_alerts > 0 or more_monitors > 0 %}
- {% if alerts|length > 6 %}+{{ alerts|length - 6 }} more alerts{% endif %} - {% if problems|length > 8 %}{% if alerts|length > 6 %} · {% endif %}+{{ problems|length - 8 }} more monitors{% endif %} - {% if session.user_id %} — Status{% endif %} + {%- if more_alerts > 0 %}{% if session.user_id %}+{{ more_alerts }} more alerts{% else %}+{{ more_alerts }} more alerts{% endif %}{% endif %} + {%- if more_alerts > 0 and more_monitors > 0 %} · {% endif %} + {%- if more_monitors > 0 %}{% if session.user_id %}+{{ more_monitors }} more monitors{% else %}+{{ more_monitors }} more monitors{% endif %}{% endif -%}
{% endif %} diff --git a/tests/core/test_alerts_widget.py b/tests/core/test_alerts_widget.py new file mode 100644 index 0000000..dff416d --- /dev/null +++ b/tests/core/test_alerts_widget.py @@ -0,0 +1,104 @@ +"""The dashboard Alerts widget (#5510): a verdict, what is firing or silent +with its reading and since when, then what someone has acknowledged, a line +when alert email is failing, and the quiet rules counted.""" +import re +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +from quart import render_template, session + +from steward.alerts.service import widget_view +from steward.core.widgets import WIDGET_REGISTRY +from steward.models.alerts import AlertOperator +from tests.test_style_tokens import hex_colours + +NOW = datetime.now(timezone.utc) +QUIET_EMAIL = {"failed": 0, "retrying": 0, "said": ""} + + +def _rule(rid, name, resource="web-1", metric="cpu_pct", op=">", threshold=90.0, + parent_id=None, enabled=True): + return SimpleNamespace(id=rid, name=name, resource_name=resource, metric_name=metric, + operator=AlertOperator(op), threshold=threshold, + parent_id=parent_id, enabled=enabled) + + +def _row(rule, label, value=None, since=None, acked_by=None, acked_at=None): + return {"rule": rule, "label": label, "value": value, "since": since, "acked_by": acked_by, + "state": SimpleNamespace(acknowledged_at=acked_at)} + + +async def _render(app, view, email=QUIET_EMAIL, role="admin"): + async with app.test_request_context("/alerts/widget"): + if role: + session["user_id"] = "u1" + session["user_role"] = role + return await render_template("alerts/widget.html", **view, email=email) + + +def test_the_widget_is_registered_and_points_at_alerts(): + w = WIDGET_REGISTRY["alerts"] + assert w["hx_url"] == "/alerts/widget" and w["detail_url"] == "/alerts/" and w["plugin"] is None + + +def test_a_covering_rule_counts_once_and_is_quiet_only_when_all_its_hosts_are(): + cover = _rule("c", "disk on every host", resource="*") + kid_a = _rule("c-a", "disk on every host", resource="nas", parent_id="c") + kid_b = _rule("c-b", "disk on every host", resource="web-1", parent_id="c") + lone = _rule("l", "web up", metric="is_up", op="<", threshold=0.5) + off = _rule("o", "old rule", enabled=False) + view = widget_view([_row(kid_a, "firing", 91.0, NOW)], [cover, kid_a, kid_b, lone, off]) + assert view["total"] == 2 and view["quiet"] == 1 # "web up"; the covering rule is busy + assert view["rows"][0]["condition"] == "CPU > 90%" + assert widget_view([], [cover, kid_a, kid_b, lone, off])["quiet"] == 2 + + +async def test_firing_and_silent_come_first_then_who_acknowledged(app): + hot = _rule("r1", "web-1 CPU high") + gone = _rule("r2", "nas agent", resource="nas") + taken = _rule("r3", "db disk", resource="db-1:/var", metric="disk_used_pct", op=">=", threshold=85.0) + rows = [_row(hot, "firing", 94.0, NOW - timedelta(minutes=12)), + _row(gone, "no_data", None, NOW - timedelta(hours=2)), + _row(taken, "acknowledged", 88.0, NOW - timedelta(hours=3), acked_by="bryan", + acked_at=NOW - timedelta(minutes=40))] + page = await _render(app, widget_view(rows, [hot, gone, taken, _rule("r4", "quiet one")])) + assert re.search(r'w-verdict tone-crit">\s*1 firing, 1 silent, 1 acknowledged\.', page) + assert page.index("web-1 CPU high") < page.index("nas agent") < page.index("db disk") + assert 'href="/alerts/history?rule=r1"' in page + assert "web-1 · fires on CPU > 90%" in page and "94%" in page and "12m ago" in page + assert "nas · no reading" in page + assert "db-1:/var · acknowledged by bryan" in page and "40m ago" in page + assert "+1 rule quiet" in page and 'href="/alerts/"' in page + assert len(re.findall(r"]", page)) == 3 + assert not hex_colours(page), hex_colours(page) + + +async def test_all_quiet_is_the_verdict_alone(app): + rules = [_rule("r1", "a"), _rule("r2", "b")] + page = await _render(app, widget_view([], rules)) + assert re.search(r'w-verdict tone-ok">\s*Nothing firing: all 2 rules are quiet\.', page) + assert "+1 more alerts · +2 more monitors' in page + assert "/status/" not in page + async with app.test_request_context("/status/widget"): + page = await render_template("status/widget.html", entries=entries[:3], up=0, down=3, pending=0, + total_hosts=1, alerts=alerts, widget_id="1") + assert "+1 more alerts" in page and "more monitors" not in page + assert 'href="/alerts/"' not in page # the share view has no links + + async def test_a_quiet_overview_says_so(app): async with app.test_request_context("/status/widget"): page = await render_template("status/widget.html", entries=[], up=0, down=0, pending=0, diff --git a/tests/integration/test_email_delivery_pages.py b/tests/integration/test_email_delivery_pages.py index 42bb7ca..fa2e7c4 100644 --- a/tests/integration/test_email_delivery_pages.py +++ b/tests/integration/test_email_delivery_pages.py @@ -96,4 +96,8 @@ def test_an_alerts_email_reads_gathering_then_failed_in_history_and_on_system(ap assert "failed in the last 7 days" in verdict assert extra in system and "550 mailbox gone" in system + # The dashboard's Alerts widget names the rule and the failed email (#5510). + widget = " ".join((await (await client.get("/alerts/widget")).get_data(as_text=True)).split()) + assert "failed in the last 7 days" in widget and 'href="/settings/system/"' in widget + asyncio.run(_go())