bvandeusenandClaude Opus 5.5
22090b02d8
Milestone 495 "Email goes to each account, in its own time zone", on the operator's ruling (System 156): notifications go to each user, times in their zone, defaults on, toggled per user or per role. - users.notify_alerts / notify_report (migration 0036): None follows the role's default, notify.<kind>.<role> in settings, all on to start. - mail.pick_recipients: active accounts that want the email and have an address mail can reach (an SSO placeholder like name@oidc is skipped and flagged), each in its own zone with no UTC beside it, then the Also send to addresses in their chosen zone with UTC beside. One copy per address, all over one SMTP connection. - Alerts, the weekly report and the test email (to you, in your zone, plus the typed addresses) send that way. The save-time smtp.tz capture is gone. - Notifications: a role × email table of defaults, who each email reaches and who can't be reached, and Also send to with its own zone, which starts as the first admin's to open the page. - Account page: your address and your two switches. Users: an Emailed column, and an admin sets an account's switches on its page. - The Alerts page counts the accounts it emails. zone_select macro and formatting.zones_by_region shared by Account and Notifications. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
275 lines
15 KiB
Python
275 lines
15 KiB
Python
"""The Alerts page says each alert as a sentence around a link to what it
|
|
watches, offers only the actions the viewer's role can take, and the sidebar
|
|
counts what needs attention (#5124)."""
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
from quart import render_template, session
|
|
|
|
from steward.alerts import service as alert_svc
|
|
from steward.alerts.service import alert_sentence
|
|
from steward.models.alerts import AlertOperator, AlertStateEnum
|
|
from tests.test_style_tokens import hex_colours
|
|
|
|
|
|
def _rule(metric="cpu_pct", op=AlertOperator.gt, threshold=90.0, resource="web-1", rid="r1"):
|
|
return SimpleNamespace(id=rid, name=f"{resource} {metric}", source_module="host_agent",
|
|
resource_name=resource, metric_name=metric, operator=op,
|
|
threshold=threshold, enabled=True,
|
|
consecutive_failures_required=1, consecutive_ok_required=1)
|
|
|
|
|
|
def _said(says):
|
|
return says["lead"] + says["resource"] + says["tail"]
|
|
|
|
|
|
@pytest.mark.parametrize("op, threshold, value, reads", [
|
|
(AlertOperator.gt, 90.0, 94.0, "CPU on web-1 is 94% (> 90%), firing for 12m"),
|
|
(AlertOperator.gte, 90.0, 90.0, "CPU on web-1 is 90% (>= 90%), firing for 12m"),
|
|
(AlertOperator.lt, 5.0, 2.5, "CPU on web-1 is 2.5% (< 5.0%), firing for 12m"),
|
|
(AlertOperator.lte, 5.0, 5.0, "CPU on web-1 is 5.0% (<= 5.0%), firing for 12m"),
|
|
(AlertOperator.eq, 100.0, 100.0, "CPU on web-1 is 100% (== 100%), firing for 12m"),
|
|
(AlertOperator.ne, 0.0, 3.0, "CPU on web-1 is 3.0% (!= 0.0%), firing for 12m"),
|
|
])
|
|
def test_a_firing_alert_reads_as_one_sentence(op, threshold, value, reads):
|
|
now = datetime.now(timezone.utc)
|
|
says = alert_sentence(_rule(op=op, threshold=threshold), "firing", value,
|
|
now - timedelta(minutes=12), now)
|
|
assert says["resource"] == "web-1"
|
|
assert _said(says) == reads
|
|
|
|
|
|
def test_units_follow_the_metric_name():
|
|
now = datetime.now(timezone.utc)
|
|
says = alert_sentence(_rule("response_ms", threshold=200.0, resource="api"), "pending", 431.0,
|
|
now - timedelta(seconds=45), now)
|
|
assert _said(says) == "Response time on api is 431 ms (> 200 ms), pending for 45s"
|
|
|
|
|
|
def test_an_up_or_down_metric_says_the_word():
|
|
now = datetime.now(timezone.utc)
|
|
says = alert_sentence(_rule("is_up", AlertOperator.lt, 1.0, "nas"), "acknowledged", 0.0,
|
|
now - timedelta(hours=2, minutes=5), now)
|
|
# Acknowledged is still breaching: the sentence says for how long (#5210).
|
|
assert _said(says) == "nas is down, breaching for 2h 5m"
|
|
|
|
|
|
def test_no_reading_yet_is_said_not_shown_as_a_number():
|
|
now = datetime.now(timezone.utc)
|
|
says = alert_sentence(_rule(), "firing", None, now - timedelta(minutes=3), now)
|
|
assert _said(says) == "CPU on web-1 has no reading yet (> 90%), firing for 3m"
|
|
|
|
|
|
def test_a_silent_series_says_how_long_it_has_been_quiet():
|
|
now = datetime.now(timezone.utc)
|
|
says = alert_sentence(_rule(), "no_data", 50.0, now - timedelta(minutes=20), now)
|
|
assert _said(says) == "No CPU reading from web-1 for 20m"
|
|
|
|
|
|
def _row(rule, label, url, state=AlertStateEnum.firing):
|
|
now = datetime.now(timezone.utc)
|
|
st = SimpleNamespace(state=state, no_data_since=None, last_sample_at=None,
|
|
last_transition_at=now - timedelta(minutes=12))
|
|
return {"state": st, "rule": rule, "label": label, "url": url,
|
|
"says": alert_sentence(rule, label, 94.0, st.last_transition_at, now)}
|
|
|
|
|
|
async def test_an_operator_sees_the_sentence_linked_and_can_acknowledge(app):
|
|
rule = _rule()
|
|
async with app.test_request_context("/alerts/"):
|
|
session["user_role"] = "operator"
|
|
page = await render_template("alerts/list.html", active=[_row(rule, "firing", "/hosts/h1")],
|
|
rules=[rule], operators=[])
|
|
|
|
assert 'CPU on <a href="/hosts/h1">web-1</a> is 94% (> 90%), firing for 12m' in page
|
|
assert '<span class="badge badge-crit">Firing</span>' in page
|
|
assert 'action="/alerts/r1/acknowledge"' in page
|
|
assert 'href="/alerts/rules/new"' in page and 'href="/alerts/rules/r1/edit"' in page
|
|
# The whole row opens the rule: one stretched link, the row marked for it.
|
|
row = page[page.index('<tr class="row-link'):]
|
|
row = row[:row.index("</tr>")]
|
|
assert row.count('class="row-target"') == 1 and 'class="row-target" href="/alerts/rules/r1/edit"' in row
|
|
assert 'action="/alerts/rules/r1/delete"' not in page # deleting a rule is the admin's
|
|
main = page[page.index("<main"):] # the sidebar brand mark carries its own colours
|
|
assert "onclick" not in main and not hex_colours(main), hex_colours(main)
|
|
|
|
|
|
async def test_a_viewer_reads_but_gets_no_actions(app):
|
|
rule = _rule()
|
|
async with app.test_request_context("/alerts/"):
|
|
session["user_role"] = "viewer"
|
|
page = await render_template("alerts/list.html", active=[_row(rule, "firing", None)],
|
|
rules=[rule], operators=[])
|
|
|
|
assert "CPU on <strong>web-1</strong>" in page # nothing to link to: no bare href
|
|
for action in ("/acknowledge", "/alerts/rules/new", "/edit", "/toggle", "/delete"):
|
|
assert action not in page, action
|
|
assert 'href="/alerts/history?rule=r1"' in page
|
|
|
|
|
|
async def test_deleting_a_rule_is_on_its_own_page_and_says_what_goes(app):
|
|
"""#5317: the list offers no Delete; the rule's page does, behind the
|
|
confirm dialog, saying its history goes with it."""
|
|
rule = _rule()
|
|
rule.ansible_action, rule.notify_no_data = None, True
|
|
async with app.test_request_context("/alerts/"):
|
|
session["user_role"] = "admin"
|
|
listing = await render_template("alerts/list.html", active=[], rules=[rule], operators=[])
|
|
page = await render_template(
|
|
"alerts/rules_form.html", rule=rule, operators=[], source_modules=["host_agent"],
|
|
initial_source="host_agent", resources=["web-1"], metrics=["cpu_pct"], ansible_sources=[],
|
|
catalog={}, history_count=12, fired=None, notify=[])
|
|
assert "/delete" not in listing
|
|
assert 'action="/alerts/rules/r1/delete"' in page
|
|
assert ('data-confirm="Delete rule web-1 cpu_pct? Its history goes with it (12 changes of state). '
|
|
'Disabling the rule keeps it."') in page
|
|
assert page.index('id="rule-watches"') < page.index('id="rule-fires"') < page.index('id="rule-happens"') \
|
|
< page.index('id="rule-delete"')
|
|
|
|
|
|
async def test_the_rule_page_is_named_for_the_rule(app):
|
|
"""#5317: the title and crumb are the rule's name; seconds read back as a
|
|
duration; the playbook's fields wait for their box; where alerts go is said."""
|
|
rule = _rule("seconds_since_seen", AlertOperator.gt, 180.0, "VDNT-Docker02")
|
|
rule.name, rule.ansible_action, rule.notify_no_data = "Agent offline: VDNT-Docker02", None, True
|
|
async with app.test_request_context("/alerts/rules/r1/edit"):
|
|
session["user_role"] = "admin"
|
|
page = await render_template(
|
|
"alerts/rules_form.html", rule=rule, operators=[(">", ">")], source_modules=["host_agent"],
|
|
initial_source="host_agent", resources=["VDNT-Docker02"], metrics=["seconds_since_seen"],
|
|
ansible_sources=[], catalog={}, history_count=10,
|
|
fired=datetime.now(timezone.utc) - timedelta(hours=6), notify=["email to me@example.com"])
|
|
page = " ".join(page.split())
|
|
assert '<h1 class="page-title">Agent offline: VDNT-Docker02</h1>' in page
|
|
assert '<span class="breadcrumb-cur" aria-current="page">Agent offline: VDNT-Docker02</span>' in page
|
|
assert "Edit Rule" not in page and "Last fired <time" in page and "10 changes of state on record" in page
|
|
assert 'value="180" placeholder="90">' in page and "data-threshold-echo >= 3m</p>" in page
|
|
assert "data-ansible-fields hidden" in page
|
|
assert "It notifies email to me@example.com." in page
|
|
|
|
|
|
def test_where_alerts_go_is_said_as_the_dispatcher_decides():
|
|
"""#5315: email needs a server and recipients; a webhook its URL."""
|
|
assert alert_svc.notify_targets({}, {}) == []
|
|
assert alert_svc.notify_targets({"host": "mx", "recipients": []}, {}) == []
|
|
assert alert_svc.notify_targets({"host": "mx", "recipients": []}, {}, accounts=3) == ["email to 3 accounts"]
|
|
assert alert_svc.notify_targets({"host": "mx", "recipients": ["ops@x"]}, {}, accounts=1) == [
|
|
"email to 1 account, ops@x"]
|
|
assert alert_svc.notify_targets({"host": "", "recipients": []}, {}, accounts=3) == [] # no server, no email
|
|
assert alert_svc.notify_targets({"host": "mx", "recipients": ["a@x", "b@x"]}, {}) == ["email to a@x, b@x"]
|
|
assert alert_svc.notify_targets({"host": "mx", "recipients": "a@x, b@x"}, {"url": "https://hooks.example/abc"}) \
|
|
== ["email to a@x, b@x", "a webhook to hooks.example"]
|
|
|
|
|
|
async def test_the_page_opens_on_a_verdict_and_says_where_alerts_go(app):
|
|
"""#5315: no card for an empty Active list; one line instead."""
|
|
rule = _rule()
|
|
async with app.test_request_context("/alerts/"):
|
|
session["user_role"] = "admin"
|
|
quiet = await render_template("alerts/list.html", active=[], rules=[rule], operators=[], notify=[])
|
|
firing = await render_template("alerts/list.html", active=[_row(rule, "firing", None)], rules=[rule],
|
|
operators=[], notify=["a webhook to hooks.example"],
|
|
last_fired={"r1": datetime.now(timezone.utc) - timedelta(minutes=12)})
|
|
quiet, firing = " ".join(quiet.split()), " ".join(firing.split())
|
|
assert '<strong class="tone-ok">Nothing is firing</strong> · 1 rule ·' in quiet
|
|
assert "notifies nobody yet</span> — <a href=\"/settings/notifications/\">set up email or a webhook</a>" in quiet
|
|
assert "empty-state" not in quiet and ">Active</h2>" not in quiet
|
|
assert '<strong class="tone-crit">1 firing</strong>' in firing and "notifies a webhook to hooks.example" in firing
|
|
assert "last fired <time" in firing and "never fired" in quiet
|
|
assert "on the first check" in quiet
|
|
|
|
|
|
async def _active_count(_db):
|
|
return 3
|
|
|
|
|
|
async def test_the_sidebar_counts_alerts_needing_attention(app, monkeypatch):
|
|
monkeypatch.setattr(alert_svc, "active_count", _active_count)
|
|
async with app.test_request_context("/alerts/"):
|
|
session["user_id"], session["user_role"] = "u1", "viewer"
|
|
page = await render_template("alerts/list.html", active=[], rules=[], operators=[])
|
|
assert '<span class="count-badge" title="3 alerts need attention">3</span>' in page
|
|
|
|
|
|
async def test_no_count_for_a_fragment_or_a_logged_out_page(app, monkeypatch):
|
|
monkeypatch.setattr(alert_svc, "active_count", _active_count)
|
|
async with app.test_request_context("/alerts/", headers={"HX-Request": "true"}):
|
|
session["user_id"], session["user_role"] = "u1", "viewer"
|
|
fragment = await render_template("alerts/list.html", active=[], rules=[], operators=[])
|
|
async with app.test_request_context("/alerts/"):
|
|
logged_out = await render_template("alerts/list.html", active=[], rules=[], operators=[])
|
|
assert "count-badge" not in fragment and "count-badge" not in logged_out
|
|
|
|
|
|
async def test_an_acknowledged_alert_says_who_and_when_apart_from_the_breach(app):
|
|
rule = _rule("is_up", AlertOperator.lt, 1.0, "nas")
|
|
now = datetime.now(timezone.utc)
|
|
st = SimpleNamespace(state=AlertStateEnum.acknowledged, no_data_since=None, last_sample_at=None,
|
|
last_transition_at=now - timedelta(minutes=3),
|
|
acknowledged_at=now - timedelta(minutes=3))
|
|
row = {"state": st, "rule": rule, "label": "acknowledged", "url": None, "acked_by": "bryan",
|
|
"says": alert_sentence(rule, "acknowledged", 0.0, now - timedelta(hours=1), now)}
|
|
async with app.test_request_context("/alerts/active"):
|
|
session["user_role"] = "operator"
|
|
page = await render_template("alerts/_active.html", active=[row])
|
|
page = " ".join(page.split())
|
|
assert "<strong>nas</strong> is down, breaching for 1h" in page
|
|
assert "Acknowledged by bryan <time" in page and ">3m ago</time>" in page
|
|
assert "/acknowledge" not in page # already taken
|
|
|
|
|
|
async def test_rules_say_what_each_is_doing_now(app):
|
|
firing, quiet, off = _rule(rid="r1"), _rule(rid="r2", resource="db"), _rule(rid="r3", resource="old")
|
|
off.enabled = False
|
|
async with app.test_request_context("/alerts/"):
|
|
session["user_role"] = "viewer"
|
|
page = await render_template("alerts/list.html", active=[], rules=[firing, quiet, off],
|
|
rule_state={"r1": "firing", "r2": "inactive"}, operators=[])
|
|
page = " ".join(page.split())
|
|
rules = page[page.index(">Rules</h2>"):]
|
|
assert '<span class="badge badge-crit">Firing</span>' in rules
|
|
assert 'title="Within its threshold">quiet</span>' in rules
|
|
assert '<span class="tone-dim">off</span>' in rules and 'class="row-link row-off"' in rules
|
|
assert 'hx-get="/alerts/active"' in page
|
|
|
|
|
|
def test_a_rule_condition_reads_in_words():
|
|
"""#5212: "is down", not "is_up < 0.5"; a threshold in its unit."""
|
|
from steward.alerts.service import rule_condition
|
|
assert rule_condition(_rule("is_up", AlertOperator.lt, 0.5)) == "is down"
|
|
assert rule_condition(_rule("is_healthy", AlertOperator.gte, 0.5)) == "is healthy"
|
|
assert rule_condition(_rule("disk_used_pct_worst", AlertOperator.gte, 85.0)) == "Fullest disk >= 85%"
|
|
assert rule_condition(_rule("seconds_since_seen", AlertOperator.gt, 180.0)) == \
|
|
"Time since the agent reported > 3m"
|
|
assert rule_condition(_rule("hrProcessorLoad", AlertOperator.gt, 3.0)) == "hrProcessorLoad > 3"
|
|
|
|
|
|
def test_the_form_turns_up_or_down_into_a_condition():
|
|
from steward.alerts.routes import _form_condition
|
|
assert _form_condition({"metric_name": "is_up", "binary_when": "down"}) == \
|
|
{"operator": "<", "threshold": "0.5"}
|
|
assert _form_condition({"metric_name": "is_healthy", "binary_when": "up"}) == \
|
|
{"operator": ">=", "threshold": "0.5"}
|
|
# Any other metric keeps what was typed, whatever binary_when says.
|
|
assert _form_condition({"metric_name": "cpu_pct", "binary_when": "down", "operator": ">",
|
|
"threshold": "90"}) == {"operator": ">", "threshold": "90"}
|
|
|
|
|
|
async def test_the_rule_form_speaks_in_names_and_units(app):
|
|
from steward.alerts.routes import _OPERATORS
|
|
rule = _rule("is_up", AlertOperator.lt, 0.5, "nas")
|
|
rule.source_module, rule.ansible_action, rule.notify_no_data = "http", None, True
|
|
async with app.test_request_context("/alerts/rules/r1/edit"):
|
|
session["user_role"] = "operator"
|
|
page = await render_template(
|
|
"alerts/rules_form.html", rule=rule, operators=_OPERATORS,
|
|
source_modules=["http", "host_agent"], initial_source="http", resources=["nas"],
|
|
metrics=["is_up", "response_ms"], ansible_sources=[],
|
|
catalog={"http": ["is_up", "response_ms"], "snmp": []})
|
|
assert ">HTTP monitors</option>" in page and ">Host agent</option>" in page
|
|
assert ">Up or down</option>" in page and 'data-unit="ms"' in page
|
|
assert '<option value="down" selected>' in page # the binary choice, shown
|
|
assert 'name="threshold" step="any" required disabled' in page # the raw threshold, off
|
|
assert ">is_up<" not in page and "SNMP" not in page # no raw key; empty source skipped
|