bvandeusenandClaude Opus 5.5
73cfbda1da
#5485 "A rule's page waits up to 25 s for its metric preview: a cold 24 h series read from plugin_metrics". Opening "Agent offline" took 25.6 s; a cold day of one series measured 18.3 s over MCP, a warm one 63 ms. The pickers still come with the page from plugin_metric_latest; the preview is its own request (hx-trigger=load), showing "Reading the last day…" until it lands. Picker swaps no longer read it either. #5486 "Send test where it can be found": the button moves from inside "What happens" to the page header beside History, its result under the verdict. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
324 lines
18 KiB
Python
324 lines
18 KiB
Python
"""The Alerts page says each alert as a sentence around a link to what it
|
|
watches, offers only the actions the viewer's role can take, and the sidebar
|
|
counts what needs attention (#5124)."""
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
from quart import render_template, session
|
|
|
|
from steward.alerts import service as alert_svc
|
|
from steward.alerts.service import alert_sentence
|
|
from steward.models.alerts import AlertOperator, AlertStateEnum
|
|
from tests.test_style_tokens import hex_colours
|
|
|
|
|
|
def _rule(metric="cpu_pct", op=AlertOperator.gt, threshold=90.0, resource="web-1", rid="r1"):
|
|
return SimpleNamespace(id=rid, name=f"{resource} {metric}", source_module="host_agent",
|
|
resource_name=resource, metric_name=metric, operator=op,
|
|
threshold=threshold, enabled=True,
|
|
consecutive_failures_required=1, consecutive_ok_required=1,
|
|
covers="resource", cover_group=None, parent_id=None)
|
|
|
|
|
|
def _said(says):
|
|
return says["lead"] + says["resource"] + says["tail"]
|
|
|
|
|
|
@pytest.mark.parametrize("op, threshold, value, reads", [
|
|
(AlertOperator.gt, 90.0, 94.0, "CPU on web-1 is 94% (> 90%), firing for 12m"),
|
|
(AlertOperator.gte, 90.0, 90.0, "CPU on web-1 is 90% (>= 90%), firing for 12m"),
|
|
(AlertOperator.lt, 5.0, 2.5, "CPU on web-1 is 2.5% (< 5.0%), firing for 12m"),
|
|
(AlertOperator.lte, 5.0, 5.0, "CPU on web-1 is 5.0% (<= 5.0%), firing for 12m"),
|
|
(AlertOperator.eq, 100.0, 100.0, "CPU on web-1 is 100% (== 100%), firing for 12m"),
|
|
(AlertOperator.ne, 0.0, 3.0, "CPU on web-1 is 3.0% (!= 0.0%), firing for 12m"),
|
|
])
|
|
def test_a_firing_alert_reads_as_one_sentence(op, threshold, value, reads):
|
|
now = datetime.now(timezone.utc)
|
|
says = alert_sentence(_rule(op=op, threshold=threshold), "firing", value,
|
|
now - timedelta(minutes=12), now)
|
|
assert says["resource"] == "web-1"
|
|
assert _said(says) == reads
|
|
|
|
|
|
def test_units_follow_the_metric_name():
|
|
now = datetime.now(timezone.utc)
|
|
says = alert_sentence(_rule("response_ms", threshold=200.0, resource="api"), "pending", 431.0,
|
|
now - timedelta(seconds=45), now)
|
|
assert _said(says) == "Response time on api is 431 ms (> 200 ms), pending for 45s"
|
|
|
|
|
|
def test_an_up_or_down_metric_says_the_word():
|
|
now = datetime.now(timezone.utc)
|
|
says = alert_sentence(_rule("is_up", AlertOperator.lt, 1.0, "nas"), "acknowledged", 0.0,
|
|
now - timedelta(hours=2, minutes=5), now)
|
|
# Acknowledged is still breaching: the sentence says for how long (#5210).
|
|
assert _said(says) == "nas is down, breaching for 2h 5m"
|
|
|
|
|
|
def test_no_reading_yet_is_said_not_shown_as_a_number():
|
|
now = datetime.now(timezone.utc)
|
|
says = alert_sentence(_rule(), "firing", None, now - timedelta(minutes=3), now)
|
|
assert _said(says) == "CPU on web-1 has no reading yet (> 90%), firing for 3m"
|
|
|
|
|
|
def test_a_silent_series_says_how_long_it_has_been_quiet():
|
|
now = datetime.now(timezone.utc)
|
|
says = alert_sentence(_rule(), "no_data", 50.0, now - timedelta(minutes=20), now)
|
|
assert _said(says) == "No CPU reading from web-1 for 20m"
|
|
|
|
|
|
def _row(rule, label, url, state=AlertStateEnum.firing):
|
|
now = datetime.now(timezone.utc)
|
|
st = SimpleNamespace(state=state, no_data_since=None, last_sample_at=None,
|
|
last_transition_at=now - timedelta(minutes=12))
|
|
return {"state": st, "rule": rule, "label": label, "url": url,
|
|
"says": alert_sentence(rule, label, 94.0, st.last_transition_at, now)}
|
|
|
|
|
|
async def test_an_operator_sees_the_sentence_linked_and_can_acknowledge(app):
|
|
rule = _rule()
|
|
async with app.test_request_context("/alerts/"):
|
|
session["user_role"] = "operator"
|
|
page = await render_template("alerts/list.html", active=[_row(rule, "firing", "/hosts/h1")],
|
|
rules=[rule], operators=[])
|
|
|
|
assert 'CPU on <a href="/hosts/h1">web-1</a> is 94% (> 90%), firing for 12m' in page
|
|
assert '<span class="badge badge-crit">Firing</span>' in page
|
|
assert 'action="/alerts/r1/acknowledge"' in page
|
|
assert 'href="/alerts/rules/new"' in page and 'href="/alerts/rules/r1/edit"' in page
|
|
# The whole row opens the rule: one stretched link, the row marked for it.
|
|
row = page[page.index('<tr class="row-link'):]
|
|
row = row[:row.index("</tr>")]
|
|
assert row.count('class="row-target"') == 1 and 'class="row-target" href="/alerts/rules/r1/edit"' in row
|
|
assert 'action="/alerts/rules/r1/delete"' not in page # deleting a rule is the admin's
|
|
main = page[page.index("<main"):] # the sidebar brand mark carries its own colours
|
|
assert "onclick" not in main and not hex_colours(main), hex_colours(main)
|
|
|
|
|
|
async def test_a_viewer_reads_but_gets_no_actions(app):
|
|
rule = _rule()
|
|
async with app.test_request_context("/alerts/"):
|
|
session["user_role"] = "viewer"
|
|
page = await render_template("alerts/list.html", active=[_row(rule, "firing", None)],
|
|
rules=[rule], operators=[])
|
|
|
|
assert "CPU on <strong>web-1</strong>" in page # nothing to link to: no bare href
|
|
for action in ("/acknowledge", "/alerts/rules/new", "/edit", "/toggle", "/delete"):
|
|
assert action not in page, action
|
|
assert 'href="/alerts/history?rule=r1"' in page
|
|
|
|
|
|
async def test_deleting_a_rule_is_on_its_own_page_and_says_what_goes(app):
|
|
"""#5317: the list offers no Delete; the rule's page does, behind the
|
|
confirm dialog, saying its history goes with it."""
|
|
rule = _rule()
|
|
rule.ansible_action, rule.notify_no_data = None, True
|
|
async with app.test_request_context("/alerts/"):
|
|
session["user_role"] = "admin"
|
|
listing = await render_template("alerts/list.html", active=[], rules=[rule], operators=[])
|
|
page = await render_template(
|
|
"alerts/rules_form.html", rule=rule, operators=[], source_modules=["host_agent"],
|
|
initial_source="host_agent", resources=["web-1"], resource="web-1", metrics=["cpu_pct"],
|
|
metric="cpu_pct", ansible_sources=[],
|
|
catalog={}, history_count=12, fired=None, notify=[])
|
|
assert "/delete" not in listing
|
|
assert 'action="/alerts/rules/r1/delete"' in page
|
|
assert ('data-confirm="Delete rule web-1 cpu_pct? Its history goes with it (12 changes of state). '
|
|
'Disabling the rule keeps it."') in page
|
|
assert page.index('id="rule-watches"') < page.index('id="rule-fires"') < page.index('id="rule-happens"') \
|
|
< page.index('id="rule-delete"')
|
|
|
|
|
|
async def test_the_rule_page_is_named_for_the_rule(app):
|
|
"""#5317: the title and crumb are the rule's name; seconds read back as a
|
|
duration; the playbook's fields wait for their box; where alerts go is said."""
|
|
rule = _rule("seconds_since_seen", AlertOperator.gt, 180.0, "VDNT-Docker02")
|
|
rule.name, rule.ansible_action, rule.notify_no_data = "Agent offline: VDNT-Docker02", None, True
|
|
async with app.test_request_context("/alerts/rules/r1/edit"):
|
|
session["user_role"] = "admin"
|
|
page = await render_template(
|
|
"alerts/rules_form.html", rule=rule, operators=[(">", ">")], source_modules=["host_agent"],
|
|
initial_source="host_agent", resources=["VDNT-Docker02"], resource="VDNT-Docker02",
|
|
metrics=["seconds_since_seen"], metric="seconds_since_seen",
|
|
ansible_sources=[], catalog={}, history_count=10,
|
|
fired=datetime.now(timezone.utc) - timedelta(hours=6), notify=["email to me@example.com"])
|
|
page = " ".join(page.split())
|
|
assert '<h1 class="page-title">Agent offline: VDNT-Docker02</h1>' in page
|
|
assert '<span class="breadcrumb-cur" aria-current="page">Agent offline: VDNT-Docker02</span>' in page
|
|
assert "Edit Rule" not in page and "Last fired <time" in page and "10 changes of state on record" in page
|
|
assert 'value="180" placeholder="90">' in page and "data-threshold-echo >= 3m</p>" in page
|
|
assert "data-ansible-fields hidden" in page
|
|
assert "It notifies email to me@example.com." in page
|
|
|
|
|
|
def test_where_alerts_go_is_said_as_the_dispatcher_decides():
|
|
"""#5315: email needs a server and recipients; a webhook its URL."""
|
|
assert alert_svc.notify_targets({}, {}) == []
|
|
assert alert_svc.notify_targets({"host": "mx", "recipients": []}, {}) == []
|
|
assert alert_svc.notify_targets({"host": "mx", "recipients": []}, {}, accounts=3) == ["email to 3 accounts"]
|
|
assert alert_svc.notify_targets({"host": "mx", "recipients": ["ops@x"]}, {}, accounts=1) == [
|
|
"email to 1 account, ops@x"]
|
|
assert alert_svc.notify_targets({"host": "", "recipients": []}, {}, accounts=3) == [] # no server, no email
|
|
assert alert_svc.notify_targets({"host": "mx", "recipients": ["a@x", "b@x"]}, {}) == ["email to a@x, b@x"]
|
|
assert alert_svc.notify_targets({"host": "mx", "recipients": "a@x, b@x"}, {"url": "https://hooks.example/abc"}) \
|
|
== ["email to a@x, b@x", "a webhook to hooks.example"]
|
|
|
|
|
|
async def test_the_page_opens_on_a_verdict_and_says_where_alerts_go(app):
|
|
"""#5315: no card for an empty Active list; one line instead."""
|
|
rule = _rule()
|
|
async with app.test_request_context("/alerts/"):
|
|
session["user_role"] = "admin"
|
|
quiet = await render_template("alerts/list.html", active=[], rules=[rule], operators=[], notify=[])
|
|
firing = await render_template("alerts/list.html", active=[_row(rule, "firing", None)], rules=[rule],
|
|
operators=[], notify=["a webhook to hooks.example"],
|
|
last_fired={"r1": datetime.now(timezone.utc) - timedelta(minutes=12)})
|
|
quiet, firing = " ".join(quiet.split()), " ".join(firing.split())
|
|
assert '<strong class="tone-ok">Nothing is firing</strong> · 1 rule ·' in quiet
|
|
assert "notifies nobody yet</span> — <a href=\"/settings/notifications/\">set up email or a webhook</a>" in quiet
|
|
assert "empty-state" not in quiet and ">Active</h2>" not in quiet
|
|
assert '<strong class="tone-crit">1 firing</strong>' in firing and "notifies a webhook to hooks.example" in firing
|
|
assert "last fired <time" in firing and "never fired" in quiet
|
|
assert "on the first check" in quiet
|
|
|
|
|
|
async def _active_count(_db):
|
|
return 3
|
|
|
|
|
|
async def test_the_sidebar_counts_alerts_needing_attention(app, monkeypatch):
|
|
monkeypatch.setattr(alert_svc, "active_count", _active_count)
|
|
async with app.test_request_context("/alerts/"):
|
|
session["user_id"], session["user_role"] = "u1", "viewer"
|
|
page = await render_template("alerts/list.html", active=[], rules=[], operators=[])
|
|
assert '<span class="count-badge" title="3 alerts need attention">3</span>' in page
|
|
|
|
|
|
async def test_no_count_for_a_fragment_or_a_logged_out_page(app, monkeypatch):
|
|
monkeypatch.setattr(alert_svc, "active_count", _active_count)
|
|
async with app.test_request_context("/alerts/", headers={"HX-Request": "true"}):
|
|
session["user_id"], session["user_role"] = "u1", "viewer"
|
|
fragment = await render_template("alerts/list.html", active=[], rules=[], operators=[])
|
|
async with app.test_request_context("/alerts/"):
|
|
logged_out = await render_template("alerts/list.html", active=[], rules=[], operators=[])
|
|
assert "count-badge" not in fragment and "count-badge" not in logged_out
|
|
|
|
|
|
async def test_an_acknowledged_alert_says_who_and_when_apart_from_the_breach(app):
|
|
rule = _rule("is_up", AlertOperator.lt, 1.0, "nas")
|
|
now = datetime.now(timezone.utc)
|
|
st = SimpleNamespace(state=AlertStateEnum.acknowledged, no_data_since=None, last_sample_at=None,
|
|
last_transition_at=now - timedelta(minutes=3),
|
|
acknowledged_at=now - timedelta(minutes=3))
|
|
row = {"state": st, "rule": rule, "label": "acknowledged", "url": None, "acked_by": "bryan",
|
|
"says": alert_sentence(rule, "acknowledged", 0.0, now - timedelta(hours=1), now)}
|
|
async with app.test_request_context("/alerts/active"):
|
|
session["user_role"] = "operator"
|
|
page = await render_template("alerts/_active.html", active=[row])
|
|
page = " ".join(page.split())
|
|
assert "<strong>nas</strong> is down, breaching for 1h" in page
|
|
assert "Acknowledged by bryan <time" in page and ">3m ago</time>" in page
|
|
assert "/acknowledge" not in page # already taken
|
|
|
|
|
|
async def test_rules_say_what_each_is_doing_now(app):
|
|
firing, quiet, off = _rule(rid="r1"), _rule(rid="r2", resource="db"), _rule(rid="r3", resource="old")
|
|
off.enabled = False
|
|
async with app.test_request_context("/alerts/"):
|
|
session["user_role"] = "viewer"
|
|
page = await render_template("alerts/list.html", active=[], rules=[firing, quiet, off],
|
|
rule_state={"r1": "firing", "r2": "inactive"}, operators=[])
|
|
page = " ".join(page.split())
|
|
rules = page[page.index(">Rules</h2>"):]
|
|
assert '<span class="badge badge-crit">Firing</span>' in rules
|
|
assert 'title="Within its threshold">quiet</span>' in rules
|
|
assert '<span class="tone-dim">off</span>' in rules and 'class="row-link row-off"' in rules
|
|
assert 'hx-get="/alerts/active"' in page
|
|
|
|
|
|
def test_a_rule_condition_reads_in_words():
|
|
"""#5212: "is down", not "is_up < 0.5"; a threshold in its unit."""
|
|
from steward.alerts.service import rule_condition
|
|
assert rule_condition(_rule("is_up", AlertOperator.lt, 0.5)) == "is down"
|
|
assert rule_condition(_rule("is_healthy", AlertOperator.gte, 0.5)) == "is healthy"
|
|
assert rule_condition(_rule("disk_used_pct_worst", AlertOperator.gte, 85.0)) == "Fullest disk >= 85%"
|
|
assert rule_condition(_rule("seconds_since_seen", AlertOperator.gt, 180.0)) == \
|
|
"Time since the agent reported > 3m"
|
|
assert rule_condition(_rule("hrProcessorLoad", AlertOperator.gt, 3.0)) == "hrProcessorLoad > 3"
|
|
|
|
|
|
def test_the_form_turns_up_or_down_into_a_condition():
|
|
from steward.alerts.routes import _form_condition
|
|
assert _form_condition({"metric_name": "is_up", "binary_when": "down"}) == \
|
|
{"operator": "<", "threshold": "0.5"}
|
|
assert _form_condition({"metric_name": "is_healthy", "binary_when": "up"}) == \
|
|
{"operator": ">=", "threshold": "0.5"}
|
|
# Any other metric keeps what was typed, whatever binary_when says.
|
|
assert _form_condition({"metric_name": "cpu_pct", "binary_when": "down", "operator": ">",
|
|
"threshold": "90"}) == {"operator": ">", "threshold": "90"}
|
|
|
|
|
|
async def test_the_rule_form_speaks_in_names_and_units(app):
|
|
from steward.alerts.routes import _OPERATORS
|
|
rule = _rule("is_up", AlertOperator.lt, 0.5, "nas")
|
|
rule.source_module, rule.ansible_action, rule.notify_no_data = "http", None, True
|
|
async with app.test_request_context("/alerts/rules/r1/edit"):
|
|
session["user_role"] = "operator"
|
|
page = await render_template(
|
|
"alerts/rules_form.html", rule=rule, operators=_OPERATORS,
|
|
source_modules=["http", "host_agent"], initial_source="http", resources=["nas"], resource="nas",
|
|
metrics=["is_up", "response_ms"], metric="is_up", ansible_sources=[],
|
|
catalog={"http": ["is_up", "response_ms"], "snmp": []})
|
|
assert ">HTTP monitors</option>" in page and ">Host agent</option>" in page
|
|
assert ">Up or down</option>" in page and 'data-unit="ms"' in page
|
|
assert '<option value="down" selected>' in page # the binary choice, shown
|
|
assert 'name="threshold" step="any" required disabled' in page # the raw threshold, off
|
|
assert ">is_up<" not in page and "SNMP" not in page # no raw key; empty source skipped
|
|
|
|
|
|
async def test_a_rule_covering_hosts_lists_its_hosts_under_it(app):
|
|
"""#5464: a covering rule reads as what it covers, with each host's rule
|
|
nested under it opening that host's history; its page names the hosts."""
|
|
parent = _rule("seconds_since_seen", AlertOperator.gt, 180.0, "", rid="p1")
|
|
parent.name, parent.covers, parent.cover_group = "Agent offline", "group", "swarm"
|
|
parent.ansible_action, parent.notify_no_data = None, True
|
|
kids = []
|
|
for i, host in enumerate(("node-a", "node-b")):
|
|
k = _rule("seconds_since_seen", AlertOperator.gt, 180.0, host, rid=f"k{i}")
|
|
k.name, k.parent_id = f"Agent offline: {host}", "p1"
|
|
kids.append(k)
|
|
async with app.test_request_context("/alerts/"):
|
|
session["user_role"] = "operator"
|
|
listing = await render_template(
|
|
"alerts/list.html", active=[], rules=[parent], children={"p1": kids},
|
|
uneasy={"p1": ["node-a"]},
|
|
rule_state={"p1": "firing", "k0": "firing", "k1": "inactive"}, operators=[])
|
|
quiet = await render_template(
|
|
"alerts/list.html", active=[], rules=[parent], children={"p1": kids},
|
|
uneasy={"p1": []}, rule_state={"p1": "inactive"}, operators=[])
|
|
page = await render_template(
|
|
"alerts/rules_form.html", rule=parent, operators=[], source_modules=["host_agent"],
|
|
initial_source="host_agent", resources=[], metrics=["seconds_since_seen"],
|
|
metric="seconds_since_seen", ansible_sources=[], catalog={}, history_count=0,
|
|
fired=None, notify=[],
|
|
covered=["node-a", "node-b"], host_source=True, groups=["swarm"],
|
|
current_covers="group", current_group="swarm", covers_locked=True)
|
|
listing, page = " ".join(listing.split()), " ".join(page.split())
|
|
assert "every host in swarm" in listing and "2 hosts" in listing
|
|
assert listing.index("Agent offline</a>") < listing.index('href="/alerts/history?rule=k0"') \
|
|
< listing.index('href="/alerts/history?rule=k1"')
|
|
assert 'class="row-link row-child"' in listing
|
|
# #5483: the hosts sit behind the rule's toggle, open while one isn't quiet.
|
|
assert 'aria-expanded="true" aria-controls="hosts-p1"' in listing and "on node-a" in listing
|
|
assert '<tbody id="hosts-p1" class="row-children">' in listing
|
|
quiet = " ".join(quiet.split())
|
|
assert 'aria-expanded="false" aria-controls="hosts-p1"' in quiet
|
|
assert '<tbody id="hosts-p1" class="row-children" hidden>' in quiet and "never fired" in quiet
|
|
assert "Covers every host in <strong>swarm</strong>: 2 now (node-a, node-b)" in page
|
|
# #5486: Send test sits in the page's header, its result under the verdict.
|
|
assert page.index('hx-post="/alerts/rules/p1/test"') < page.index('id="rule-test-result"') \
|
|
< page.index('id="rule-watches"')
|
|
assert 'name="covers" value="group"' in page and "fixed once the rule exists" in page
|
|
assert 'data-cover-resource hidden' in page
|