bvandeusenandClaude Opus 5.5
48a0de5600
#5464 "Group rules: one rule covers every host, or every host in a group", step 2 of milestone 499 "Alerts you can trust". Each rule named one host, so the live instance carried six rules for three hosts, and a new host had no agent-offline or disk alert until someone copied them. - alert_rules.covers / cover_group / parent_id (migration 0041). A rule on host_agent may cover "all_hosts" or a "group" (an inventory group: the Hosts ruling, #5294). A covering rule has no state; Steward keeps one ordinary child rule per covered host, named "<rule>: <host>", so each host's alert keeps its own state, history and acknowledgement. - alerts.service.reconcile_children: a covered host that has reported in the last 7 days gets a child; a host no longer covered loses its child; children take the parent's condition, checks, notify setting, action and on/off. Run on save and each minute (alert_reconcile). - A child is changed through its parent: edit, toggle and delete of a child are refused (409 / PartOfCoveringRule); its edit page opens the parent. - Rule form: a Covers choice for host sources, with the group picker; fixed once the rule exists. Rules list: a covering rule reads "every host in swarm · 3 hosts" with its hosts' rules nested under it, each opening its history. The history picker and the rule count leave covering rules out. - MCP: create_alert_rule takes covers/group; update_alert_rule takes group and passes changes down; list_alert_rules shows covers, hosts and part_of. - docs/core/alerting.md: "Rules that cover hosts". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
174 lines
8.4 KiB
Python
174 lines
8.4 KiB
Python
"""Integration: a rule covering hosts (#5464) keeps one ordinary rule per
|
|
covered host, adds one for a host that joins its group, removes one for a host
|
|
that leaves, passes edits and on/off down to them, and refuses a change made
|
|
to a host's rule alone. Each host's rule fires on its own readings."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import uuid
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
def test_a_group_rule_keeps_one_rule_per_host_in_the_group(app):
|
|
from sqlalchemy import select
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.alerts import record_metric
|
|
from steward.models.alerts import AlertRule, AlertState, AlertStateEnum
|
|
from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget
|
|
from steward.models.hosts import Host
|
|
from steward.models.users import User, UserRole
|
|
|
|
async def _go():
|
|
tag = uuid.uuid4().hex[:8]
|
|
names = [f"cv-{tag}-{i}" for i in range(3)]
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
user = User(username=f"cv-{tag}", email=f"cv-{tag}@test", password_hash="x",
|
|
role=UserRole.admin)
|
|
hosts = [Host(name=n, address=f"10.8.{i}.1") for i, n in enumerate(names)]
|
|
s.add_all([user, *hosts])
|
|
await s.flush()
|
|
group = AnsibleGroup(name=f"web {tag}")
|
|
targets = [AnsibleTarget(name=f"t-{n}", address=h.address, host_id=h.id)
|
|
for n, h in zip(names, hosts)]
|
|
for t in targets[:2]:
|
|
t.groups = [group]
|
|
s.add_all([group, *targets])
|
|
for n in names:
|
|
await record_metric(s, "host_agent", n, "cpu_pct", 10.0)
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=f"Hot {tag}", source_module="host_agent", resource_name="",
|
|
metric_name="cpu_pct", operator=">", threshold=90,
|
|
covers="group", cover_group=group.name)
|
|
parent = await alert_svc.add_rule(s, fields, created_by=user.id)
|
|
ids = {"parent": parent.id, "user": user.id, "third": targets[2].id,
|
|
"group": group.id}
|
|
|
|
async def kids():
|
|
async with app.db_sessionmaker() as s:
|
|
rows = (await s.execute(select(AlertRule).where(
|
|
AlertRule.parent_id == ids["parent"]))).scalars().all()
|
|
return {r.resource_name: r for r in rows}
|
|
|
|
async def reconcile():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
return await alert_svc.reconcile_children(
|
|
s, await s.get(AlertRule, ids["parent"]))
|
|
|
|
# The two hosts in the group, each its own rule; the parent has no state.
|
|
got = await kids()
|
|
assert sorted(got) == names[:2]
|
|
assert got[names[0]].name == f"Hot {tag}: {names[0]}"
|
|
assert got[names[0]].threshold == 90 and got[names[0]].covers == "resource"
|
|
async with app.db_sessionmaker() as s:
|
|
assert (await s.execute(select(AlertState).where(
|
|
AlertState.rule_id == ids["parent"]))).scalar_one_or_none() is None
|
|
|
|
# A host's own readings fire its own rule only.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await record_metric(s, "host_agent", names[0], "cpu_pct", 97.0)
|
|
async with app.db_sessionmaker() as s:
|
|
states = {st.rule_id: st.state for st in (await s.execute(select(AlertState).where(
|
|
AlertState.rule_id.in_([r.id for r in got.values()])))).scalars()}
|
|
assert states[got[names[0]].id] == AlertStateEnum.firing
|
|
assert states[got[names[1]].id] == AlertStateEnum.inactive
|
|
|
|
# The third host joins the group: covered on the next reconcile.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
from sqlalchemy.orm import selectinload
|
|
t = (await s.execute(select(AnsibleTarget).where(AnsibleTarget.id == ids["third"])
|
|
.options(selectinload(AnsibleTarget.groups)))).scalar_one()
|
|
t.groups = [await s.get(AnsibleGroup, ids["group"])]
|
|
assert await reconcile() == ([names[2]], [])
|
|
assert sorted(await kids()) == names
|
|
|
|
# Raising the threshold passes down; the firing host's alert resets.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
p = await s.get(AlertRule, ids["parent"])
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=p.name, source_module="host_agent", resource_name="",
|
|
metric_name="cpu_pct", operator=">", threshold=99,
|
|
covers="group", cover_group=p.cover_group)
|
|
await alert_svc.apply_rule_change(s, p, fields, actor_id=ids["user"])
|
|
await alert_svc.reconcile_children(s, p, actor_id=ids["user"])
|
|
got = await kids()
|
|
assert {r.threshold for r in got.values()} == {99}
|
|
async with app.db_sessionmaker() as s:
|
|
st = (await s.execute(select(AlertState).where(
|
|
AlertState.rule_id == got[names[0]].id))).scalar_one()
|
|
assert st.state == AlertStateEnum.inactive
|
|
|
|
# Disabling the parent disables every host's rule.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
p = await s.get(AlertRule, ids["parent"])
|
|
await alert_svc.apply_rule_change(s, p, {}, enabled=False)
|
|
await alert_svc.reconcile_children(s, p)
|
|
assert not any(r.enabled for r in (await kids()).values())
|
|
|
|
# A host leaving the group loses its rule.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
t = (await s.execute(select(AnsibleTarget).where(AnsibleTarget.id == ids["third"])
|
|
.options(selectinload(AnsibleTarget.groups)))).scalar_one()
|
|
t.groups = []
|
|
assert await reconcile() == ([], [names[2]])
|
|
assert sorted(await kids()) == names[:2]
|
|
|
|
asyncio.run(_go())
|
|
|
|
|
|
def test_a_host_rule_is_changed_through_its_covering_rule(app):
|
|
"""Over MCP, a host's rule under a covering rule refuses an edit made to it
|
|
alone, and the covering rule lists the hosts it covers."""
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.alerts import record_metric
|
|
from steward.core.api_tokens import Caller
|
|
from steward.mcp import _context
|
|
from steward.mcp.tools import alerts as a
|
|
from steward.models.hosts import Host
|
|
from steward.models.users import User, UserRole
|
|
|
|
async def _go():
|
|
tag = uuid.uuid4().hex[:8]
|
|
name = f"cvm-{tag}"
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
user = User(username=f"cvm-{tag}", email=f"cvm-{tag}@test", password_hash="x",
|
|
role=UserRole.admin)
|
|
s.add_all([user, Host(name=name, address="10.8.9.1")])
|
|
await s.flush()
|
|
await record_metric(s, "host_agent", name, "seconds_since_seen", 5.0)
|
|
uid = user.id
|
|
_context._app_ctx.set(app)
|
|
_context._caller_ctx.set(Caller(user_id=uid, username=f"cvm-{tag}",
|
|
role=UserRole.admin, token_id=7))
|
|
made = await a.create_alert_rule(f"Offline {tag}", "host_agent", "",
|
|
"seconds_since_seen", ">", 180, covers="all_hosts")
|
|
assert made["covers"] == "all_hosts" and name in made["hosts"]
|
|
with pytest.raises(alert_svc.PartOfCoveringRule, match="change or disable it there"):
|
|
await a.update_alert_rule(f"Offline {tag}: {name}", threshold=60)
|
|
listed = {r["rule_name"]: r for r in (await a.list_alert_rules(limit=500))["rules"]}
|
|
assert listed[f"Offline {tag}: {name}"]["part_of"] == f"Offline {tag}"
|
|
await a.update_alert_rule(f"Offline {tag}", threshold=300)
|
|
listed = {r["rule_name"]: r for r in (await a.list_alert_rules(limit=500))["rules"]}
|
|
assert listed[f"Offline {tag}: {name}"]["condition"].startswith("seconds_since_seen > 300")
|
|
|
|
asyncio.run(_go())
|