bvandeusen/FabledSteward
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(alerts): one rule can cover every host, or every host in a group (#5464)
CI / lint (push) Successful in 3s
Details
CI / integration (push) Successful in 1m7s
Details
CI / unit (push) Successful in 1m25s
Details
CI / publish (push) Successful in 1m12s
Details

Browse Source 
#5464 "Group rules: one rule covers every host, or every host in a group",
step 2 of milestone 499 "Alerts you can trust". Each rule named one host, so
the live instance carried six rules for three hosts, and a new host had no
agent-offline or disk alert until someone copied them.

- alert_rules.covers / cover_group / parent_id (migration 0041). A rule on
  host_agent may cover "all_hosts" or a "group" (an inventory group: the
  Hosts ruling, #5294). A covering rule has no state; Steward keeps one
  ordinary child rule per covered host, named "<rule>: <host>", so each
  host's alert keeps its own state, history and acknowledgement.
- alerts.service.reconcile_children: a covered host that has reported in
  the last 7 days gets a child; a host no longer covered loses its child;
  children take the parent's condition, checks, notify setting, action and
  on/off. Run on save and each minute (alert_reconcile).
- A child is changed through its parent: edit, toggle and delete of a child
  are refused (409 / PartOfCoveringRule); its edit page opens the parent.
- Rule form: a Covers choice for host sources, with the group picker; fixed
  once the rule exists. Rules list: a covering rule reads "every host in
  swarm · 3 hosts" with its hosts' rules nested under it, each opening its
  history. The history picker and the rule count leave covering rules out.
- MCP: create_alert_rule takes covers/group; update_alert_rule takes group
  and passes changes down; list_alert_rules shows covers, hosts and part_of.
- docs/core/alerting.md: "Rules that cover hosts".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-10-09 08:25:51 -04:00
co-authored by Claude Opus 5.5
parent 734d313a54
commit 48a0de5600
13 changed files with 647 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
steward/alerts/routes.py
+70 -8
View File
@@ -36,6 +36,8 @@ def _rule_form(form) -> dict:
"consecutive_failures_required": form.get("consecutive_failures_required"),
"consecutive_ok_required": form.get("consecutive_ok_required"),
"notify_no_data": "notify_no_data" in form,
"covers": form.get("covers") or "resource",
"cover_group": form.get("cover_group", ""),
}
@@ -47,6 +49,15 @@ def _form_time(value) -> datetime:
raise ValueError(f"{value!r} is not a date and time.") from None
async def _child_conflict(db, rule: AlertRule):
"""A 409 for a host's rule under a covering rule, which is changed
through its parent (#5464); None for any other rule."""
if not rule.parent_id:
return None
parent = await db.get(AlertRule, rule.parent_id)
return str(alert_svc.PartOfCoveringRule(rule, parent.name if parent else "?")), 409
def _is_admin() -> bool:
return session.get("user_role") == UserRole.admin.value
@@ -123,11 +134,27 @@ async def list_alerts():
states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
fired = await alert_svc.last_fired(db)
# Every rule with what it is doing now; disabled rules last.
rules.sort(key=lambda r: (not r.enabled, r.name.lower()))
# Every rule with what it is doing now; disabled rules last. A covering
# rule's per-host children sit under it (#5464), and it reads as the most
# pressing of their states.
rule_state = {r.id: alert_svc.alert_label(states[r.id]) for r in rules if r.id in states}
children: dict[str, list] = {}
for r in rules:
if r.parent_id:
children.setdefault(r.parent_id, []).append(r)
for kids in children.values():
kids.sort(key=lambda r: r.resource_name.lower())
parent_id = kids[0].parent_id
labels = [rule_state.get(k.id) for k in kids]
rule_state[parent_id] = min(
(lb for lb in labels if lb and lb not in ("inactive", "resolved")),
key=lambda lb: alert_svc.ACTIVE_ORDER.get(lb, 9), default="inactive")
fired[parent_id] = max((fired[k.id] for k in kids if k.id in fired), default=None)
rules = sorted((r for r in rules if not r.parent_id),
key=lambda r: (not r.enabled, r.name.lower()))
return await render_template(
"alerts/list.html", active=active, rules=rules, rule_state=rule_state, last_fired=fired,
"alerts/list.html", active=active, rules=rules, children=children,
rule_state=rule_state, last_fired={k: v for k, v in fired.items() if v},
notify=await _notify(), operators=_OPERATORS,
poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60))
@@ -138,7 +165,8 @@ async def active_fragment():
"""The verdict and Active section alone, which the alerts page polls."""
async with current_app.db_sessionmaker() as db:
active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
rules = await alert_svc.list_rules(db)
# Rules as people made them: a covering rule counts once, not per host.
rules = [r for r in await alert_svc.list_rules(db) if not r.parent_id]
return await render_template(
"alerts/_active.html", active=active, notify=await _notify(), rules_total=len(rules),
rules_off=sum(1 for r in rules if not r.enabled))
@@ -195,7 +223,8 @@ async def alert_history():
since, range_key = parse_range(request.args.get("range") or "7d")
before = request.args.get("before")
async with current_app.db_sessionmaker() as db:
rules = await alert_svc.list_rules(db)
# A covering rule has no history of its own; its hosts' rules do.
rules = [r for r in await alert_svc.list_rules(db) if r.covers == "resource"]
chosen = next((r for r in rules if r.id == request.args.get("rule")), None)
rule_id = chosen.id if chosen else None
rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before)
@@ -232,26 +261,40 @@ async def rule_fields_api():
current_metric = request.args.get("current_metric", "")
async with current_app.db_sessionmaker() as db:
resources = await alert_svc.resources_for(db, source) if source else []
async with current_app.db_sessionmaker() as db:
metrics = await alert_svc.metrics_for(db, source)
cover = await _cover_choices(db, source)
return await render_template(
"alerts/_rule_fields.html",
resources=resources,
metrics=metrics,
current_resource=current_resource,
current_metric=current_metric,
**cover,
)
async def _cover_choices(db, source: str, rule: AlertRule | None = None) -> dict:
"""What the rule form offers for covering hosts (#5464): whether this
source's resources are hosts, the inventory groups, and the rule's own
choice, fixed once it exists."""
from steward.hosts.service import group_names
host_source = source in alert_svc.HOST_SOURCES
return {"host_source": host_source, "groups": await group_names(db) if host_source else [],
"current_covers": rule.covers if rule else "resource",
"current_group": (rule.cover_group or "") if rule else "",
"covers_locked": rule is not None}
@alerts_bp.get("/rules/new")
@require_role(UserRole.operator)
async def new_rule():
first_source = alert_svc.source_modules()[0]
async with current_app.db_sessionmaker() as db:
resources = await alert_svc.resources_for(db, first_source)
cover = await _cover_choices(db, first_source)
return await render_template(
"alerts/rules_form.html",
rule=None,
rule=None, **cover,
operators=_OPERATORS,
source_modules=alert_svc.source_modules(),
initial_source=first_source,
@@ -271,13 +314,20 @@ async def edit_rule(rule_id: str):
rule = result.scalar_one_or_none()
if rule is None:
return "Not found", 404
if rule.parent_id:
# A host's rule under a covering rule is changed through it.
return redirect(f"/alerts/rules/{rule.parent_id}/edit")
resources = await alert_svc.resources_for(db, rule.source_module)
cover = await _cover_choices(db, rule.source_module, rule)
hosts = sorted((c.resource_name for c in (await db.execute(
select(AlertRule).where(AlertRule.parent_id == rule.id))).scalars()), key=str.lower)
history_count = (await db.execute(
select(func.count()).select_from(AlertEvent).where(AlertEvent.rule_id == rule.id))).scalar_one()
fired = (await alert_svc.last_fired(db, rule.id)).get(rule.id)
return await render_template(
"alerts/rules_form.html",
rule=rule, history_count=history_count, fired=fired, notify=await _notify(),
covered=hosts, **cover,
operators=_OPERATORS,
source_modules=alert_svc.source_modules(),
initial_source=rule.source_module,
@@ -331,13 +381,19 @@ async def update_rule(rule_id: str):
rule = result.scalar_one_or_none()
if rule is None:
return "Not found", 404
if (conflict := await _child_conflict(db, rule)):
return conflict
try:
fields = await alert_svc.rule_fields(db, **_rule_form(form))
# What a rule covers is fixed once it exists; its group may change.
fields = await alert_svc.rule_fields(db, **{**_rule_form(form),
"covers": rule.covers})
except ValueError as exc:
return str(exc), 400
await alert_svc.apply_rule_change(db, rule, fields, actor_id=session.get("user_id"))
if apply_action:
rule.ansible_action = ansible_action
if rule.covers != "resource":
await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id"))
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
"alert_rule.updated", entity_type="alert_rule", entity_id=rule.name)
return redirect(url_for("alerts.list_alerts"))
@@ -352,8 +408,12 @@ async def toggle_rule(rule_id: str):
rule = result.scalar_one_or_none()
if rule is None:
return "Not found", 404
if (conflict := await _child_conflict(db, rule)):
return conflict
await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled,
actor_id=session.get("user_id"))
if rule.covers != "resource":
await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id"))
action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
action, entity_type="alert_rule", entity_id=rule.name)
@@ -369,6 +429,8 @@ async def delete_rule(rule_id: str):
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if rule:
if (conflict := await _child_conflict(db, rule)):
return conflict
rule_name = rule.name
await db.delete(rule)
if rule_name:
steward/alerts/service.py
+150 -9
View File
@@ -378,12 +378,16 @@ def _choices(values: list[str], limit: int = 20) -> str:
async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
operator: str, threshold, consecutive_failures_required=1,
consecutive_ok_required=1, notify_no_data=True) -> dict:
consecutive_ok_required=1, notify_no_data=True,
covers: str = "resource", cover_group: str = "") -> dict:
"""Validated column values for a new or edited alert rule.
The same choices the rule form offers: a known source, a resource that
source has reported (or, for monitors, a monitor of that type), and a
metric it emits. ValueError says what was wrong and what is valid.
metric it emits. A covering rule (`covers` "all_hosts", or "group" with
an inventory group's name in `cover_group`, #5464) names no resource: it
is for a source whose resources are hosts. ValueError says what was wrong
and what is valid.
"""
name = (name or "").strip()
if not name:
@@ -392,11 +396,29 @@ async def rule_fields(db, *, name: str, source_module: str, resource_name: str,
catalog = metric_catalog()
if source_module not in catalog:
raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.")
resource_name = (resource_name or "").strip()
resources = await resources_for(db, source_module)
if resource_name not in resources:
raise ValueError(f"{source_module} has no resource {resource_name!r}. "
f"Known: {_choices(resources)}.")
covers = (covers or "resource").strip()
if covers not in COVERS:
raise ValueError(f"covers {covers!r} is not one of {', '.join(COVERS)}.")
cover_group = (cover_group or "").strip()
if covers == "resource":
resource_name, cover_group = (resource_name or "").strip(), None
resources = await resources_for(db, source_module)
if resource_name not in resources:
raise ValueError(f"{source_module} has no resource {resource_name!r}. "
f"Known: {_choices(resources)}.")
else:
if source_module not in HOST_SOURCES:
raise ValueError(f"only a rule on {', '.join(HOST_SOURCES)} can cover hosts; "
f"{source_module}'s resources are not hosts.")
resource_name = ""
if covers == "group":
from steward.hosts.service import group_names
groups = await group_names(db)
if cover_group not in groups:
raise ValueError(f"there is no inventory group {cover_group!r}. "
f"Groups: {_choices(groups)}.")
else:
cover_group = None
metric_name = (metric_name or "").strip()
metrics = await metrics_for(db, source_module)
if metric_name not in metrics:
@@ -428,7 +450,8 @@ async def rule_fields(db, *, name: str, source_module: str, resource_name: str,
return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
"metric_name": metric_name, "operator": op, "threshold": threshold,
"consecutive_failures_required": consecutive,
"consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data)}
"consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data),
"covers": covers, "cover_group": cover_group}
# Changing any of these changes what the rule is watching, so whatever state
@@ -460,15 +483,133 @@ async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool
async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
ansible_action: dict | None = None) -> AlertRule:
"""Insert a rule and its inactive state row, in the caller's transaction."""
"""Insert a rule and its inactive state row, in the caller's transaction.
A covering rule gets no state; its per-host children are made here."""
rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
db.add(rule)
await db.flush()
if rule.covers != "resource":
await reconcile_children(db, rule)
return rule
db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
last_transition_at=datetime.now(timezone.utc)))
return rule
# ── Covering rules (#5464) ───────────────────────────────────────────────────
# A rule can cover every host, or every host in an inventory group (inventory
# groups are Steward's grouping system: the Hosts ruling, #5294). Steward keeps
# one ordinary child rule per covered host, so each host's alert has its own
# state, history and acknowledgement, and a host that appears later is covered
# without anyone copying the rule.
COVERS = ("resource", "all_hosts", "group")
# Sources whose bare resource names are hosts ("<host>", parts are "<host>:…").
HOST_SOURCES = ("host_agent",)
# A host counts as present while it has reported within this long: the raw
# metrics window, so a decommissioned host's children go once its data does.
COVER_SEEN_DAYS = 7
def covers_text(rule: AlertRule) -> str:
"""What a covering rule spans, in words: "every host", "every host in web"."""
if rule.covers == "group":
return f"every host in {rule.cover_group}"
return "every host" if rule.covers == "all_hosts" else rule.resource_name
async def covered_hosts(db, rule: AlertRule, now: datetime | None = None) -> list[str]:
"""The host names a covering rule spans now: hosts that have reported
under its source within COVER_SEEN_DAYS, all of them or those whose
linked Ansible target is in its group."""
from datetime import timedelta
from steward.hosts.service import host_groups
from steward.models.hosts import Host
now = now or datetime.now(timezone.utc)
reported = set((await db.execute(
select(PluginMetricLatest.resource_name)
.where(PluginMetricLatest.source_module == rule.source_module,
PluginMetricLatest.recorded_at >= now - timedelta(days=COVER_SEEN_DAYS),
~PluginMetricLatest.resource_name.contains(":"))
.distinct())).scalars())
if rule.covers != "group":
return sorted(reported, key=str.lower)
hosts = (await db.execute(select(Host.id, Host.name).where(Host.name.in_(reported)))).all()
groups = await host_groups(db, [hid for hid, _ in hosts])
return sorted((name for hid, name in hosts if rule.cover_group in groups.get(hid, [])),
key=str.lower)
def child_fields(parent: AlertRule, host: str) -> dict:
"""A covering rule's child for one host: the parent's condition on that
host, named "<parent>: <host>"."""
return {"name": f"{parent.name}: {host}"[:128], "source_module": parent.source_module,
"resource_name": host, "metric_name": parent.metric_name,
"operator": parent.operator, "threshold": parent.threshold,
"consecutive_failures_required": parent.consecutive_failures_required,
"consecutive_ok_required": parent.consecutive_ok_required,
"notify_no_data": parent.notify_no_data}
async def reconcile_children(db, parent: AlertRule, *, actor_id: str | None = None,
now: datetime | None = None) -> tuple[list[str], list[str]]:
"""Bring a covering rule's children in line with it, in the caller's
transaction: one per covered host, each carrying the parent's condition,
action and on/off. A host newly covered gets a child; a child whose host
is no longer covered is deleted, its history with it. Returns (hosts
added, hosts removed)."""
hosts = set(await covered_hosts(db, parent, now))
children = {c.resource_name: c for c in (await db.execute(
select(AlertRule).where(AlertRule.parent_id == parent.id))).scalars()}
added = sorted(hosts - children.keys(), key=str.lower)
removed = sorted(children.keys() - hosts, key=str.lower)
for host in added:
await add_rule(db, {**child_fields(parent, host), "parent_id": parent.id},
created_by=parent.created_by, enabled=parent.enabled,
ansible_action=parent.ansible_action)
for host in removed:
await db.delete(children[host])
for host, child in children.items():
if host in hosts:
fields = child_fields(parent, host)
if (any(getattr(child, k) != v for k, v in fields.items())
or child.enabled != parent.enabled):
await apply_rule_change(db, child, fields, enabled=parent.enabled,
actor_id=actor_id)
if child.ansible_action != parent.ansible_action:
child.ansible_action = parent.ansible_action
return added, removed
async def reconcile_covering_rules(app) -> None:
"""Every covering rule's children brought in line with the hosts there are
now. Run each minute by the core scheduler (alert_reconcile), so a host
that joins a group, or starts reporting, is covered within a minute."""
async with app.db_sessionmaker() as db:
async with db.begin():
parents = (await db.execute(
select(AlertRule).where(AlertRule.covers != "resource"))).scalars().all()
for parent in parents:
added, removed = await reconcile_children(db, parent)
if added or removed:
logger.info("Covering rule %r: added %s, removed %s", parent.name,
", ".join(added) or "none", ", ".join(removed) or "none")
def part_of(rule: AlertRule, rules) -> AlertRule | None:
"""The covering rule a child belongs to, from a list of rules; None for a
rule that stands alone."""
return next((r for r in rules if r.id == rule.parent_id), None) if rule.parent_id else None
class PartOfCoveringRule(ValueError):
"""A child of a covering rule is changed through its parent, never alone."""
def __init__(self, child: AlertRule, parent_name: str):
super().__init__(f"{child.name} is part of the rule {parent_name!r}, which keeps one "
f"rule per host; change or disable it there.")
async def acknowledge(db, rule_id: str, user_id: str | None) -> AlertState:
"""Acknowledge a firing alert, in the caller's transaction, and record it
in the alert's history as a firing → acknowledged event by `user_id`, so