From 48a0de5600657b32bf83f0ad9ee39d8823fd566d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 9 Oct 2026 08:25:51 -0400 Subject: [PATCH] feat(alerts): one rule can cover every host, or every host in a group (#5464) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #5464 "Group rules: one rule covers every host, or every host in a group", step 2 of milestone 499 "Alerts you can trust". Each rule named one host, so the live instance carried six rules for three hosts, and a new host had no agent-offline or disk alert until someone copied them. - alert_rules.covers / cover_group / parent_id (migration 0041). A rule on host_agent may cover "all_hosts" or a "group" (an inventory group: the Hosts ruling, #5294). A covering rule has no state; Steward keeps one ordinary child rule per covered host, named ": ", so each host's alert keeps its own state, history and acknowledgement. - alerts.service.reconcile_children: a covered host that has reported in the last 7 days gets a child; a host no longer covered loses its child; children take the parent's condition, checks, notify setting, action and on/off. Run on save and each minute (alert_reconcile). - A child is changed through its parent: edit, toggle and delete of a child are refused (409 / PartOfCoveringRule); its edit page opens the parent. - Rule form: a Covers choice for host sources, with the group picker; fixed once the rule exists. Rules list: a covering rule reads "every host in swarm · 3 hosts" with its hosts' rules nested under it, each opening its history. The history picker and the rule count leave covering rules out. - MCP: create_alert_rule takes covers/group; update_alert_rule takes group and passes changes down; list_alert_rules shows covers, hosts and part_of. - docs/core/alerting.md: "Rules that cover hosts". Co-Authored-By: Claude Opus 5.5 --- docs/core/alerting.md | 23 +++ steward/alerts/routes.py | 78 +++++++- steward/alerts/service.py | 159 +++++++++++++++- steward/app.py | 6 +- steward/mcp/tools/alerts.py | 81 ++++++-- .../versions/0041_alert_rule_covers.py | 32 ++++ steward/models/alerts.py | 12 ++ steward/static/steward.css | 3 + steward/templates/alerts/_rule_fields.html | 39 +++- steward/templates/alerts/list.html | 28 ++- steward/templates/alerts/rules_form.html | 17 ++ tests/core/test_alerts_page.py | 35 +++- .../integration/test_alert_covering_rules.py | 173 ++++++++++++++++++ 13 files changed, 647 insertions(+), 39 deletions(-) create mode 100644 steward/migrations/versions/0041_alert_rule_covers.py create mode 100644 tests/integration/test_alert_covering_rules.py diff --git a/docs/core/alerting.md b/docs/core/alerting.md index 55872d9..3b7f504 100644 --- a/docs/core/alerting.md +++ b/docs/core/alerting.md @@ -75,6 +75,29 @@ Alert rules are created in the UI at `/alerts/`. Each rule requires: - **Threshold** — numeric value - **Consecutive failures required** — how many consecutive breaches before FIRING (default 1) +### Rules that cover hosts + +A rule on `host_agent` can cover **every host**, or **every host in an +inventory group**, instead of naming one resource (#5464). Inventory groups +are Steward's grouping system: a host is in a group through its linked +Ansible target. + +A covering rule has no state of its own. Steward keeps one ordinary rule per +covered host under it, named `: `, so each host's alert keeps its +own state, history and acknowledgement. Each minute (the `alert_reconcile` +task), and whenever the rule is saved: + +- a host that has reported in the last 7 days, and is covered, gets a rule; +- a host that is no longer covered loses its rule, and that rule's history; +- the hosts' rules take the covering rule's condition, checks, notification + setting, action and on/off. + +A host's rule is changed through its covering rule; editing, disabling or +deleting it alone is refused. What a rule covers is fixed once it exists, but +a group rule can move to another group. Over MCP, `create_alert_rule` takes +`covers` ("all_hosts" or "group") and `group`, and `list_alert_rules` shows +each covering rule's `hosts` and each host's rule's `part_of`. + ### Available Metrics by Module | `source_module` | `metric_name` | Description | diff --git a/steward/alerts/routes.py b/steward/alerts/routes.py index e1712ae..c49e49c 100644 --- a/steward/alerts/routes.py +++ b/steward/alerts/routes.py @@ -36,6 +36,8 @@ def _rule_form(form) -> dict: "consecutive_failures_required": form.get("consecutive_failures_required"), "consecutive_ok_required": form.get("consecutive_ok_required"), "notify_no_data": "notify_no_data" in form, + "covers": form.get("covers") or "resource", + "cover_group": form.get("cover_group", ""), } @@ -47,6 +49,15 @@ def _form_time(value) -> datetime: raise ValueError(f"{value!r} is not a date and time.") from None +async def _child_conflict(db, rule: AlertRule): + """A 409 for a host's rule under a covering rule, which is changed + through its parent (#5464); None for any other rule.""" + if not rule.parent_id: + return None + parent = await db.get(AlertRule, rule.parent_id) + return str(alert_svc.PartOfCoveringRule(rule, parent.name if parent else "?")), 409 + + def _is_admin() -> bool: return session.get("user_role") == UserRole.admin.value @@ -123,11 +134,27 @@ async def list_alerts(): states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()} fired = await alert_svc.last_fired(db) - # Every rule with what it is doing now; disabled rules last. - rules.sort(key=lambda r: (not r.enabled, r.name.lower())) + # Every rule with what it is doing now; disabled rules last. A covering + # rule's per-host children sit under it (#5464), and it reads as the most + # pressing of their states. rule_state = {r.id: alert_svc.alert_label(states[r.id]) for r in rules if r.id in states} + children: dict[str, list] = {} + for r in rules: + if r.parent_id: + children.setdefault(r.parent_id, []).append(r) + for kids in children.values(): + kids.sort(key=lambda r: r.resource_name.lower()) + parent_id = kids[0].parent_id + labels = [rule_state.get(k.id) for k in kids] + rule_state[parent_id] = min( + (lb for lb in labels if lb and lb not in ("inactive", "resolved")), + key=lambda lb: alert_svc.ACTIVE_ORDER.get(lb, 9), default="inactive") + fired[parent_id] = max((fired[k.id] for k in kids if k.id in fired), default=None) + rules = sorted((r for r in rules if not r.parent_id), + key=lambda r: (not r.enabled, r.name.lower())) return await render_template( - "alerts/list.html", active=active, rules=rules, rule_state=rule_state, last_fired=fired, + "alerts/list.html", active=active, rules=rules, children=children, + rule_state=rule_state, last_fired={k: v for k, v in fired.items() if v}, notify=await _notify(), operators=_OPERATORS, poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60)) @@ -138,7 +165,8 @@ async def active_fragment(): """The verdict and Active section alone, which the alerts page polls.""" async with current_app.db_sessionmaker() as db: active = await alert_svc.active_rows(db, datetime.now(timezone.utc)) - rules = await alert_svc.list_rules(db) + # Rules as people made them: a covering rule counts once, not per host. + rules = [r for r in await alert_svc.list_rules(db) if not r.parent_id] return await render_template( "alerts/_active.html", active=active, notify=await _notify(), rules_total=len(rules), rules_off=sum(1 for r in rules if not r.enabled)) @@ -195,7 +223,8 @@ async def alert_history(): since, range_key = parse_range(request.args.get("range") or "7d") before = request.args.get("before") async with current_app.db_sessionmaker() as db: - rules = await alert_svc.list_rules(db) + # A covering rule has no history of its own; its hosts' rules do. + rules = [r for r in await alert_svc.list_rules(db) if r.covers == "resource"] chosen = next((r for r in rules if r.id == request.args.get("rule")), None) rule_id = chosen.id if chosen else None rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before) @@ -232,26 +261,40 @@ async def rule_fields_api(): current_metric = request.args.get("current_metric", "") async with current_app.db_sessionmaker() as db: resources = await alert_svc.resources_for(db, source) if source else [] - async with current_app.db_sessionmaker() as db: metrics = await alert_svc.metrics_for(db, source) + cover = await _cover_choices(db, source) return await render_template( "alerts/_rule_fields.html", resources=resources, metrics=metrics, current_resource=current_resource, current_metric=current_metric, + **cover, ) +async def _cover_choices(db, source: str, rule: AlertRule | None = None) -> dict: + """What the rule form offers for covering hosts (#5464): whether this + source's resources are hosts, the inventory groups, and the rule's own + choice, fixed once it exists.""" + from steward.hosts.service import group_names + host_source = source in alert_svc.HOST_SOURCES + return {"host_source": host_source, "groups": await group_names(db) if host_source else [], + "current_covers": rule.covers if rule else "resource", + "current_group": (rule.cover_group or "") if rule else "", + "covers_locked": rule is not None} + + @alerts_bp.get("/rules/new") @require_role(UserRole.operator) async def new_rule(): first_source = alert_svc.source_modules()[0] async with current_app.db_sessionmaker() as db: resources = await alert_svc.resources_for(db, first_source) + cover = await _cover_choices(db, first_source) return await render_template( "alerts/rules_form.html", - rule=None, + rule=None, **cover, operators=_OPERATORS, source_modules=alert_svc.source_modules(), initial_source=first_source, @@ -271,13 +314,20 @@ async def edit_rule(rule_id: str): rule = result.scalar_one_or_none() if rule is None: return "Not found", 404 + if rule.parent_id: + # A host's rule under a covering rule is changed through it. + return redirect(f"/alerts/rules/{rule.parent_id}/edit") resources = await alert_svc.resources_for(db, rule.source_module) + cover = await _cover_choices(db, rule.source_module, rule) + hosts = sorted((c.resource_name for c in (await db.execute( + select(AlertRule).where(AlertRule.parent_id == rule.id))).scalars()), key=str.lower) history_count = (await db.execute( select(func.count()).select_from(AlertEvent).where(AlertEvent.rule_id == rule.id))).scalar_one() fired = (await alert_svc.last_fired(db, rule.id)).get(rule.id) return await render_template( "alerts/rules_form.html", rule=rule, history_count=history_count, fired=fired, notify=await _notify(), + covered=hosts, **cover, operators=_OPERATORS, source_modules=alert_svc.source_modules(), initial_source=rule.source_module, @@ -331,13 +381,19 @@ async def update_rule(rule_id: str): rule = result.scalar_one_or_none() if rule is None: return "Not found", 404 + if (conflict := await _child_conflict(db, rule)): + return conflict try: - fields = await alert_svc.rule_fields(db, **_rule_form(form)) + # What a rule covers is fixed once it exists; its group may change. + fields = await alert_svc.rule_fields(db, **{**_rule_form(form), + "covers": rule.covers}) except ValueError as exc: return str(exc), 400 await alert_svc.apply_rule_change(db, rule, fields, actor_id=session.get("user_id")) if apply_action: rule.ansible_action = ansible_action + if rule.covers != "resource": + await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id")) await log_audit(current_app, session.get("user_id"), session.get("username", ""), "alert_rule.updated", entity_type="alert_rule", entity_id=rule.name) return redirect(url_for("alerts.list_alerts")) @@ -352,8 +408,12 @@ async def toggle_rule(rule_id: str): rule = result.scalar_one_or_none() if rule is None: return "Not found", 404 + if (conflict := await _child_conflict(db, rule)): + return conflict await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled, actor_id=session.get("user_id")) + if rule.covers != "resource": + await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id")) action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled" await log_audit(current_app, session.get("user_id"), session.get("username", ""), action, entity_type="alert_rule", entity_id=rule.name) @@ -369,6 +429,8 @@ async def delete_rule(rule_id: str): result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id)) rule = result.scalar_one_or_none() if rule: + if (conflict := await _child_conflict(db, rule)): + return conflict rule_name = rule.name await db.delete(rule) if rule_name: diff --git a/steward/alerts/service.py b/steward/alerts/service.py index cdd1e62..f591c70 100644 --- a/steward/alerts/service.py +++ b/steward/alerts/service.py @@ -378,12 +378,16 @@ def _choices(values: list[str], limit: int = 20) -> str: async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str, operator: str, threshold, consecutive_failures_required=1, - consecutive_ok_required=1, notify_no_data=True) -> dict: + consecutive_ok_required=1, notify_no_data=True, + covers: str = "resource", cover_group: str = "") -> dict: """Validated column values for a new or edited alert rule. The same choices the rule form offers: a known source, a resource that source has reported (or, for monitors, a monitor of that type), and a - metric it emits. ValueError says what was wrong and what is valid. + metric it emits. A covering rule (`covers` "all_hosts", or "group" with + an inventory group's name in `cover_group`, #5464) names no resource: it + is for a source whose resources are hosts. ValueError says what was wrong + and what is valid. """ name = (name or "").strip() if not name: @@ -392,11 +396,29 @@ async def rule_fields(db, *, name: str, source_module: str, resource_name: str, catalog = metric_catalog() if source_module not in catalog: raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.") - resource_name = (resource_name or "").strip() - resources = await resources_for(db, source_module) - if resource_name not in resources: - raise ValueError(f"{source_module} has no resource {resource_name!r}. " - f"Known: {_choices(resources)}.") + covers = (covers or "resource").strip() + if covers not in COVERS: + raise ValueError(f"covers {covers!r} is not one of {', '.join(COVERS)}.") + cover_group = (cover_group or "").strip() + if covers == "resource": + resource_name, cover_group = (resource_name or "").strip(), None + resources = await resources_for(db, source_module) + if resource_name not in resources: + raise ValueError(f"{source_module} has no resource {resource_name!r}. " + f"Known: {_choices(resources)}.") + else: + if source_module not in HOST_SOURCES: + raise ValueError(f"only a rule on {', '.join(HOST_SOURCES)} can cover hosts; " + f"{source_module}'s resources are not hosts.") + resource_name = "" + if covers == "group": + from steward.hosts.service import group_names + groups = await group_names(db) + if cover_group not in groups: + raise ValueError(f"there is no inventory group {cover_group!r}. " + f"Groups: {_choices(groups)}.") + else: + cover_group = None metric_name = (metric_name or "").strip() metrics = await metrics_for(db, source_module) if metric_name not in metrics: @@ -428,7 +450,8 @@ async def rule_fields(db, *, name: str, source_module: str, resource_name: str, return {"name": name[:128], "source_module": source_module, "resource_name": resource_name, "metric_name": metric_name, "operator": op, "threshold": threshold, "consecutive_failures_required": consecutive, - "consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data)} + "consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data), + "covers": covers, "cover_group": cover_group} # Changing any of these changes what the rule is watching, so whatever state @@ -460,15 +483,133 @@ async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True, ansible_action: dict | None = None) -> AlertRule: - """Insert a rule and its inactive state row, in the caller's transaction.""" + """Insert a rule and its inactive state row, in the caller's transaction. + A covering rule gets no state; its per-host children are made here.""" rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields) db.add(rule) await db.flush() + if rule.covers != "resource": + await reconcile_children(db, rule) + return rule db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive, last_transition_at=datetime.now(timezone.utc))) return rule +# ── Covering rules (#5464) ─────────────────────────────────────────────────── +# A rule can cover every host, or every host in an inventory group (inventory +# groups are Steward's grouping system: the Hosts ruling, #5294). Steward keeps +# one ordinary child rule per covered host, so each host's alert has its own +# state, history and acknowledgement, and a host that appears later is covered +# without anyone copying the rule. + +COVERS = ("resource", "all_hosts", "group") +# Sources whose bare resource names are hosts ("", parts are ":…"). +HOST_SOURCES = ("host_agent",) +# A host counts as present while it has reported within this long: the raw +# metrics window, so a decommissioned host's children go once its data does. +COVER_SEEN_DAYS = 7 + + +def covers_text(rule: AlertRule) -> str: + """What a covering rule spans, in words: "every host", "every host in web".""" + if rule.covers == "group": + return f"every host in {rule.cover_group}" + return "every host" if rule.covers == "all_hosts" else rule.resource_name + + +async def covered_hosts(db, rule: AlertRule, now: datetime | None = None) -> list[str]: + """The host names a covering rule spans now: hosts that have reported + under its source within COVER_SEEN_DAYS, all of them or those whose + linked Ansible target is in its group.""" + from datetime import timedelta + from steward.hosts.service import host_groups + from steward.models.hosts import Host + now = now or datetime.now(timezone.utc) + reported = set((await db.execute( + select(PluginMetricLatest.resource_name) + .where(PluginMetricLatest.source_module == rule.source_module, + PluginMetricLatest.recorded_at >= now - timedelta(days=COVER_SEEN_DAYS), + ~PluginMetricLatest.resource_name.contains(":")) + .distinct())).scalars()) + if rule.covers != "group": + return sorted(reported, key=str.lower) + hosts = (await db.execute(select(Host.id, Host.name).where(Host.name.in_(reported)))).all() + groups = await host_groups(db, [hid for hid, _ in hosts]) + return sorted((name for hid, name in hosts if rule.cover_group in groups.get(hid, [])), + key=str.lower) + + +def child_fields(parent: AlertRule, host: str) -> dict: + """A covering rule's child for one host: the parent's condition on that + host, named ": ".""" + return {"name": f"{parent.name}: {host}"[:128], "source_module": parent.source_module, + "resource_name": host, "metric_name": parent.metric_name, + "operator": parent.operator, "threshold": parent.threshold, + "consecutive_failures_required": parent.consecutive_failures_required, + "consecutive_ok_required": parent.consecutive_ok_required, + "notify_no_data": parent.notify_no_data} + + +async def reconcile_children(db, parent: AlertRule, *, actor_id: str | None = None, + now: datetime | None = None) -> tuple[list[str], list[str]]: + """Bring a covering rule's children in line with it, in the caller's + transaction: one per covered host, each carrying the parent's condition, + action and on/off. A host newly covered gets a child; a child whose host + is no longer covered is deleted, its history with it. Returns (hosts + added, hosts removed).""" + hosts = set(await covered_hosts(db, parent, now)) + children = {c.resource_name: c for c in (await db.execute( + select(AlertRule).where(AlertRule.parent_id == parent.id))).scalars()} + added = sorted(hosts - children.keys(), key=str.lower) + removed = sorted(children.keys() - hosts, key=str.lower) + for host in added: + await add_rule(db, {**child_fields(parent, host), "parent_id": parent.id}, + created_by=parent.created_by, enabled=parent.enabled, + ansible_action=parent.ansible_action) + for host in removed: + await db.delete(children[host]) + for host, child in children.items(): + if host in hosts: + fields = child_fields(parent, host) + if (any(getattr(child, k) != v for k, v in fields.items()) + or child.enabled != parent.enabled): + await apply_rule_change(db, child, fields, enabled=parent.enabled, + actor_id=actor_id) + if child.ansible_action != parent.ansible_action: + child.ansible_action = parent.ansible_action + return added, removed + + +async def reconcile_covering_rules(app) -> None: + """Every covering rule's children brought in line with the hosts there are + now. Run each minute by the core scheduler (alert_reconcile), so a host + that joins a group, or starts reporting, is covered within a minute.""" + async with app.db_sessionmaker() as db: + async with db.begin(): + parents = (await db.execute( + select(AlertRule).where(AlertRule.covers != "resource"))).scalars().all() + for parent in parents: + added, removed = await reconcile_children(db, parent) + if added or removed: + logger.info("Covering rule %r: added %s, removed %s", parent.name, + ", ".join(added) or "none", ", ".join(removed) or "none") + + +def part_of(rule: AlertRule, rules) -> AlertRule | None: + """The covering rule a child belongs to, from a list of rules; None for a + rule that stands alone.""" + return next((r for r in rules if r.id == rule.parent_id), None) if rule.parent_id else None + + +class PartOfCoveringRule(ValueError): + """A child of a covering rule is changed through its parent, never alone.""" + + def __init__(self, child: AlertRule, parent_name: str): + super().__init__(f"{child.name} is part of the rule {parent_name!r}, which keeps one " + f"rule per host; change or disable it there.") + + async def acknowledge(db, rule_id: str, user_id: str | None) -> AlertState: """Acknowledge a firing alert, in the caller's transaction, and record it in the alert's history as a firing → acknowledged event by `user_id`, so diff --git a/steward/app.py b/steward/app.py index 9732f02..9310215 100644 --- a/steward/app.py +++ b/steward/app.py @@ -408,8 +408,9 @@ def _register_core_tasks(app: Quart) -> None: await _cleanup(app) async def run_alert_reconcile(): + from .alerts.service import reconcile_covering_rules from .core.alerts import check_silence, notify_after_maintenance - for step in (check_silence, notify_after_maintenance): + for step in (reconcile_covering_rules, check_silence, notify_after_maintenance): try: await step(app) except Exception: @@ -439,7 +440,8 @@ def _register_core_tasks(app: Quart) -> None: interval_seconds=60, run_on_startup=True, ), - # Rules whose series has gone quiet get a visible no_data mark; alerts + # Covering rules gain a child for each newly covered host (#5464); + # rules whose series has gone quiet get a visible no_data mark; alerts # a maintenance window muted notify once it ends. ScheduledTask( name="alert_reconcile", diff --git a/steward/mcp/tools/alerts.py b/steward/mcp/tools/alerts.py index 3a3ceec..4718ef8 100644 --- a/steward/mcp/tools/alerts.py +++ b/steward/mcp/tools/alerts.py @@ -134,6 +134,11 @@ async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict if it has no state row. `no_data_since` is set while its metric is silent, whatever the state. `ansible_action` is the playbook the rule runs when it fires, if any (`check: true` = dry run). + + `covers` is "resource" for a rule on one resource, or "all_hosts" / + "group" (with `group`) for a rule covering hosts: it has no state of its + own, `hosts` names the hosts it covers, and each host's rule is listed + with `part_of` naming it. """ limit = clamp_limit(limit, 200, 500) async with db_session() as db: @@ -141,13 +146,19 @@ async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()} if enabled_only: rules = [r for r in rules if r.enabled] + names = {r.id: r.name for r in rules} + hosts: dict[str, list[str]] = {} + for r in rules: + if r.parent_id: + hosts.setdefault(r.parent_id, []).append(r.resource_name) rows = [ { "rule_id": r.id, "rule_name": r.name, "enabled": r.enabled, "source_module": r.source_module, - "resource": r.resource_name, + "resource": r.resource_name or None, + **_covers(r, hosts.get(r.id, []), names.get(r.parent_id)), "metric": r.metric_name, "condition": _condition(r), "state": alert_svc.alert_label(states[r.id]) if r.id in states else None, @@ -219,21 +230,51 @@ async def _rule(db, rule: str) -> AlertRule: return pick(rule, await alert_svc.list_rules(db), kind="alert rule", list_tool="list_alert_rules") -def _rule_row(r: AlertRule) -> dict: +def _covers(r: AlertRule, hosts: list[str], parent_name: str | None) -> dict: + """What a rule covers (#5464): the hosts under a covering rule, or the + covering rule a host's rule is part of.""" + out: dict = {"covers": r.covers} + if r.covers == "group": + out["group"] = r.cover_group + if r.covers != "resource": + out["hosts"] = sorted(hosts, key=str.lower) + if r.parent_id: + out["part_of"] = parent_name + return out + + +async def _rule_row(db, r: AlertRule) -> dict: + hosts = [c.resource_name for c in (await db.execute( + select(AlertRule).where(AlertRule.parent_id == r.id))).scalars()] if r.covers != "resource" else [] + parent = await db.get(AlertRule, r.parent_id) if r.parent_id else None return {"rule_id": r.id, "rule_name": r.name, "enabled": r.enabled, - "source_module": r.source_module, "resource": r.resource_name, + "source_module": r.source_module, "resource": r.resource_name or None, + **_covers(r, hosts, parent.name if parent else None), "metric": r.metric_name, "condition": _condition(r), "notify_no_data": r.notify_no_data, "ansible_action": _action(r)} +async def _not_a_child(db, r: AlertRule) -> None: + """A host's rule under a covering rule is changed through its parent.""" + if r.parent_id: + parent = await db.get(AlertRule, r.parent_id) + raise alert_svc.PartOfCoveringRule(r, parent.name if parent else "?") + + async def create_alert_rule(name: str, source_module: str, resource: str, metric: str, operator: str, threshold: float, consecutive_checks: int = 1, clear_after_checks: int = 1, notify_no_data: bool = True, - enabled: bool = True) -> dict: + enabled: bool = True, covers: str = "resource", + group: str = "") -> dict: """Add an alert rule: notify when `metric` of `resource` crosses `threshold`. source_module, resource, metric: get_alert_rule_options lists the valid choices; a resource must have reported (or, for a monitor, exist). + covers: "resource" (default) watches `resource`. "all_hosts" covers every + host, "group" every host in the inventory group `group`; pass + `resource` as "". Steward then keeps one rule per host, named + ": ", and adds one within a minute for a host that starts + reporting or joins the group. Only for host_agent. operator: ">", "<", ">=", "<=", "==" or "!=". consecutive_checks: how many checks in a row must breach before it fires (default 1); higher values ride out a single blip. @@ -254,12 +295,15 @@ async def create_alert_rule(name: str, source_module: str, resource: str, metric db, name=name, source_module=source_module, resource_name=resource, metric_name=metric, operator=operator, threshold=threshold, consecutive_failures_required=consecutive_checks, - consecutive_ok_required=clear_after_checks, notify_no_data=notify_no_data) + consecutive_ok_required=clear_after_checks, notify_no_data=notify_no_data, + covers=covers, cover_group=group) r = await alert_svc.add_rule(db, fields, created_by=caller.user_id, enabled=enabled) + row = await _rule_row(db, r) await audit_mcp("alert_rule.created", "alert_rule", r.name, - {"rule_id": r.id, "source": r.source_module, "resource": r.resource_name, + {"rule_id": r.id, "source": r.source_module, + "resource": r.resource_name or alert_svc.covers_text(r), "metric": r.metric_name}) - return _rule_row(r) + return row async def update_alert_rule(rule: str, name: str | None = None, source_module: str | None = None, @@ -268,7 +312,7 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s consecutive_checks: int | None = None, clear_after_checks: int | None = None, notify_no_data: bool | None = None, - enabled: bool | None = None) -> dict: + enabled: bool | None = None, group: str | None = None) -> dict: """Change an alert rule. Only the arguments you pass change; the rest stay. rule: its name or id (list_alert_rules). Pass enabled=false to silence it @@ -277,9 +321,15 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s operator, threshold), or disabling or re-enabling it, puts its alert back to inactive; a firing alert records a resolved event saying why. Recorded in the audit log. + + A rule covering hosts passes every change on to its per-host rules; + `group` moves a group-covering rule to another inventory group. What a + rule covers is fixed once it exists. A host's rule under a covering rule + is changed through that rule, not alone. """ async with db_session() as db, db.begin(): r = await _rule(db, rule) + await _not_a_child(db, r) fields = await alert_svc.rule_fields( db, name=r.name if name is None else name, @@ -292,24 +342,31 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s if consecutive_checks is None else consecutive_checks), consecutive_ok_required=(r.consecutive_ok_required if clear_after_checks is None else clear_after_checks), - notify_no_data=r.notify_no_data if notify_no_data is None else notify_no_data) + notify_no_data=r.notify_no_data if notify_no_data is None else notify_no_data, + covers=r.covers, cover_group=(r.cover_group or "") if group is None else group) await alert_svc.apply_rule_change(db, r, fields, enabled=enabled, actor_id=current_caller().user_id) + if r.covers != "resource": + await alert_svc.reconcile_children(db, r, actor_id=current_caller().user_id) + row = await _rule_row(db, r) changed = sorted(k for k, v in {"name": name, "source_module": source_module, "resource": resource, "metric": metric, "operator": operator, "threshold": threshold, "consecutive_checks": consecutive_checks, "clear_after_checks": clear_after_checks, "notify_no_data": notify_no_data, - "enabled": enabled}.items() if v is not None) + "enabled": enabled, "group": group}.items() if v is not None) await audit_mcp("alert_rule.updated", "alert_rule", r.name, {"rule_id": r.id, "changed": changed}) - return _rule_row(r) + return row async def delete_alert_rule(rule: str) -> dict: """Delete an alert rule and its history. To silence it but keep it, - update_alert_rule(enabled=false). Admin only; recorded in the audit log.""" + update_alert_rule(enabled=false). Deleting a rule that covers hosts + deletes its per-host rules and their history too. Admin only; recorded + in the audit log.""" async with db_session() as db, db.begin(): r = await _rule(db, rule) + await _not_a_child(db, r) await db.delete(r) await audit_mcp("alert_rule.deleted", "alert_rule", r.name, {"rule_id": r.id}) return {"deleted": r.name, "rule_id": r.id} diff --git a/steward/migrations/versions/0041_alert_rule_covers.py b/steward/migrations/versions/0041_alert_rule_covers.py new file mode 100644 index 0000000..7364776 --- /dev/null +++ b/steward/migrations/versions/0041_alert_rule_covers.py @@ -0,0 +1,32 @@ +"""A rule can cover every host, or every host in an inventory group (#5464): +the covering rule keeps one child rule per host under it. + +Revision ID: 0041_alert_rule_covers +Revises: 0040_alert_event_actor +Create Date: 2026-10-09 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "0041_alert_rule_covers" +down_revision: Union[str, None] = "0040_alert_event_actor" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("alert_rules", sa.Column("covers", sa.String(16), nullable=False, + server_default="resource")) + op.add_column("alert_rules", sa.Column("cover_group", sa.String(128), nullable=True)) + op.add_column("alert_rules", sa.Column( + "parent_id", sa.String(36), sa.ForeignKey("alert_rules.id", ondelete="CASCADE"), + nullable=True)) + op.create_index("ix_alert_rules_parent_id", "alert_rules", ["parent_id"]) + + +def downgrade() -> None: + op.drop_index("ix_alert_rules_parent_id", table_name="alert_rules") + op.drop_column("alert_rules", "parent_id") + op.drop_column("alert_rules", "cover_group") + op.drop_column("alert_rules", "covers") diff --git a/steward/models/alerts.py b/steward/models/alerts.py index 6bf5702..0b4e0ce 100644 --- a/steward/models/alerts.py +++ b/steward/models/alerts.py @@ -55,6 +55,18 @@ class AlertRule(Base): created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc) ) + # What the rule covers (#5464): "resource" watches resource_name itself; + # "all_hosts" and "group" make it a covering rule over hosts (every host, + # or every host in the inventory group cover_group). A covering rule has + # no state and is never evaluated: Steward keeps one ordinary child rule + # per covered host (parent_id), so each host's alert keeps its own state, + # history and acknowledgement. Fixed when the rule is created. + covers: Mapped[str] = mapped_column(String(16), nullable=False, default="resource", + server_default="resource") + cover_group: Mapped[str | None] = mapped_column(String(128), nullable=True) + parent_id: Mapped[str | None] = mapped_column( + String(36), ForeignKey("alert_rules.id", ondelete="CASCADE"), nullable=True, index=True + ) class AlertState(Base): diff --git a/steward/static/steward.css b/steward/static/steward.css index 7bb1fba..7ddabf9 100644 --- a/steward/static/steward.css +++ b/steward/static/steward.css @@ -216,6 +216,9 @@ strong, b, th { font-weight: 500; } /* The Docker page's one-line answer, above its tiles (#5298). */ .page-verdict { margin: 0 0 0.75rem; font-size: 0.95rem; } .table tr.row-off > td { opacity: 0.55; } +/* A covering alert rule's per-host rules (#5464), indented under it. */ +.table tr.row-child > td { border-top-style: dashed; } +.table tr.row-child .cell-name { padding-left: 2rem; } /* A row that opens as a whole: its one .row-target link stretches over the row, and the actions cell sits above it so its buttons keep their own clicks. One real link per row, so keyboard and screen readers meet it once. */ diff --git a/steward/templates/alerts/_rule_fields.html b/steward/templates/alerts/_rule_fields.html index f3f3a44..7f8e11d 100644 --- a/steward/templates/alerts/_rule_fields.html +++ b/steward/templates/alerts/_rule_fields.html @@ -1,16 +1,49 @@ {# alerts/_rule_fields.html — the resource and metric pickers for one source; - the rule form includes it and swaps it in again when the source changes. #} + the rule form includes it and swaps it in again when the source changes. + For a source whose resources are hosts, the rule can instead cover every + host, or every host in an inventory group (#5464): Steward then keeps one + rule per host under it. What a rule covers is fixed once it exists. #} +{% set covers = current_covers|default("resource") %} +{% set one = covers == "resource" %} +{% if host_source|default(false) %}
+ + {% if covers_locked|default(false) %} + +

{% if one %}One host, or one part of a host{% elif covers == "all_hosts" %}Every host{% else %}Every host in a group{% endif %} + · fixed once the rule exists

+ {% else %} + + {% endif %} +

Covering hosts keeps one alert per host under this rule, each named + "rule: host". A host that starts reporting, or joins the group, is covered within a minute; + one that leaves loses its alert and that alert's history.

+
+
+ + +

A host is in a group through its linked Ansible target (Ansible → Inventory).

+
+{% endif %} +
{% if resources %} - {% if not current_resource %}{% endif %} {% for r in resources %} {% endfor %} {% else %} -

diff --git a/steward/templates/alerts/list.html b/steward/templates/alerts/list.html index 27ac079..715b74b 100644 --- a/steward/templates/alerts/list.html +++ b/steward/templates/alerts/list.html @@ -2,7 +2,8 @@ (alerts/_active.html, polled), then every rule: what it is doing now and when it last fired, what it fires on, and its History and Disable. The whole row opens the rule (.row-link, its name is the link); deleting one is - on its own page, behind the confirm dialog. #} + on its own page, behind the confirm dialog. A rule covering hosts (#5464) + lists its per-host rules under it; each opens its history. #} {% extends "base.html" %} {% block title %}Alerts — Steward{% endblock %} {% from "_macros.html" import page_header, status_badge, empty_state %} @@ -36,11 +37,13 @@ {% for rule in rules %} + {% set kids = (children or {}).get(rule.id, []) %} {% set fired = (last_fired or {}).get(rule.id) %} {{ rule.name }} -

{{ source_label(rule.source_module) }}
+
{{ source_label(rule.source_module) }} + {%- if rule.covers != "resource" %} · {{ kids|length }} host{{ "s" if kids|length != 1 }}{% endif %}
{%- set now_label = (rule_state or {}).get(rule.id) %} @@ -50,7 +53,7 @@
{% if fired %}last fired {{ fired|ago }}{% else %}never fired{% endif %}
- {{ rule.resource_name }} · {{ rule_condition(rule) }} + {% if rule.covers == "group" %}every host in {{ rule.cover_group }}{% elif rule.covers == "all_hosts" %}every host{% else %}{{ rule.resource_name }}{% endif %} · {{ rule_condition(rule) }}
{%- set n_fire, n_ok = rule.consecutive_failures_required, rule.consecutive_ok_required %} {%- if n_fire > 1 %}after {{ n_fire }} checks in a row{% else %}on the first check{% endif %} @@ -58,7 +61,7 @@
- History + {% if rule.covers == "resource" %}History{% endif %} {% if can_edit %}
{{ csrf_field() }}