feat(alerts): one rule can cover every host, or every host in a group (#5464)

#5464 "Group rules: one rule covers every host, or every host in a group",
step 2 of milestone 499 "Alerts you can trust". Each rule named one host, so
the live instance carried six rules for three hosts, and a new host had no
agent-offline or disk alert until someone copied them.

- alert_rules.covers / cover_group / parent_id (migration 0041). A rule on
  host_agent may cover "all_hosts" or a "group" (an inventory group: the
  Hosts ruling, #5294). A covering rule has no state; Steward keeps one
  ordinary child rule per covered host, named "<rule>: <host>", so each
  host's alert keeps its own state, history and acknowledgement.
- alerts.service.reconcile_children: a covered host that has reported in
  the last 7 days gets a child; a host no longer covered loses its child;
  children take the parent's condition, checks, notify setting, action and
  on/off. Run on save and each minute (alert_reconcile).
- A child is changed through its parent: edit, toggle and delete of a child
  are refused (409 / PartOfCoveringRule); its edit page opens the parent.
- Rule form: a Covers choice for host sources, with the group picker; fixed
  once the rule exists. Rules list: a covering rule reads "every host in
  swarm · 3 hosts" with its hosts' rules nested under it, each opening its
  history. The history picker and the rule count leave covering rules out.
- MCP: create_alert_rule takes covers/group; update_alert_rule takes group
  and passes changes down; list_alert_rules shows covers, hosts and part_of.
- docs/core/alerting.md: "Rules that cover hosts".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/routes.py

@@ -36,6 +36,8 @@ def _rule_form(form) -> dict:
         "consecutive_failures_required": form.get("consecutive_failures_required"),
         "consecutive_ok_required": form.get("consecutive_ok_required"),
         "notify_no_data": "notify_no_data" in form,
+        "covers": form.get("covers") or "resource",
+        "cover_group": form.get("cover_group", ""),
     }
 
 
@@ -47,6 +49,15 @@ def _form_time(value) -> datetime:
         raise ValueError(f"{value!r} is not a date and time.") from None
 
 
+async def _child_conflict(db, rule: AlertRule):
+    """A 409 for a host's rule under a covering rule, which is changed
+    through its parent (#5464); None for any other rule."""
+    if not rule.parent_id:
+        return None
+    parent = await db.get(AlertRule, rule.parent_id)
+    return str(alert_svc.PartOfCoveringRule(rule, parent.name if parent else "?")), 409
+
+
 def _is_admin() -> bool:
     return session.get("user_role") == UserRole.admin.value
 
@@ -123,11 +134,27 @@ async def list_alerts():
         states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
         fired = await alert_svc.last_fired(db)
 
-    # Every rule with what it is doing now; disabled rules last.
-    rules.sort(key=lambda r: (not r.enabled, r.name.lower()))
+    # Every rule with what it is doing now; disabled rules last. A covering
+    # rule's per-host children sit under it (#5464), and it reads as the most
+    # pressing of their states.
     rule_state = {r.id: alert_svc.alert_label(states[r.id]) for r in rules if r.id in states}
+    children: dict[str, list] = {}
+    for r in rules:
+        if r.parent_id:
+            children.setdefault(r.parent_id, []).append(r)
+    for kids in children.values():
+        kids.sort(key=lambda r: r.resource_name.lower())
+        parent_id = kids[0].parent_id
+        labels = [rule_state.get(k.id) for k in kids]
+        rule_state[parent_id] = min(
+            (lb for lb in labels if lb and lb not in ("inactive", "resolved")),
+            key=lambda lb: alert_svc.ACTIVE_ORDER.get(lb, 9), default="inactive")
+        fired[parent_id] = max((fired[k.id] for k in kids if k.id in fired), default=None)
+    rules = sorted((r for r in rules if not r.parent_id),
+                   key=lambda r: (not r.enabled, r.name.lower()))
     return await render_template(
-        "alerts/list.html", active=active, rules=rules, rule_state=rule_state, last_fired=fired,
+        "alerts/list.html", active=active, rules=rules, children=children,
+        rule_state=rule_state, last_fired={k: v for k, v in fired.items() if v},
         notify=await _notify(), operators=_OPERATORS,
         poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60))
 
@@ -138,7 +165,8 @@ async def active_fragment():
     """The verdict and Active section alone, which the alerts page polls."""
     async with current_app.db_sessionmaker() as db:
         active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
-        rules = await alert_svc.list_rules(db)
+        # Rules as people made them: a covering rule counts once, not per host.
+        rules = [r for r in await alert_svc.list_rules(db) if not r.parent_id]
     return await render_template(
         "alerts/_active.html", active=active, notify=await _notify(), rules_total=len(rules),
         rules_off=sum(1 for r in rules if not r.enabled))
@@ -195,7 +223,8 @@ async def alert_history():
     since, range_key = parse_range(request.args.get("range") or "7d")
     before = request.args.get("before")
     async with current_app.db_sessionmaker() as db:
-        rules = await alert_svc.list_rules(db)
+        # A covering rule has no history of its own; its hosts' rules do.
+        rules = [r for r in await alert_svc.list_rules(db) if r.covers == "resource"]
         chosen = next((r for r in rules if r.id == request.args.get("rule")), None)
         rule_id = chosen.id if chosen else None
         rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before)
@@ -232,26 +261,40 @@ async def rule_fields_api():
     current_metric = request.args.get("current_metric", "")
     async with current_app.db_sessionmaker() as db:
         resources = await alert_svc.resources_for(db, source) if source else []
-    async with current_app.db_sessionmaker() as db:
         metrics = await alert_svc.metrics_for(db, source)
+        cover = await _cover_choices(db, source)
     return await render_template(
         "alerts/_rule_fields.html",
         resources=resources,
         metrics=metrics,
         current_resource=current_resource,
         current_metric=current_metric,
+        **cover,
     )
 
 
+async def _cover_choices(db, source: str, rule: AlertRule | None = None) -> dict:
+    """What the rule form offers for covering hosts (#5464): whether this
+    source's resources are hosts, the inventory groups, and the rule's own
+    choice, fixed once it exists."""
+    from steward.hosts.service import group_names
+    host_source = source in alert_svc.HOST_SOURCES
+    return {"host_source": host_source, "groups": await group_names(db) if host_source else [],
+            "current_covers": rule.covers if rule else "resource",
+            "current_group": (rule.cover_group or "") if rule else "",
+            "covers_locked": rule is not None}
+
+
 @alerts_bp.get("/rules/new")
 @require_role(UserRole.operator)
 async def new_rule():
     first_source = alert_svc.source_modules()[0]
     async with current_app.db_sessionmaker() as db:
         resources = await alert_svc.resources_for(db, first_source)
+        cover = await _cover_choices(db, first_source)
     return await render_template(
         "alerts/rules_form.html",
-        rule=None,
+        rule=None, **cover,
         operators=_OPERATORS,
         source_modules=alert_svc.source_modules(),
         initial_source=first_source,
@@ -271,13 +314,20 @@ async def edit_rule(rule_id: str):
         rule = result.scalar_one_or_none()
         if rule is None:
             return "Not found", 404
+        if rule.parent_id:
+            # A host's rule under a covering rule is changed through it.
+            return redirect(f"/alerts/rules/{rule.parent_id}/edit")
         resources = await alert_svc.resources_for(db, rule.source_module)
+        cover = await _cover_choices(db, rule.source_module, rule)
+        hosts = sorted((c.resource_name for c in (await db.execute(
+            select(AlertRule).where(AlertRule.parent_id == rule.id))).scalars()), key=str.lower)
         history_count = (await db.execute(
             select(func.count()).select_from(AlertEvent).where(AlertEvent.rule_id == rule.id))).scalar_one()
         fired = (await alert_svc.last_fired(db, rule.id)).get(rule.id)
     return await render_template(
         "alerts/rules_form.html",
         rule=rule, history_count=history_count, fired=fired, notify=await _notify(),
+        covered=hosts, **cover,
         operators=_OPERATORS,
         source_modules=alert_svc.source_modules(),
         initial_source=rule.source_module,
@@ -331,13 +381,19 @@ async def update_rule(rule_id: str):
             rule = result.scalar_one_or_none()
             if rule is None:
                 return "Not found", 404
+            if (conflict := await _child_conflict(db, rule)):
+                return conflict
             try:
-                fields = await alert_svc.rule_fields(db, **_rule_form(form))
+                # What a rule covers is fixed once it exists; its group may change.
+                fields = await alert_svc.rule_fields(db, **{**_rule_form(form),
+                                                            "covers": rule.covers})
             except ValueError as exc:
                 return str(exc), 400
             await alert_svc.apply_rule_change(db, rule, fields, actor_id=session.get("user_id"))
             if apply_action:
                 rule.ansible_action = ansible_action
+            if rule.covers != "resource":
+                await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id"))
     await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                     "alert_rule.updated", entity_type="alert_rule", entity_id=rule.name)
     return redirect(url_for("alerts.list_alerts"))
@@ -352,8 +408,12 @@ async def toggle_rule(rule_id: str):
             rule = result.scalar_one_or_none()
             if rule is None:
                 return "Not found", 404
+            if (conflict := await _child_conflict(db, rule)):
+                return conflict
             await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled,
                                               actor_id=session.get("user_id"))
+            if rule.covers != "resource":
+                await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id"))
     action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
     await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                     action, entity_type="alert_rule", entity_id=rule.name)
@@ -369,6 +429,8 @@ async def delete_rule(rule_id: str):
             result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
             rule = result.scalar_one_or_none()
             if rule:
+                if (conflict := await _child_conflict(db, rule)):
+                    return conflict
                 rule_name = rule.name
                 await db.delete(rule)
     if rule_name: