feat(alerts): one rule can cover every host, or every host in a group (#5464)

#5464 "Group rules: one rule covers every host, or every host in a group",
step 2 of milestone 499 "Alerts you can trust". Each rule named one host, so
the live instance carried six rules for three hosts, and a new host had no
agent-offline or disk alert until someone copied them.

- alert_rules.covers / cover_group / parent_id (migration 0041). A rule on
  host_agent may cover "all_hosts" or a "group" (an inventory group: the
  Hosts ruling, #5294). A covering rule has no state; Steward keeps one
  ordinary child rule per covered host, named "<rule>: <host>", so each
  host's alert keeps its own state, history and acknowledgement.
- alerts.service.reconcile_children: a covered host that has reported in
  the last 7 days gets a child; a host no longer covered loses its child;
  children take the parent's condition, checks, notify setting, action and
  on/off. Run on save and each minute (alert_reconcile).
- A child is changed through its parent: edit, toggle and delete of a child
  are refused (409 / PartOfCoveringRule); its edit page opens the parent.
- Rule form: a Covers choice for host sources, with the group picker; fixed
  once the rule exists. Rules list: a covering rule reads "every host in
  swarm · 3 hosts" with its hosts' rules nested under it, each opening its
  history. The history picker and the rule count leave covering rules out.
- MCP: create_alert_rule takes covers/group; update_alert_rule takes group
  and passes changes down; list_alert_rules shows covers, hosts and part_of.
- docs/core/alerting.md: "Rules that cover hosts".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/mcp/tools/alerts.py

@@ -134,6 +134,11 @@ async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict
     if it has no state row. `no_data_since` is set while its metric is silent,
     whatever the state. `ansible_action` is the
     playbook the rule runs when it fires, if any (`check: true` = dry run).
+
+    `covers` is "resource" for a rule on one resource, or "all_hosts" /
+    "group" (with `group`) for a rule covering hosts: it has no state of its
+    own, `hosts` names the hosts it covers, and each host's rule is listed
+    with `part_of` naming it.
     """
     limit = clamp_limit(limit, 200, 500)
     async with db_session() as db:
@@ -141,13 +146,19 @@ async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict
         states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
     if enabled_only:
         rules = [r for r in rules if r.enabled]
+    names = {r.id: r.name for r in rules}
+    hosts: dict[str, list[str]] = {}
+    for r in rules:
+        if r.parent_id:
+            hosts.setdefault(r.parent_id, []).append(r.resource_name)
     rows = [
         {
             "rule_id": r.id,
             "rule_name": r.name,
             "enabled": r.enabled,
             "source_module": r.source_module,
-            "resource": r.resource_name,
+            "resource": r.resource_name or None,
+            **_covers(r, hosts.get(r.id, []), names.get(r.parent_id)),
             "metric": r.metric_name,
             "condition": _condition(r),
             "state": alert_svc.alert_label(states[r.id]) if r.id in states else None,
@@ -219,21 +230,51 @@ async def _rule(db, rule: str) -> AlertRule:
     return pick(rule, await alert_svc.list_rules(db), kind="alert rule", list_tool="list_alert_rules")
 
 
-def _rule_row(r: AlertRule) -> dict:
+def _covers(r: AlertRule, hosts: list[str], parent_name: str | None) -> dict:
+    """What a rule covers (#5464): the hosts under a covering rule, or the
+    covering rule a host's rule is part of."""
+    out: dict = {"covers": r.covers}
+    if r.covers == "group":
+        out["group"] = r.cover_group
+    if r.covers != "resource":
+        out["hosts"] = sorted(hosts, key=str.lower)
+    if r.parent_id:
+        out["part_of"] = parent_name
+    return out
+
+
+async def _rule_row(db, r: AlertRule) -> dict:
+    hosts = [c.resource_name for c in (await db.execute(
+        select(AlertRule).where(AlertRule.parent_id == r.id))).scalars()] if r.covers != "resource" else []
+    parent = await db.get(AlertRule, r.parent_id) if r.parent_id else None
     return {"rule_id": r.id, "rule_name": r.name, "enabled": r.enabled,
-            "source_module": r.source_module, "resource": r.resource_name,
+            "source_module": r.source_module, "resource": r.resource_name or None,
+            **_covers(r, hosts, parent.name if parent else None),
             "metric": r.metric_name, "condition": _condition(r),
             "notify_no_data": r.notify_no_data, "ansible_action": _action(r)}
 
 
+async def _not_a_child(db, r: AlertRule) -> None:
+    """A host's rule under a covering rule is changed through its parent."""
+    if r.parent_id:
+        parent = await db.get(AlertRule, r.parent_id)
+        raise alert_svc.PartOfCoveringRule(r, parent.name if parent else "?")
+
+
 async def create_alert_rule(name: str, source_module: str, resource: str, metric: str,
                             operator: str, threshold: float, consecutive_checks: int = 1,
                             clear_after_checks: int = 1, notify_no_data: bool = True,
-                            enabled: bool = True) -> dict:
+                            enabled: bool = True, covers: str = "resource",
+                            group: str = "") -> dict:
     """Add an alert rule: notify when `metric` of `resource` crosses `threshold`.
 
     source_module, resource, metric: get_alert_rule_options lists the valid
       choices; a resource must have reported (or, for a monitor, exist).
+    covers: "resource" (default) watches `resource`. "all_hosts" covers every
+      host, "group" every host in the inventory group `group`; pass
+      `resource` as "". Steward then keeps one rule per host, named
+      "<name>: <host>", and adds one within a minute for a host that starts
+      reporting or joins the group. Only for host_agent.
     operator: ">", "<", ">=", "<=", "==" or "!=".
     consecutive_checks: how many checks in a row must breach before it fires
       (default 1); higher values ride out a single blip.
@@ -254,12 +295,15 @@ async def create_alert_rule(name: str, source_module: str, resource: str, metric
             db, name=name, source_module=source_module, resource_name=resource,
             metric_name=metric, operator=operator, threshold=threshold,
             consecutive_failures_required=consecutive_checks,
-            consecutive_ok_required=clear_after_checks, notify_no_data=notify_no_data)
+            consecutive_ok_required=clear_after_checks, notify_no_data=notify_no_data,
+            covers=covers, cover_group=group)
         r = await alert_svc.add_rule(db, fields, created_by=caller.user_id, enabled=enabled)
+        row = await _rule_row(db, r)
     await audit_mcp("alert_rule.created", "alert_rule", r.name,
-                    {"rule_id": r.id, "source": r.source_module, "resource": r.resource_name,
+                    {"rule_id": r.id, "source": r.source_module,
+                     "resource": r.resource_name or alert_svc.covers_text(r),
                      "metric": r.metric_name})
-    return _rule_row(r)
+    return row
 
 
 async def update_alert_rule(rule: str, name: str | None = None, source_module: str | None = None,
@@ -268,7 +312,7 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s
                             consecutive_checks: int | None = None,
                             clear_after_checks: int | None = None,
                             notify_no_data: bool | None = None,
-                            enabled: bool | None = None) -> dict:
+                            enabled: bool | None = None, group: str | None = None) -> dict:
     """Change an alert rule. Only the arguments you pass change; the rest stay.
 
     rule: its name or id (list_alert_rules). Pass enabled=false to silence it
@@ -277,9 +321,15 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s
     operator, threshold), or disabling or re-enabling it, puts its alert back
     to inactive; a firing alert records a resolved event saying why. Recorded
     in the audit log.
+
+    A rule covering hosts passes every change on to its per-host rules;
+    `group` moves a group-covering rule to another inventory group. What a
+    rule covers is fixed once it exists. A host's rule under a covering rule
+    is changed through that rule, not alone.
     """
     async with db_session() as db, db.begin():
         r = await _rule(db, rule)
+        await _not_a_child(db, r)
         fields = await alert_svc.rule_fields(
             db,
             name=r.name if name is None else name,
@@ -292,24 +342,31 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s
                                            if consecutive_checks is None else consecutive_checks),
             consecutive_ok_required=(r.consecutive_ok_required
                                      if clear_after_checks is None else clear_after_checks),
-            notify_no_data=r.notify_no_data if notify_no_data is None else notify_no_data)
+            notify_no_data=r.notify_no_data if notify_no_data is None else notify_no_data,
+            covers=r.covers, cover_group=(r.cover_group or "") if group is None else group)
         await alert_svc.apply_rule_change(db, r, fields, enabled=enabled,
                                           actor_id=current_caller().user_id)
+        if r.covers != "resource":
+            await alert_svc.reconcile_children(db, r, actor_id=current_caller().user_id)
+        row = await _rule_row(db, r)
     changed = sorted(k for k, v in {"name": name, "source_module": source_module,
                                     "resource": resource, "metric": metric, "operator": operator,
                                     "threshold": threshold, "consecutive_checks": consecutive_checks,
                                     "clear_after_checks": clear_after_checks,
                                     "notify_no_data": notify_no_data,
-                                    "enabled": enabled}.items() if v is not None)
+                                    "enabled": enabled, "group": group}.items() if v is not None)
     await audit_mcp("alert_rule.updated", "alert_rule", r.name, {"rule_id": r.id, "changed": changed})
-    return _rule_row(r)
+    return row
 
 
 async def delete_alert_rule(rule: str) -> dict:
     """Delete an alert rule and its history. To silence it but keep it,
-    update_alert_rule(enabled=false). Admin only; recorded in the audit log."""
+    update_alert_rule(enabled=false). Deleting a rule that covers hosts
+    deletes its per-host rules and their history too. Admin only; recorded
+    in the audit log."""
     async with db_session() as db, db.begin():
         r = await _rule(db, rule)
+        await _not_a_child(db, r)
         await db.delete(r)
     await audit_mcp("alert_rule.deleted", "alert_rule", r.name, {"rule_id": r.id})
     return {"deleted": r.name, "rule_id": r.id}