feat(alerts): one rule can cover every host, or every host in a group (#5464)

#5464 "Group rules: one rule covers every host, or every host in a group",
step 2 of milestone 499 "Alerts you can trust". Each rule named one host, so
the live instance carried six rules for three hosts, and a new host had no
agent-offline or disk alert until someone copied them.

- alert_rules.covers / cover_group / parent_id (migration 0041). A rule on
  host_agent may cover "all_hosts" or a "group" (an inventory group: the
  Hosts ruling, #5294). A covering rule has no state; Steward keeps one
  ordinary child rule per covered host, named "<rule>: <host>", so each
  host's alert keeps its own state, history and acknowledgement.
- alerts.service.reconcile_children: a covered host that has reported in
  the last 7 days gets a child; a host no longer covered loses its child;
  children take the parent's condition, checks, notify setting, action and
  on/off. Run on save and each minute (alert_reconcile).
- A child is changed through its parent: edit, toggle and delete of a child
  are refused (409 / PartOfCoveringRule); its edit page opens the parent.
- Rule form: a Covers choice for host sources, with the group picker; fixed
  once the rule exists. Rules list: a covering rule reads "every host in
  swarm · 3 hosts" with its hosts' rules nested under it, each opening its
  history. The history picker and the rule count leave covering rules out.
- MCP: create_alert_rule takes covers/group; update_alert_rule takes group
  and passes changes down; list_alert_rules shows covers, hosts and part_of.
- docs/core/alerting.md: "Rules that cover hosts".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/service.py

@@ -378,12 +378,16 @@ def _choices(values: list[str], limit: int = 20) -> str:
 
 async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
                       operator: str, threshold, consecutive_failures_required=1,
-                      consecutive_ok_required=1, notify_no_data=True) -> dict:
+                      consecutive_ok_required=1, notify_no_data=True,
+                      covers: str = "resource", cover_group: str = "") -> dict:
     """Validated column values for a new or edited alert rule.
 
     The same choices the rule form offers: a known source, a resource that
     source has reported (or, for monitors, a monitor of that type), and a
-    metric it emits. ValueError says what was wrong and what is valid.
+    metric it emits. A covering rule (`covers` "all_hosts", or "group" with
+    an inventory group's name in `cover_group`, #5464) names no resource: it
+    is for a source whose resources are hosts. ValueError says what was wrong
+    and what is valid.
     """
     name = (name or "").strip()
     if not name:
@@ -392,11 +396,29 @@ async def rule_fields(db, *, name: str, source_module: str, resource_name: str,
     catalog = metric_catalog()
     if source_module not in catalog:
         raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.")
-    resource_name = (resource_name or "").strip()
-    resources = await resources_for(db, source_module)
-    if resource_name not in resources:
-        raise ValueError(f"{source_module} has no resource {resource_name!r}. "
-                         f"Known: {_choices(resources)}.")
+    covers = (covers or "resource").strip()
+    if covers not in COVERS:
+        raise ValueError(f"covers {covers!r} is not one of {', '.join(COVERS)}.")
+    cover_group = (cover_group or "").strip()
+    if covers == "resource":
+        resource_name, cover_group = (resource_name or "").strip(), None
+        resources = await resources_for(db, source_module)
+        if resource_name not in resources:
+            raise ValueError(f"{source_module} has no resource {resource_name!r}. "
+                             f"Known: {_choices(resources)}.")
+    else:
+        if source_module not in HOST_SOURCES:
+            raise ValueError(f"only a rule on {', '.join(HOST_SOURCES)} can cover hosts; "
+                             f"{source_module}'s resources are not hosts.")
+        resource_name = ""
+        if covers == "group":
+            from steward.hosts.service import group_names
+            groups = await group_names(db)
+            if cover_group not in groups:
+                raise ValueError(f"there is no inventory group {cover_group!r}. "
+                                 f"Groups: {_choices(groups)}.")
+        else:
+            cover_group = None
     metric_name = (metric_name or "").strip()
     metrics = await metrics_for(db, source_module)
     if metric_name not in metrics:
@@ -428,7 +450,8 @@ async def rule_fields(db, *, name: str, source_module: str, resource_name: str,
     return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
             "metric_name": metric_name, "operator": op, "threshold": threshold,
             "consecutive_failures_required": consecutive,
-            "consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data)}
+            "consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data),
+            "covers": covers, "cover_group": cover_group}
 
 
 # Changing any of these changes what the rule is watching, so whatever state
@@ -460,15 +483,133 @@ async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool
 
 async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
                    ansible_action: dict | None = None) -> AlertRule:
-    """Insert a rule and its inactive state row, in the caller's transaction."""
+    """Insert a rule and its inactive state row, in the caller's transaction.
+    A covering rule gets no state; its per-host children are made here."""
     rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
     db.add(rule)
     await db.flush()
+    if rule.covers != "resource":
+        await reconcile_children(db, rule)
+        return rule
     db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
                       last_transition_at=datetime.now(timezone.utc)))
     return rule
 
 
+# ── Covering rules (#5464) ───────────────────────────────────────────────────
+# A rule can cover every host, or every host in an inventory group (inventory
+# groups are Steward's grouping system: the Hosts ruling, #5294). Steward keeps
+# one ordinary child rule per covered host, so each host's alert has its own
+# state, history and acknowledgement, and a host that appears later is covered
+# without anyone copying the rule.
+
+COVERS = ("resource", "all_hosts", "group")
+# Sources whose bare resource names are hosts ("<host>", parts are "<host>:…").
+HOST_SOURCES = ("host_agent",)
+# A host counts as present while it has reported within this long: the raw
+# metrics window, so a decommissioned host's children go once its data does.
+COVER_SEEN_DAYS = 7
+
+
+def covers_text(rule: AlertRule) -> str:
+    """What a covering rule spans, in words: "every host", "every host in web"."""
+    if rule.covers == "group":
+        return f"every host in {rule.cover_group}"
+    return "every host" if rule.covers == "all_hosts" else rule.resource_name
+
+
+async def covered_hosts(db, rule: AlertRule, now: datetime | None = None) -> list[str]:
+    """The host names a covering rule spans now: hosts that have reported
+    under its source within COVER_SEEN_DAYS, all of them or those whose
+    linked Ansible target is in its group."""
+    from datetime import timedelta
+    from steward.hosts.service import host_groups
+    from steward.models.hosts import Host
+    now = now or datetime.now(timezone.utc)
+    reported = set((await db.execute(
+        select(PluginMetricLatest.resource_name)
+        .where(PluginMetricLatest.source_module == rule.source_module,
+               PluginMetricLatest.recorded_at >= now - timedelta(days=COVER_SEEN_DAYS),
+               ~PluginMetricLatest.resource_name.contains(":"))
+        .distinct())).scalars())
+    if rule.covers != "group":
+        return sorted(reported, key=str.lower)
+    hosts = (await db.execute(select(Host.id, Host.name).where(Host.name.in_(reported)))).all()
+    groups = await host_groups(db, [hid for hid, _ in hosts])
+    return sorted((name for hid, name in hosts if rule.cover_group in groups.get(hid, [])),
+                  key=str.lower)
+
+
+def child_fields(parent: AlertRule, host: str) -> dict:
+    """A covering rule's child for one host: the parent's condition on that
+    host, named "<parent>: <host>"."""
+    return {"name": f"{parent.name}: {host}"[:128], "source_module": parent.source_module,
+            "resource_name": host, "metric_name": parent.metric_name,
+            "operator": parent.operator, "threshold": parent.threshold,
+            "consecutive_failures_required": parent.consecutive_failures_required,
+            "consecutive_ok_required": parent.consecutive_ok_required,
+            "notify_no_data": parent.notify_no_data}
+
+
+async def reconcile_children(db, parent: AlertRule, *, actor_id: str | None = None,
+                             now: datetime | None = None) -> tuple[list[str], list[str]]:
+    """Bring a covering rule's children in line with it, in the caller's
+    transaction: one per covered host, each carrying the parent's condition,
+    action and on/off. A host newly covered gets a child; a child whose host
+    is no longer covered is deleted, its history with it. Returns (hosts
+    added, hosts removed)."""
+    hosts = set(await covered_hosts(db, parent, now))
+    children = {c.resource_name: c for c in (await db.execute(
+        select(AlertRule).where(AlertRule.parent_id == parent.id))).scalars()}
+    added = sorted(hosts - children.keys(), key=str.lower)
+    removed = sorted(children.keys() - hosts, key=str.lower)
+    for host in added:
+        await add_rule(db, {**child_fields(parent, host), "parent_id": parent.id},
+                       created_by=parent.created_by, enabled=parent.enabled,
+                       ansible_action=parent.ansible_action)
+    for host in removed:
+        await db.delete(children[host])
+    for host, child in children.items():
+        if host in hosts:
+            fields = child_fields(parent, host)
+            if (any(getattr(child, k) != v for k, v in fields.items())
+                    or child.enabled != parent.enabled):
+                await apply_rule_change(db, child, fields, enabled=parent.enabled,
+                                        actor_id=actor_id)
+            if child.ansible_action != parent.ansible_action:
+                child.ansible_action = parent.ansible_action
+    return added, removed
+
+
+async def reconcile_covering_rules(app) -> None:
+    """Every covering rule's children brought in line with the hosts there are
+    now. Run each minute by the core scheduler (alert_reconcile), so a host
+    that joins a group, or starts reporting, is covered within a minute."""
+    async with app.db_sessionmaker() as db:
+        async with db.begin():
+            parents = (await db.execute(
+                select(AlertRule).where(AlertRule.covers != "resource"))).scalars().all()
+            for parent in parents:
+                added, removed = await reconcile_children(db, parent)
+                if added or removed:
+                    logger.info("Covering rule %r: added %s, removed %s", parent.name,
+                                ", ".join(added) or "none", ", ".join(removed) or "none")
+
+
+def part_of(rule: AlertRule, rules) -> AlertRule | None:
+    """The covering rule a child belongs to, from a list of rules; None for a
+    rule that stands alone."""
+    return next((r for r in rules if r.id == rule.parent_id), None) if rule.parent_id else None
+
+
+class PartOfCoveringRule(ValueError):
+    """A child of a covering rule is changed through its parent, never alone."""
+
+    def __init__(self, child: AlertRule, parent_name: str):
+        super().__init__(f"{child.name} is part of the rule {parent_name!r}, which keeps one "
+                         f"rule per host; change or disable it there.")
+
+
 async def acknowledge(db, rule_id: str, user_id: str | None) -> AlertState:
     """Acknowledge a firing alert, in the caller's transaction, and record it
     in the alert's history as a firing → acknowledged event by `user_id`, so