FabledSteward/steward/alerts/service.py

"""Alert queries and validation shared by the alerts pages and the MCP tools.

Each function takes an open session (or plain values), with no Quart request
context, so the routes and the /mcp tools (which run outside Quart's
pipeline) call the same code and refuse the same input for the same reasons.
"""
from __future__ import annotations

import logging
from datetime import datetime, timezone

from sqlalchemy import func, or_, select

from steward.models.alerts import (
    AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
)
from steward.models.metrics import PluginMetricLatest
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor

logger = logging.getLogger(__name__)

# What core itself records. Unified monitors emit metrics under their TYPE
# (icmp/tcp/dns/http), each with is_up (1.0 up / 0.0 down) and response_ms
# (where the probe times it); "steward" is the app watching itself.
CORE_METRICS: dict[str, list[str]] = {
    "icmp":    ["is_up", "response_ms"],
    "tcp":     ["is_up", "response_ms"],
    "dns":     ["is_up", "response_ms"],
    "http":    ["is_up", "response_ms"],
    "steward": ["ingest_posts", "ingest_failed", "ingest_slowest_ms", "db_roundtrip_ms",
                "unwatched_secs", "restart_gap_secs", "cleanup_secs", "open_fds", "open_fds_pct",
                "backup_age_secs", "backup_bytes"],
}


def metric_catalog() -> dict[str, list[str]]:
    """Every source an alert rule can watch, and its metric names.

    Core's own sources, then each loaded plugin's get_alert_metrics() export
    ({source: [metric names]}), so the catalog is written by whatever emits
    the metrics and a disabled plugin's sources drop out. An empty list means
    the names are dynamic (snmp: the OID labels recorded so far).
    """
    from steward.core.plugin_manager import get_loaded_plugin_modules
    catalog = {src: list(names) for src, names in CORE_METRICS.items()}
    for name, module in get_loaded_plugin_modules().items():
        hook = getattr(module, "get_alert_metrics", None)
        if hook is None:
            continue
        try:
            contributed = hook() or {}
        except Exception:
            logger.exception("Plugin %r: get_alert_metrics() raised; its metrics are skipped", name)
            continue
        for src, names in contributed.items():
            catalog[str(src)] = [str(n) for n in names]
    return catalog


def source_modules() -> list[str]:
    """The sources in picker order: core's first, then plugins'."""
    return list(metric_catalog())


# States that need someone's attention: pending (breaching, not yet for long
# enough), firing, and acknowledged-but-still-breaching.
ACTIVE_STATES = (AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending)


async def active_alerts(db) -> list:
    """(AlertState, AlertRule) rows that need attention, latest transition
    first: not yet resolved, or with a series that has gone silent (no_data)."""
    return list((await db.execute(
        select(AlertState, AlertRule)
        .join(AlertRule, AlertState.rule_id == AlertRule.id)
        .where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
        .order_by(AlertState.last_transition_at.desc())
    )).all())


def alert_label(state: AlertState) -> str:
    """What an alert is, in one word: its state, or "no_data" for a quiet rule
    whose series has gone silent (a firing alert that goes silent stays
    "firing"; check no_data_since for the mark)."""
    if state.no_data_since is not None and state.state == AlertStateEnum.inactive:
        return "no_data"
    return state.state.value


async def latest_value(db, rule: AlertRule) -> float | None:
    """The newest sample of the series a rule watches, or None before the first
    (a primary-key read of the latest-value table, #5208)."""
    return (await db.execute(
        select(PluginMetricLatest.value)
        .where(PluginMetricLatest.source_module == rule.source_module,
               PluginMetricLatest.resource_name == rule.resource_name,
               PluginMetricLatest.metric_name == rule.metric_name)
    )).scalar_one_or_none()


# Sources whose resources have no page of their own: the plugin's page lists them.
PLUGIN_PAGES = {"traefik": "/plugins/traefik/", "unifi": "/plugins/unifi/",
                "snmp": "/plugins/snmp/"}


async def resource_url(db, rule: AlertRule) -> str | None:
    """Where the thing a rule watches is shown, or None when nothing shows it."""
    return await resource_url_for(db, rule.source_module, rule.resource_name)


async def resource_url_for(db, source: str, resource: str) -> str | None:
    """Where a (source, resource) of the metric store is shown, or None.

    host_agent resources are the host's name, or "<host>:<mount|nic|core>";
    docker's are "<host>/<container>"; a monitor's is its name under its type.
    """
    from steward.models.hosts import Host
    if source == "host_agent":
        name = resource.split(":", 1)[0]
        host_id = (await db.execute(
            select(Host.id).where(Host.name == name).limit(1))).scalar_one_or_none()
        return f"/hosts/{host_id}" if host_id else None
    if source in MONITOR_TYPE_VALUES:
        monitor_id = (await db.execute(
            select(Monitor.id).where(Monitor.type == source, Monitor.name == resource).limit(1)
        )).scalar_one_or_none()
        return f"/monitors/{monitor_id}" if monitor_id else None
    if source == "docker" and "/" in resource:
        host_name, container = resource.split("/", 1)
        host_id = (await db.execute(
            select(Host.id).where(Host.name == host_name).limit(1))).scalar_one_or_none()
        return f"/plugins/docker/container/{host_id}/{container}" if host_id else None
    return PLUGIN_PAGES.get(source)


# Metrics whose value is a word: the sentence says the word, not the condition.
_BINARY = ("is_up", "is_healthy")


def rule_condition(rule: AlertRule) -> str:
    """What a rule fires on, in words: "CPU > 90%", "Fullest disk >= 85%",
    or "is down" for an up/down metric rather than "is_up < 0.5"."""
    return condition_text(rule.metric_name, rule.operator, rule.threshold)


def condition_text(metric: str, operator: AlertOperator | str, threshold: float) -> str:
    """rule_condition from a rule's parts, for an alert carried as a dict (an
    email); the operator may be its value, ">=", rather than the enum."""
    from steward.core.alerts import _is_breached
    from steward.core.formatting import metric_label, metric_value
    operator = AlertOperator(operator)
    if metric in _BINARY:
        fires_on = [metric_value(metric, v) for v in (1.0, 0.0)
                    if _is_breached(v, operator, threshold)]
        if len(fires_on) == 1:
            return f"is {fires_on[0]}"
    return f"{metric_label(metric)} {operator.value} {metric_value(metric, threshold)}"


def alert_sentence(rule: AlertRule, label: str, value: float | None,
             since: datetime | None, now: datetime) -> dict:
    """An alert as one sentence around its resource: {"lead", "resource",
    "tail"}, read lead + resource + tail, e.g. "CPU on " + "web-1" +
    " is 94% (> 90%), firing for 12m". label is alert_label(state); since is
    when that label began (no_data_since for no_data)."""
    from steward.core.formatting import human_duration, metric_label, metric_value
    metric = rule.metric_name
    said = metric_label(metric)
    held = human_duration((now - since).total_seconds()) if since else None
    if label == "no_data":
        return {"lead": f"No {said} reading from ", "resource": rule.resource_name,
                "tail": f" for {held}" if held else ""}
    # An acknowledged alert is still breaching: say for how long it has been,
    # not how long ago someone acknowledged it (the page says that apart).
    word = "breaching" if label == "acknowledged" else label
    for_how_long = f", {word} for {held}" if held else f", {word}"
    if value is None:
        reading = " has no reading yet"
    elif metric in _BINARY:
        return {"lead": "", "resource": rule.resource_name,
                "tail": f" is {metric_value(metric, value)}{for_how_long}"}
    else:
        reading = f" is {metric_value(metric, value)}"
    condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})"
    return {"lead": f"{said} on ", "resource": rule.resource_name,
            "tail": f"{reading}{condition}{for_how_long}"}


# The order the active list reads in: what is breaching and unhandled, then
# what has gone silent, then what someone has taken, then what may yet clear.
ACTIVE_ORDER = {"firing": 0, "no_data": 1, "acknowledged": 2, "pending": 3}

_BREACHING = (AlertStateEnum.pending.value, AlertStateEnum.firing.value,
              AlertStateEnum.acknowledged.value)


async def breach_started(db, rule_ids) -> dict[str, datetime]:
    """{rule_id: when its current breach began}: the newest transition into
    pending or firing from a quiet state. Acknowledging moves an alert's
    last_transition_at, so that column cannot say this."""
    if not rule_ids:
        return {}
    return dict((await db.execute(
        select(AlertEvent.rule_id, func.max(AlertEvent.transitioned_at))
        .where(AlertEvent.rule_id.in_(list(rule_ids)),
               AlertEvent.from_state.not_in(_BREACHING),
               AlertEvent.to_state.in_((AlertStateEnum.pending.value, AlertStateEnum.firing.value)))
        .group_by(AlertEvent.rule_id)
    )).all())


async def usernames(db, ids) -> dict[str, str]:
    """{user id: username} for the given ids (None and repeats ignored)."""
    from steward.models.users import User
    ids = {i for i in ids if i}
    if not ids:
        return {}
    return dict((await db.execute(select(User.id, User.username).where(User.id.in_(ids)))).all())


async def active_rows(db, now: datetime) -> list[dict]:
    """The alerts page's active list, firing first, then no data, then
    acknowledged, then pending; within each the longest-running first. Each
    row: its state, rule, label, sentence, the link to what it watches, when
    the breach began, and who acknowledged it."""
    alerts = await active_alerts(db)
    started = await breach_started(db, {rule.id for state, rule in alerts
                                        if state.state.value in _BREACHING})
    names = await usernames(db, (s.acknowledged_by for s, _ in alerts))
    rows = []
    for state, rule in alerts:
        label = alert_label(state)
        if label == "no_data":
            since = state.no_data_since
        elif label == "acknowledged":
            since = started.get(rule.id, state.last_transition_at)
        else:
            since = state.last_transition_at
        value = await latest_value(db, rule)
        rows.append({
            "state": state, "rule": rule, "label": label, "since": since,
            "breach_since": started.get(rule.id) if label != "no_data" else None,
            "acked_by": names.get(state.acknowledged_by, state.acknowledged_by),
            "value": value,
            "says": alert_sentence(rule, label, value, since, now),
            "url": await resource_url(db, rule),
        })
    epoch = datetime.max.replace(tzinfo=timezone.utc)
    rows.sort(key=lambda r: (ACTIVE_ORDER.get(r["label"], 9), r["since"] or epoch))
    return rows


def widget_view(active: list[dict], rules: list[AlertRule]) -> dict:
    """The dashboard Alerts widget (#5510) from active_rows and every rule:
    the alerts in that order, each with what its rule fires on, and how many
    enabled rules, as people made them, are quiet (a covering rule counts
    once, and is quiet only while every host under it is).
    {"rows", "quiet", "total"}."""
    busy = {r["rule"].id for r in active}
    busy |= {r["rule"].parent_id for r in active if r["rule"].parent_id}
    made = [r for r in rules if r.enabled and not r.parent_id]
    return {"rows": [{**r, "condition": rule_condition(r["rule"])} for r in active],
            "quiet": sum(1 for r in made if r.id not in busy), "total": len(made)}


def watches_host(rule: AlertRule, host_name: str, monitor_keys: set) -> bool:
    """Whether a rule watches something on this host: its agent series
    ("<host>" or "<host>:<part>"), one of its containers ("<host>/<name>"),
    or one of its monitors ((type, name) in monitor_keys)."""
    source, resource = rule.source_module, rule.resource_name
    if source == "host_agent":
        return resource == host_name or resource.startswith(host_name + ":")
    if source == "docker":
        return resource.startswith(host_name + "/")
    return (source, resource) in monitor_keys


async def active_count(db) -> int:
    """How many alerts need attention: the sidebar's number."""
    from sqlalchemy import func
    return (await db.execute(
        select(func.count()).select_from(AlertState)
        .where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
    )).scalar_one()


def notify_targets(smtp: dict, webhook: dict, accounts: int = 0) -> list[str]:
    """Where an alert's notifications go, in words, by the same tests
    alerts are sent by: email when a server is set and `accounts` (that get
    alerts) or Also send to addresses exist (core.outbox.hold_alert), a
    webhook when its URL is (core.notifications.dispatch_notifications). Empty: nobody (#5315, milestone 495)."""
    from urllib.parse import urlsplit
    out = []
    recipients = smtp.get("recipients") or []
    if isinstance(recipients, str):
        recipients = [r.strip() for r in recipients.split(",") if r.strip()]
    who = ([f"{accounts} account{'s' if accounts != 1 else ''}"] if accounts else []) + recipients
    if smtp.get("host") and who:
        out.append("email to " + ", ".join(who))
    if url := webhook.get("url"):
        out.append("a webhook to " + (urlsplit(url).hostname or url))
    return out


async def last_fired(db, rule_id: str | None = None) -> dict[str, datetime]:
    """{rule id: when it last went to firing}, for the rules that ever have;
    one rule's when `rule_id` is given."""
    stmt = (select(AlertEvent.rule_id, func.max(AlertEvent.transitioned_at))
            .where(AlertEvent.to_state == AlertStateEnum.firing.value)
            .group_by(AlertEvent.rule_id))
    if rule_id:
        stmt = stmt.where(AlertEvent.rule_id == rule_id)
    rows = await db.execute(stmt)
    return {rid: at for rid, at in rows.all()}


async def list_rules(db) -> list[AlertRule]:
    return list((await db.execute(select(AlertRule).order_by(AlertRule.name))).scalars())


async def history(db, since: datetime, rule_id: str | None = None,
                  limit: int = 200, before: str | None = None) -> list[tuple[AlertEvent, AlertRule]]:
    """Alert transitions since `since`, newest first, with their rule; one
    rule's when `rule_id` is given; only those older than the `before` cursor
    when given (core.paging). Returns up to limit+1 rows so a caller can tell
    the list was cut."""
    from steward.core.paging import keyset
    stmt = (select(AlertEvent, AlertRule)
            .join(AlertRule, AlertEvent.rule_id == AlertRule.id)
            .where(AlertEvent.transitioned_at >= since))
    if rule_id:
        stmt = stmt.where(AlertEvent.rule_id == rule_id)
    stmt = keyset(stmt, AlertEvent.transitioned_at, AlertEvent.id, before, limit)
    return [tuple(row) for row in (await db.execute(stmt)).all()]


async def history_columns(db, since: datetime, rule_id: str | None = None) -> tuple[bool, bool]:
    """(whether any transition in the range notified someone or was muted,
    whether any started an Ansible run): the history page shows its Notified
    and Action columns only then (#5316)."""
    from steward.core.alerts import MUTED
    base = select(AlertEvent.id).where(AlertEvent.transitioned_at >= since)
    if rule_id:
        base = base.where(AlertEvent.rule_id == rule_id)
    from steward.models.outbox import email_outbox_events
    queued = select(email_outbox_events.c.event_id).where(
        email_outbox_events.c.event_id == AlertEvent.id).exists()
    notified = await db.scalar(select(base.where(or_(
        AlertEvent.email_sent.is_not(None), AlertEvent.webhook_sent.is_not(None),
        AlertEvent.note.like(f"%{MUTED}%"), queued)).exists()))
    acted = await db.scalar(select(base.where(AlertEvent.ansible_run_id.is_not(None)).exists()))
    return bool(notified), bool(acted)


async def list_maintenance_windows(db) -> list[MaintenanceWindow]:
    """Every window, latest start first."""
    return list((await db.execute(
        select(MaintenanceWindow).order_by(MaintenanceWindow.start_at.desc())
    )).scalars())


async def resources_for(db, source_module: str) -> list[str]:
    """Sorted distinct resource names for a source module.

    Read from the latest-value table, one row per series: a DISTINCT over
    plugin_metrics reads every raw sample (Postgres has no skip scan), which
    held the rule page for up to 15 s on host_agent (#5208's trap again).

    For monitor types (icmp/tcp/dns/http), supplements with monitor names so
    the list is populated even before any metrics have been recorded.
    """
    result = await db.execute(
        select(PluginMetricLatest.resource_name)
        .where(PluginMetricLatest.source_module == source_module)
        .distinct()
    )
    resources: set[str] = {r for (r,) in result}
    if source_module in MONITOR_TYPE_VALUES:
        mon_result = await db.execute(
            select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name))
        resources.update(name for (name,) in mon_result)
    return sorted(resources)


# A host's parts, by the shape of the name after "<host>:" (#5475): the order
# and heading each kind takes in the rule form's Part picker.
PART_KINDS = (("mount", "Mounts"), ("core", "CPU cores"), ("disk", "Disks"),
              ("drive", "Drive health"), ("net", "Interfaces"), ("temp", "Temperatures"),
              ("other", "Other"))


def part_kind(part: str) -> tuple[str, str]:
    """(kind, label) for a host part: "/var" is a mount, "core3" a core,
    "diskio:sda" a disk, "drive:sda" its health (#5789), "net:eth0" an
    interface, "temp:Core 0" a sensor."""
    if part.startswith("/"):
        return "mount", part
    if part.startswith("core") and part[4:].isdigit():
        return "core", f"core {part[4:]}"
    for prefix, kind in (("diskio:", "disk"), ("drive:", "drive"), ("net:", "net"), ("temp:", "temp")):
        if part.startswith(prefix):
            return kind, part[len(prefix):]
    return "other", part


def _part_sort(part: str):
    kind, label = part_kind(part)
    return (int(part[4:]), "") if kind == "core" else (0, label.lower())


def alertable(source_module: str, reported: set[str]) -> list[str]:
    """The metrics a rule on a resource can watch, from what it reports: the
    source's catalog, in its order, narrowed to those; every reported name
    for a source with no catalog (snmp)."""
    catalog = metric_catalog().get(source_module, [])
    return [m for m in catalog if m in reported] if catalog else sorted(reported)


async def rule_target(db, source_module: str, *, host: str = "", resource: str = "",
                      metric: str = "", keep: bool = False) -> dict:
    """What the rule form's target pickers offer (#5475), from the latest-value
    table: the resources that report something a rule can watch, and for the
    chosen one the metrics it reports. For a host source the resources split
    into a Host picker and a Part picker ("the whole host", then the host's
    mounts, cores and so on). With `keep` (a rule's own page), its resource
    and metric stay on offer even after they go quiet, so opening a rule
    never drops them; a metric carried over from another part is dropped.

    Returns host_source, hosts, host, parts [(heading, [(value, label)])],
    resources, resource, metrics, metric."""
    catalog = metric_catalog().get(source_module, [])
    seen: dict[str, set[str]] = {}
    for res, name in (await db.execute(
            select(PluginMetricLatest.resource_name, PluginMetricLatest.metric_name)
            .where(PluginMetricLatest.source_module == source_module))).all():
        seen.setdefault(res, set()).add(name)
    pickable = {r: ms for r, names in seen.items() if (ms := alertable(source_module, names))}
    if source_module in MONITOR_TYPE_VALUES:
        for (name,) in (await db.execute(select(Monitor.name).where(Monitor.type == source_module))).all():
            pickable.setdefault(name, list(catalog))
    if resource and resource not in pickable:
        pickable[resource] = list(catalog) or ([metric] if metric else [])

    out: dict = {"host_source": source_module in HOST_SOURCES, "hosts": [], "host": "",
                 "parts": [], "resources": [], "resource": resource}
    if out["host_source"]:
        hosts = sorted({r.split(":", 1)[0] for r in pickable}, key=str.lower)
        host = host or resource.split(":", 1)[0]
        if host not in hosts:
            host = hosts[0] if hosts else ""
        mine = [r for r in pickable if r == host or r.startswith(host + ":")]
        if resource not in mine:
            resource = host if host in pickable else (sorted(mine)[0] if mine else "")
        by_kind: dict[str, list] = {}
        for r in sorted((r for r in mine if r != host), key=lambda r: _part_sort(r[len(host) + 1:])):
            kind, label = part_kind(r[len(host) + 1:])
            by_kind.setdefault(kind, []).append((r, label))
        parts = [("", [(host, "The whole host")])] if host in pickable else []
        parts += [(heading, by_kind[k]) for k, heading in PART_KINDS if k in by_kind]
        out.update(hosts=hosts, host=host, parts=parts, resource=resource)
    else:
        out["resources"] = sorted(pickable, key=str.lower)
    metrics = list(pickable.get(out["resource"]) or catalog)
    if keep and metric and metric not in metrics and (metric in catalog or not catalog):
        metrics.append(metric)
    out.update(metrics=metrics, metric=metric if metric in metrics else "")
    return out


# How far back the rule form's metric preview looks: a day shows the daily
# swing (backups, batch jobs) that a threshold has to sit above.
PREVIEW_HOURS = 24


async def metric_preview(db, source_module: str, resource: str, metric: str,
                         now: datetime | None = None) -> dict | None:
    """What the chosen metric reads (#5475), so a threshold is set against real
    numbers: the latest reading and when, and the last PREVIEW_HOURS as hourly
    averages with the lowest and highest. Reads one series: the latest-value
    row, then a recorded_at range on plugin_metrics' series index. None when
    it has no reading."""
    from datetime import timedelta
    from steward.models.metrics import PluginMetric
    latest = (await db.execute(select(PluginMetricLatest).where(
        PluginMetricLatest.source_module == source_module,
        PluginMetricLatest.resource_name == resource,
        PluginMetricLatest.metric_name == metric))).scalar_one_or_none()
    if latest is None:
        return None
    now = now or datetime.now(timezone.utc)
    hour = func.date_trunc("hour", PluginMetric.recorded_at)
    rows = (await db.execute(
        select(hour, func.avg(PluginMetric.value), func.min(PluginMetric.value),
               func.max(PluginMetric.value))
        .where(PluginMetric.source_module == source_module,
               PluginMetric.resource_name == resource,
               PluginMetric.metric_name == metric,
               PluginMetric.recorded_at >= now - timedelta(hours=PREVIEW_HOURS))
        .group_by(hour).order_by(hour))).all()
    return {"now": latest.value, "at": latest.recorded_at,
            "series": [avg for _, avg, _, _ in rows],
            "low": min((lo for _, _, lo, _ in rows), default=None),
            "high": max((hi for _, _, _, hi in rows), default=None)}


async def metrics_for(db, source_module: str) -> list[str]:
    """The metric names a rule on this source can watch: the static catalog, or
    for dynamic sources (snmp) the names recorded so far."""
    metrics = metric_catalog().get(source_module, [])
    if source_module and not metrics:
        result = await db.execute(
            select(PluginMetricLatest.metric_name)
            .where(PluginMetricLatest.source_module == source_module)
            .distinct()
            .order_by(PluginMetricLatest.metric_name))
        metrics = [r for (r,) in result]
    return metrics


def _choices(values: list[str], limit: int = 20) -> str:
    if not values:
        return "none yet"
    shown = ", ".join(values[:limit])
    return shown + (f" (and {len(values) - limit} more)" if len(values) > limit else "")


async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
                      operator: str, threshold, consecutive_failures_required=1,
                      consecutive_ok_required=1, notify_no_data=True,
                      covers: str = "resource", cover_group: str = "") -> dict:
    """Validated column values for a new or edited alert rule.

    The same choices the rule form offers: a known source, a resource that
    source has reported (or, for monitors, a monitor of that type), and a
    metric it emits. A covering rule (`covers` "all_hosts", or "group" with
    an inventory group's name in `cover_group`, #5464) names no resource: it
    is for a source whose resources are hosts. ValueError says what was wrong
    and what is valid.
    """
    name = (name or "").strip()
    if not name:
        raise ValueError("an alert rule needs a name.")
    source_module = (source_module or "").strip()
    catalog = metric_catalog()
    if source_module not in catalog:
        raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.")
    covers = (covers or "resource").strip()
    if covers not in COVERS:
        raise ValueError(f"covers {covers!r} is not one of {', '.join(COVERS)}.")
    cover_group = (cover_group or "").strip()
    if covers == "resource":
        resource_name, cover_group = (resource_name or "").strip(), None
        resources = await resources_for(db, source_module)
        if resource_name not in resources:
            raise ValueError(f"{source_module} has no resource {resource_name!r}. "
                             f"Known: {_choices(resources)}.")
    else:
        if source_module not in HOST_SOURCES:
            raise ValueError(f"only a rule on {', '.join(HOST_SOURCES)} can cover hosts; "
                             f"{source_module}'s resources are not hosts.")
        resource_name = ""
        if covers == "group":
            from steward.hosts.service import group_names
            groups = await group_names(db)
            if cover_group not in groups:
                raise ValueError(f"there is no inventory group {cover_group!r}. "
                                 f"Groups: {_choices(groups)}.")
        else:
            cover_group = None
    metric_name = (metric_name or "").strip()
    metrics = await metrics_for(db, source_module)
    if metric_name not in metrics:
        raise ValueError(f"{source_module} does not report {metric_name!r}. "
                         f"Its metrics: {_choices(metrics)}.")
    try:
        op = AlertOperator((operator or "").strip())
    except ValueError:
        raise ValueError(f"operator {operator!r} is not one of "
                         f"{', '.join(o.value for o in AlertOperator)}.") from None
    try:
        threshold = float(threshold)
    except (TypeError, ValueError):
        raise ValueError(f"threshold must be a number, got {threshold!r}.") from None
    try:
        consecutive = (1 if consecutive_failures_required in (None, "")
                       else int(consecutive_failures_required))
    except (TypeError, ValueError):
        raise ValueError("consecutive checks must be a whole number.") from None
    if consecutive < 1:
        raise ValueError("consecutive checks must be at least 1.")
    try:
        ok_checks = (1 if consecutive_ok_required in (None, "")
                     else int(consecutive_ok_required))
    except (TypeError, ValueError):
        raise ValueError("checks to clear must be a whole number.") from None
    if ok_checks < 1:
        raise ValueError("checks to clear must be at least 1.")
    return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
            "metric_name": metric_name, "operator": op, "threshold": threshold,
            "consecutive_failures_required": consecutive,
            "consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data),
            "covers": covers, "cover_group": cover_group}


# Changing any of these changes what the rule is watching, so whatever state
# it had built up no longer means anything.
WATCHED_FIELDS = ("source_module", "resource_name", "metric_name", "operator", "threshold")


async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool | None = None,
                            actor_id: str | None = None) -> None:
    """Apply validated fields (and optionally enabled) to a rule, in the
    caller's transaction. A change to what it watches, or disabling or
    re-enabling it, puts its alert back to inactive and starts its silence
    watch over (core.alerts.reset_rule_state); the event says who did it."""
    from steward.core.alerts import reset_rule_state
    changed = [k for k in WATCHED_FIELDS if k in fields and getattr(rule, k) != fields[k]]
    for k, v in fields.items():
        setattr(rule, k, v)
    disabling = enabled is False and rule.enabled
    enabling = enabled is True and not rule.enabled
    if enabled is not None:
        rule.enabled = bool(enabled)
    if disabling:
        await reset_rule_state(db, rule, "rule disabled", actor_id)
    elif enabling:
        await reset_rule_state(db, rule, "rule enabled", actor_id)
    elif changed:
        await reset_rule_state(db, rule, "rule changed: " + ", ".join(changed), actor_id)


async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
                   ansible_action: dict | None = None) -> AlertRule:
    """Insert a rule and its inactive state row, in the caller's transaction.
    A covering rule gets no state; its per-host children are made here."""
    rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
    db.add(rule)
    await db.flush()
    if rule.covers != "resource":
        await reconcile_children(db, rule)
        return rule
    db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
                      last_transition_at=datetime.now(timezone.utc)))
    return rule


# ── Covering rules (#5464) ───────────────────────────────────────────────────
# A rule can cover every host, or every host in an inventory group (inventory
# groups are Steward's grouping system: the Hosts ruling, #5294). Steward keeps
# one ordinary child rule per covered host, so each host's alert has its own
# state, history and acknowledgement, and a host that appears later is covered
# without anyone copying the rule.

COVERS = ("resource", "all_hosts", "group")
# Sources whose bare resource names are hosts ("<host>", parts are "<host>:…").
HOST_SOURCES = ("host_agent",)
# A host counts as present while it has reported within this long: the raw
# metrics window, so a decommissioned host's children go once its data does.
COVER_SEEN_DAYS = 7


def covers_text(rule: AlertRule) -> str:
    """What a covering rule spans, in words: "every host", "every host in web"."""
    if rule.covers == "group":
        return f"every host in {rule.cover_group}"
    return "every host" if rule.covers == "all_hosts" else rule.resource_name


async def covered_hosts(db, rule: AlertRule, now: datetime | None = None) -> list[str]:
    """The host names a covering rule spans now: hosts that have reported
    under its source within COVER_SEEN_DAYS, all of them or those whose
    linked Ansible target is in its group."""
    from datetime import timedelta
    from steward.hosts.service import host_groups
    from steward.models.hosts import Host
    now = now or datetime.now(timezone.utc)
    reported = set((await db.execute(
        select(PluginMetricLatest.resource_name)
        .where(PluginMetricLatest.source_module == rule.source_module,
               PluginMetricLatest.recorded_at >= now - timedelta(days=COVER_SEEN_DAYS),
               ~PluginMetricLatest.resource_name.contains(":"))
        .distinct())).scalars())
    if rule.covers != "group":
        return sorted(reported, key=str.lower)
    hosts = (await db.execute(select(Host.id, Host.name).where(Host.name.in_(reported)))).all()
    groups = await host_groups(db, [hid for hid, _ in hosts])
    return sorted((name for hid, name in hosts if rule.cover_group in groups.get(hid, [])),
                  key=str.lower)


def child_fields(parent: AlertRule, host: str) -> dict:
    """A covering rule's child for one host: the parent's condition on that
    host, named "<parent>: <host>"."""
    return {"name": f"{parent.name}: {host}"[:128], "source_module": parent.source_module,
            "resource_name": host, "metric_name": parent.metric_name,
            "operator": parent.operator, "threshold": parent.threshold,
            "consecutive_failures_required": parent.consecutive_failures_required,
            "consecutive_ok_required": parent.consecutive_ok_required,
            "notify_no_data": parent.notify_no_data}


async def reconcile_children(db, parent: AlertRule, *, actor_id: str | None = None,
                             now: datetime | None = None) -> tuple[list[str], list[str]]:
    """Bring a covering rule's children in line with it, in the caller's
    transaction: one per covered host, each carrying the parent's condition,
    action and on/off. A host newly covered gets a child; a child whose host
    is no longer covered is deleted, its history with it. Returns (hosts
    added, hosts removed)."""
    hosts = set(await covered_hosts(db, parent, now))
    children = {c.resource_name: c for c in (await db.execute(
        select(AlertRule).where(AlertRule.parent_id == parent.id))).scalars()}
    added = sorted(hosts - children.keys(), key=str.lower)
    removed = sorted(children.keys() - hosts, key=str.lower)
    for host in added:
        await add_rule(db, {**child_fields(parent, host), "parent_id": parent.id},
                       created_by=parent.created_by, enabled=parent.enabled,
                       ansible_action=parent.ansible_action)
    for host in removed:
        await db.delete(children[host])
    for host, child in children.items():
        if host in hosts:
            fields = child_fields(parent, host)
            if (any(getattr(child, k) != v for k, v in fields.items())
                    or child.enabled != parent.enabled):
                await apply_rule_change(db, child, fields, enabled=parent.enabled,
                                        actor_id=actor_id)
            if child.ansible_action != parent.ansible_action:
                child.ansible_action = parent.ansible_action
    return added, removed


async def reconcile_covering_rules(app) -> None:
    """Every covering rule's children brought in line with the hosts there are
    now. Run each minute by the core scheduler (alert_reconcile), so a host
    that joins a group, or starts reporting, is covered within a minute."""
    async with app.db_sessionmaker() as db:
        async with db.begin():
            parents = (await db.execute(
                select(AlertRule).where(AlertRule.covers != "resource"))).scalars().all()
            for parent in parents:
                added, removed = await reconcile_children(db, parent)
                if added or removed:
                    logger.info("Covering rule %r: added %s, removed %s", parent.name,
                                ", ".join(added) or "none", ", ".join(removed) or "none")


async def ensure_rule_states(app) -> int:
    """A state row, inactive, for every rule watching one resource that has
    none. Evaluation skips a rule without one, and a backup leaves alert state
    out (System 180, 2026-10-15), so a restored rule comes back without it.
    Run each minute by the core scheduler (alert_reconcile): a restored rule is
    watched again within a minute. Returns how many rows were added."""
    async with app.db_sessionmaker() as db:
        async with db.begin():
            has_state = select(AlertState.id).where(AlertState.rule_id == AlertRule.id).exists()
            missing = (await db.execute(select(AlertRule.id).where(
                AlertRule.covers == "resource", ~has_state))).scalars().all()
            now = datetime.now(timezone.utc)
            db.add_all([AlertState(rule_id=rule_id, state=AlertStateEnum.inactive,
                                   last_transition_at=now) for rule_id in missing])
    if missing:
        logger.info("Alert rules without a state row: added %d", len(missing))
    return len(missing)


def part_of(rule: AlertRule, rules) -> AlertRule | None:
    """The covering rule a child belongs to, from a list of rules; None for a
    rule that stands alone."""
    return next((r for r in rules if r.id == rule.parent_id), None) if rule.parent_id else None


async def test_alert(db, rule: AlertRule) -> dict:
    """What a Send test carries for `rule` (#5466): a TEST alert with the
    rule's name and condition, on its resource at the latest reading. A rule
    covering hosts tests on its first host."""
    target = rule
    if rule.covers != "resource":
        target = (await db.execute(
            select(AlertRule).where(AlertRule.parent_id == rule.id)
            .order_by(AlertRule.resource_name).limit(1))).scalar_one_or_none() or rule
    return {"rule_name": rule.name, "state": "TEST", "metric": rule.metric_name,
            "value": await latest_value(db, target) if target.resource_name else None,
            "operator": rule.operator.value, "threshold": rule.threshold,
            "resource": target.resource_name or covers_text(rule),
            "source_module": rule.source_module,
            "note": "A test of this rule's notifications. Nothing has fired.",
            "timestamp": datetime.now(timezone.utc).isoformat()}


async def send_test(app, rule_id: str) -> dict:
    """Send a TEST notification for a rule now, by every channel a real alert
    of it would use: email to whoever gets alerts, each in their own zone, and
    the webhook. Not held for a burst, and not recorded in the rule's history.

    Returns {"rule", "ok", "lines"}: ok when something was sent and nothing
    failed; lines say what happened to each channel. LookupError for no rule.
    """
    from urllib.parse import urlsplit
    from steward.core import outbox
    from steward.core.mail import alert_email, recipients
    from steward.core.notifications import dispatch_notifications
    smtp, webhook = app.config.get("SMTP", {}), app.config.get("WEBHOOK", {})
    async with app.db_sessionmaker() as db:
        rule = await db.get(AlertRule, rule_id)
        if rule is None:
            raise LookupError(rule_id)
        alert = await test_alert(db, rule)
        to = await recipients(db, "alerts", smtp) if smtp.get("host") else []
    results = await dispatch_notifications(webhook, alert)
    lines = []
    if to:
        # Queued and recorded like any email, tried once, not retried.
        ok, said = outbox.said(await outbox.send_each(
            app, "test", to, lambda r: alert_email(smtp, alert, r), retry=False))
        results["email"] = {"sent": ok, "error": None if ok else said}
        lines.append(f"Email: {said}")
    if hook := results.get("webhook"):
        host = urlsplit(webhook["url"]).hostname or webhook["url"]
        lines.append(f"Webhook to {host} delivered." if hook["sent"]
                     else f"Webhook to {host} failed: {hook['error']}")
    if not lines:
        lines.append("Nothing is set up to send to: no email server with someone to send to, "
                     "and no webhook.")
    return {"rule": rule.name, "ok": bool(results) and all(r["sent"] for r in results.values()),
            "lines": lines, "email": results.get("email"), "webhook": results.get("webhook")}


class PartOfCoveringRule(ValueError):
    """A child of a covering rule is changed through its parent, never alone."""

    def __init__(self, child: AlertRule, parent_name: str):
        super().__init__(f"{child.name} is part of the rule {parent_name!r}, which keeps one "
                         f"rule per host; change or disable it there.")


async def acknowledge(db, rule_id: str, user_id: str | None) -> AlertState:
    """Acknowledge a firing alert, in the caller's transaction, and record it
    in the alert's history as a firing → acknowledged event by `user_id`, so
    who saw it outlives the alert (#5463).

    LookupError when the rule has no state; ValueError when it is not firing.
    """
    from steward.core.alerts import _add_event
    state = (await db.execute(
        select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one_or_none()
    if state is None:
        raise LookupError(rule_id)
    if state.state != AlertStateEnum.firing:
        raise ValueError(f"the alert is {state.state.value}, not firing; only a firing alert "
                         "can be acknowledged.")
    now = datetime.now(timezone.utc)
    state.state = AlertStateEnum.acknowledged
    state.acknowledged_by = user_id
    state.acknowledged_at = now
    state.last_transition_at = now
    _add_event(db, rule_id, AlertStateEnum.firing.value, AlertStateEnum.acknowledged.value,
               None, now, actor_id=user_id)
    return state


def window_fields(*, name: str, start_at: datetime, end_at: datetime,
                  scope_source: str | None, scope_resource: str | None) -> dict:
    """Validated columns for a maintenance window. A resource needs a source."""
    name = (name or "").strip()
    if not name:
        raise ValueError("a maintenance window needs a name.")
    if end_at <= start_at:
        raise ValueError("the window must end after it starts.")
    scope_source = (scope_source or "").strip() or None
    scope_resource = (scope_resource or "").strip() or None
    catalog = metric_catalog()
    if scope_source is not None and scope_source not in catalog:
        raise ValueError(f"source {scope_source!r} is not one of {', '.join(catalog)}.")
    if scope_source is None:
        scope_resource = None  # can't have resource without source
    return {"name": name[:128], "start_at": start_at, "end_at": end_at,
            "scope_source": scope_source, "scope_resource": scope_resource}


def window_scope(w) -> str:
    if not w.scope_source:
        return "all"
    return f"{w.scope_source}/{w.scope_resource}" if w.scope_resource else f"{w.scope_source}/*"