FabledSteward/steward/alerts/service.py

"""Alert queries and validation shared by the alerts pages and the MCP tools.

Each function takes an open session (or plain values), with no Quart request
context, so the routes and the /mcp tools (which run outside Quart's
pipeline) call the same code and refuse the same input for the same reasons.
"""
from __future__ import annotations

import logging
from datetime import datetime, timezone

from sqlalchemy import func, or_, select

from steward.models.alerts import (
    AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
)
from steward.models.metrics import PluginMetric, PluginMetricLatest
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor

logger = logging.getLogger(__name__)

# What core itself records. Unified monitors emit metrics under their TYPE
# (icmp/tcp/dns/http), each with is_up (1.0 up / 0.0 down) and response_ms
# (where the probe times it); "steward" is the app watching itself.
CORE_METRICS: dict[str, list[str]] = {
    "icmp":    ["is_up", "response_ms"],
    "tcp":     ["is_up", "response_ms"],
    "dns":     ["is_up", "response_ms"],
    "http":    ["is_up", "response_ms"],
    "steward": ["open_fds", "open_fds_pct"],
}


def metric_catalog() -> dict[str, list[str]]:
    """Every source an alert rule can watch, and its metric names.

    Core's own sources, then each loaded plugin's get_alert_metrics() export
    ({source: [metric names]}), so the catalog is written by whatever emits
    the metrics and a disabled plugin's sources drop out. An empty list means
    the names are dynamic (snmp: the OID labels recorded so far).
    """
    from steward.core.plugin_manager import get_loaded_plugin_modules
    catalog = {src: list(names) for src, names in CORE_METRICS.items()}
    for name, module in get_loaded_plugin_modules().items():
        hook = getattr(module, "get_alert_metrics", None)
        if hook is None:
            continue
        try:
            contributed = hook() or {}
        except Exception:
            logger.exception("Plugin %r: get_alert_metrics() raised; its metrics are skipped", name)
            continue
        for src, names in contributed.items():
            catalog[str(src)] = [str(n) for n in names]
    return catalog


def source_modules() -> list[str]:
    """The sources in picker order: core's first, then plugins'."""
    return list(metric_catalog())


# States that need someone's attention: pending (breaching, not yet for long
# enough), firing, and acknowledged-but-still-breaching.
ACTIVE_STATES = (AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending)


async def active_alerts(db) -> list:
    """(AlertState, AlertRule) rows that need attention, latest transition
    first: not yet resolved, or with a series that has gone silent (no_data)."""
    return list((await db.execute(
        select(AlertState, AlertRule)
        .join(AlertRule, AlertState.rule_id == AlertRule.id)
        .where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
        .order_by(AlertState.last_transition_at.desc())
    )).all())


def alert_label(state: AlertState) -> str:
    """What an alert is, in one word: its state, or "no_data" for a quiet rule
    whose series has gone silent (a firing alert that goes silent stays
    "firing"; check no_data_since for the mark)."""
    if state.no_data_since is not None and state.state == AlertStateEnum.inactive:
        return "no_data"
    return state.state.value


async def latest_value(db, rule: AlertRule) -> float | None:
    """The newest sample of the series a rule watches, or None before the first
    (a primary-key read of the latest-value table, #5208)."""
    return (await db.execute(
        select(PluginMetricLatest.value)
        .where(PluginMetricLatest.source_module == rule.source_module,
               PluginMetricLatest.resource_name == rule.resource_name,
               PluginMetricLatest.metric_name == rule.metric_name)
    )).scalar_one_or_none()


# Sources whose resources have no page of their own: the plugin's page lists them.
PLUGIN_PAGES = {"traefik": "/plugins/traefik/", "unifi": "/plugins/unifi/",
                "snmp": "/plugins/snmp/"}


async def resource_url(db, rule: AlertRule) -> str | None:
    """Where the thing a rule watches is shown, or None when nothing shows it.

    host_agent resources are the host's name, or "<host>:<mount|nic|core>";
    docker's are "<host>/<container>"; a monitor's is its name under its type.
    """
    from steward.models.hosts import Host
    source, resource = rule.source_module, rule.resource_name
    if source == "host_agent":
        name = resource.split(":", 1)[0]
        host_id = (await db.execute(
            select(Host.id).where(Host.name == name).limit(1))).scalar_one_or_none()
        return f"/hosts/{host_id}" if host_id else None
    if source in MONITOR_TYPE_VALUES:
        monitor_id = (await db.execute(
            select(Monitor.id).where(Monitor.type == source, Monitor.name == resource).limit(1)
        )).scalar_one_or_none()
        return f"/monitors/{monitor_id}" if monitor_id else None
    if source == "docker" and "/" in resource:
        host_name, container = resource.split("/", 1)
        host_id = (await db.execute(
            select(Host.id).where(Host.name == host_name).limit(1))).scalar_one_or_none()
        return f"/plugins/docker/container/{host_id}/{container}" if host_id else None
    return PLUGIN_PAGES.get(source)


# Metrics whose value is a word: the sentence says the word, not the condition.
_BINARY = ("is_up", "is_healthy")


def alert_sentence(rule: AlertRule, label: str, value: float | None,
             since: datetime | None, now: datetime) -> dict:
    """An alert as one sentence around its resource: {"lead", "resource",
    "tail"}, read lead + resource + tail, e.g. "cpu_pct on " + "web-1" +
    " is 94% (> 90%), firing for 12m". label is alert_label(state); since is
    when that label began (no_data_since for no_data)."""
    from steward.core.formatting import human_duration, metric_value
    metric = rule.metric_name
    held = human_duration((now - since).total_seconds()) if since else None
    if label == "no_data":
        return {"lead": f"No {metric} from ", "resource": rule.resource_name,
                "tail": f" for {held}" if held else ""}
    # An acknowledged alert is still breaching: say for how long it has been,
    # not how long ago someone acknowledged it (the page says that apart).
    word = "breaching" if label == "acknowledged" else label
    for_how_long = f", {word} for {held}" if held else f", {word}"
    if value is None:
        reading = " has no reading yet"
    elif metric in _BINARY:
        return {"lead": "", "resource": rule.resource_name,
                "tail": f" is {metric_value(metric, value)}{for_how_long}"}
    else:
        reading = f" is {metric_value(metric, value)}"
    condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})"
    return {"lead": f"{metric} on ", "resource": rule.resource_name,
            "tail": f"{reading}{condition}{for_how_long}"}


# The order the active list reads in: what is breaching and unhandled, then
# what has gone silent, then what someone has taken, then what may yet clear.
ACTIVE_ORDER = {"firing": 0, "no_data": 1, "acknowledged": 2, "pending": 3}

_BREACHING = (AlertStateEnum.pending.value, AlertStateEnum.firing.value,
              AlertStateEnum.acknowledged.value)


async def breach_started(db, rule_ids) -> dict[str, datetime]:
    """{rule_id: when its current breach began}: the newest transition into
    pending or firing from a quiet state. Acknowledging moves an alert's
    last_transition_at, so that column cannot say this."""
    if not rule_ids:
        return {}
    return dict((await db.execute(
        select(AlertEvent.rule_id, func.max(AlertEvent.transitioned_at))
        .where(AlertEvent.rule_id.in_(list(rule_ids)),
               AlertEvent.from_state.not_in(_BREACHING),
               AlertEvent.to_state.in_((AlertStateEnum.pending.value, AlertStateEnum.firing.value)))
        .group_by(AlertEvent.rule_id)
    )).all())


async def active_rows(db, now: datetime) -> list[dict]:
    """The alerts page's active list, firing first, then no data, then
    acknowledged, then pending; within each the longest-running first. Each
    row: its state, rule, label, sentence, the link to what it watches, when
    the breach began, and who acknowledged it."""
    from steward.models.users import User
    alerts = await active_alerts(db)
    started = await breach_started(db, {rule.id for state, rule in alerts
                                        if state.state.value in _BREACHING})
    ackers = {s.acknowledged_by for s, _ in alerts if s.acknowledged_by}
    names = dict((await db.execute(
        select(User.id, User.username).where(User.id.in_(ackers)))).all()) if ackers else {}
    rows = []
    for state, rule in alerts:
        label = alert_label(state)
        if label == "no_data":
            since = state.no_data_since
        elif label == "acknowledged":
            since = started.get(rule.id, state.last_transition_at)
        else:
            since = state.last_transition_at
        rows.append({
            "state": state, "rule": rule, "label": label, "since": since,
            "breach_since": started.get(rule.id) if label != "no_data" else None,
            "acked_by": names.get(state.acknowledged_by, state.acknowledged_by),
            "says": alert_sentence(rule, label, await latest_value(db, rule), since, now),
            "url": await resource_url(db, rule),
        })
    epoch = datetime.max.replace(tzinfo=timezone.utc)
    rows.sort(key=lambda r: (ACTIVE_ORDER.get(r["label"], 9), r["since"] or epoch))
    return rows


async def active_count(db) -> int:
    """How many alerts need attention: the sidebar's number."""
    from sqlalchemy import func
    return (await db.execute(
        select(func.count()).select_from(AlertState)
        .where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
    )).scalar_one()


async def list_rules(db) -> list[AlertRule]:
    return list((await db.execute(select(AlertRule).order_by(AlertRule.name))).scalars())


async def history(db, since: datetime, rule_id: str | None = None,
                  limit: int = 200) -> list[tuple[AlertEvent, AlertRule]]:
    """Alert transitions since `since`, newest first, with their rule; one
    rule's when `rule_id` is given. Returns up to limit+1 rows so a caller can
    tell the list was cut."""
    stmt = (select(AlertEvent, AlertRule)
            .join(AlertRule, AlertEvent.rule_id == AlertRule.id)
            .where(AlertEvent.transitioned_at >= since)
            .order_by(AlertEvent.transitioned_at.desc())
            .limit(limit + 1))
    if rule_id:
        stmt = stmt.where(AlertEvent.rule_id == rule_id)
    return [tuple(row) for row in (await db.execute(stmt)).all()]


async def list_maintenance_windows(db) -> list[MaintenanceWindow]:
    """Every window, latest start first."""
    return list((await db.execute(
        select(MaintenanceWindow).order_by(MaintenanceWindow.start_at.desc())
    )).scalars())


async def resources_for(db, source_module: str) -> list[str]:
    """Sorted distinct resource names for a source module.

    For monitor types (icmp/tcp/dns/http), supplements with monitor names so
    the list is populated even before any metrics have been recorded.
    """
    result = await db.execute(
        select(PluginMetric.resource_name)
        .where(PluginMetric.source_module == source_module)
        .distinct()
        .order_by(PluginMetric.resource_name)
    )
    resources: set[str] = {r for (r,) in result}
    if source_module in MONITOR_TYPE_VALUES:
        mon_result = await db.execute(
            select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name))
        resources.update(name for (name,) in mon_result)
    return sorted(resources)


async def metrics_for(db, source_module: str) -> list[str]:
    """The metric names a rule on this source can watch: the static catalog, or
    for dynamic sources (snmp) the names recorded so far."""
    metrics = metric_catalog().get(source_module, [])
    if source_module and not metrics:
        result = await db.execute(
            select(PluginMetric.metric_name)
            .where(PluginMetric.source_module == source_module)
            .distinct()
            .order_by(PluginMetric.metric_name))
        metrics = [r for (r,) in result]
    return metrics


def _choices(values: list[str], limit: int = 20) -> str:
    if not values:
        return "none yet"
    shown = ", ".join(values[:limit])
    return shown + (f" (and {len(values) - limit} more)" if len(values) > limit else "")


async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
                      operator: str, threshold, consecutive_failures_required=1,
                      consecutive_ok_required=1, notify_no_data=True) -> dict:
    """Validated column values for a new or edited alert rule.

    The same choices the rule form offers: a known source, a resource that
    source has reported (or, for monitors, a monitor of that type), and a
    metric it emits. ValueError says what was wrong and what is valid.
    """
    name = (name or "").strip()
    if not name:
        raise ValueError("an alert rule needs a name.")
    source_module = (source_module or "").strip()
    catalog = metric_catalog()
    if source_module not in catalog:
        raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.")
    resource_name = (resource_name or "").strip()
    resources = await resources_for(db, source_module)
    if resource_name not in resources:
        raise ValueError(f"{source_module} has no resource {resource_name!r}. "
                         f"Known: {_choices(resources)}.")
    metric_name = (metric_name or "").strip()
    metrics = await metrics_for(db, source_module)
    if metric_name not in metrics:
        raise ValueError(f"{source_module} does not report {metric_name!r}. "
                         f"Its metrics: {_choices(metrics)}.")
    try:
        op = AlertOperator((operator or "").strip())
    except ValueError:
        raise ValueError(f"operator {operator!r} is not one of "
                         f"{', '.join(o.value for o in AlertOperator)}.") from None
    try:
        threshold = float(threshold)
    except (TypeError, ValueError):
        raise ValueError(f"threshold must be a number, got {threshold!r}.") from None
    try:
        consecutive = (1 if consecutive_failures_required in (None, "")
                       else int(consecutive_failures_required))
    except (TypeError, ValueError):
        raise ValueError("consecutive checks must be a whole number.") from None
    if consecutive < 1:
        raise ValueError("consecutive checks must be at least 1.")
    try:
        ok_checks = (1 if consecutive_ok_required in (None, "")
                     else int(consecutive_ok_required))
    except (TypeError, ValueError):
        raise ValueError("checks to clear must be a whole number.") from None
    if ok_checks < 1:
        raise ValueError("checks to clear must be at least 1.")
    return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
            "metric_name": metric_name, "operator": op, "threshold": threshold,
            "consecutive_failures_required": consecutive,
            "consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data)}


# Changing any of these changes what the rule is watching, so whatever state
# it had built up no longer means anything.
WATCHED_FIELDS = ("source_module", "resource_name", "metric_name", "operator", "threshold")


async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool | None = None) -> None:
    """Apply validated fields (and optionally enabled) to a rule, in the
    caller's transaction. A change to what it watches, or disabling or
    re-enabling it, puts its alert back to inactive and starts its silence
    watch over (core.alerts.reset_rule_state)."""
    from steward.core.alerts import reset_rule_state
    changed = [k for k in WATCHED_FIELDS if k in fields and getattr(rule, k) != fields[k]]
    for k, v in fields.items():
        setattr(rule, k, v)
    disabling = enabled is False and rule.enabled
    enabling = enabled is True and not rule.enabled
    if enabled is not None:
        rule.enabled = bool(enabled)
    if disabling:
        await reset_rule_state(db, rule, "rule disabled")
    elif enabling:
        await reset_rule_state(db, rule, "rule enabled")
    elif changed:
        await reset_rule_state(db, rule, "rule changed: " + ", ".join(changed))


async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
                   ansible_action: dict | None = None) -> AlertRule:
    """Insert a rule and its inactive state row, in the caller's transaction."""
    rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
    db.add(rule)
    await db.flush()
    db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
                      last_transition_at=datetime.now(timezone.utc)))
    return rule


async def acknowledge(db, rule_id: str, user_id: str) -> AlertState:
    """Acknowledge a firing alert, in the caller's transaction.

    LookupError when the rule has no state; ValueError when it is not firing.
    """
    state = (await db.execute(
        select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one_or_none()
    if state is None:
        raise LookupError(rule_id)
    if state.state != AlertStateEnum.firing:
        raise ValueError(f"the alert is {state.state.value}, not firing; only a firing alert "
                         "can be acknowledged.")
    now = datetime.now(timezone.utc)
    state.state = AlertStateEnum.acknowledged
    state.acknowledged_by = user_id
    state.acknowledged_at = now
    state.last_transition_at = now
    return state


def window_fields(*, name: str, start_at: datetime, end_at: datetime,
                  scope_source: str | None, scope_resource: str | None) -> dict:
    """Validated columns for a maintenance window. A resource needs a source."""
    name = (name or "").strip()
    if not name:
        raise ValueError("a maintenance window needs a name.")
    if end_at <= start_at:
        raise ValueError("the window must end after it starts.")
    scope_source = (scope_source or "").strip() or None
    scope_resource = (scope_resource or "").strip() or None
    catalog = metric_catalog()
    if scope_source is not None and scope_source not in catalog:
        raise ValueError(f"source {scope_source!r} is not one of {', '.join(catalog)}.")
    if scope_source is None:
        scope_resource = None  # can't have resource without source
    return {"name": name[:128], "start_at": start_at, "end_at": end_at,
            "scope_source": scope_source, "scope_resource": scope_resource}


def window_scope(w) -> str:
    if not w.scope_source:
        return "all"
    return f"{w.scope_source}/{w.scope_resource}" if w.scope_resource else f"{w.scope_source}/*"