FabledSteward/steward/alerts/service.py

"""Alert queries and validation shared by the alerts pages and the MCP tools.

Each function takes an open session (or plain values), with no Quart request
context, so the routes and the /mcp tools (which run outside Quart's
pipeline) call the same code and refuse the same input for the same reasons.
"""
from __future__ import annotations

from datetime import datetime, timezone

from sqlalchemy import select

from steward.models.alerts import (
    AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
)
from steward.models.metrics import PluginMetric
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor

# Static catalog: source_module → available metric names.
# Unified monitors emit metrics under their TYPE (icmp/tcp/dns/http), each with
# is_up (1.0 up / 0.0 down) and response_ms (where the probe times it).
METRIC_CATALOG: dict[str, list[str]] = {
    "icmp":    ["is_up", "response_ms"],
    "tcp":     ["is_up", "response_ms"],
    "dns":     ["is_up", "response_ms"],
    "http":    ["is_up", "response_ms"],
    "traefik": ["request_rate", "error_rate", "latency_p50_ms", "latency_p95_ms",
                "latency_p99_ms", "response_bytes_rate", "cert_expiry_days"],
    "unifi":   ["is_up", "latency_ms", "total_clients"],
    # restart_count = cumulative restarts (alert on crash-looping); is_healthy =
    # 1.0 healthy / 0.0 unhealthy from the container HEALTHCHECK (alert on <1).
    "docker":  ["cpu_pct", "mem_pct", "restart_count", "is_healthy"],
    "host_agent": [
        "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
        "disk_used_pct_worst", "load_1m", "load_5m", "load_15m", "uptime_secs",
    ],
    # snmp metrics are user-defined OID labels — populated dynamically from plugin_metrics
    "snmp":    [],
}

# Ordered list for the source module picker.
SOURCE_MODULES = list(METRIC_CATALOG.keys())

# States that need someone's attention: pending (breaching, not yet for long
# enough), firing, and acknowledged-but-still-breaching.
ACTIVE_STATES = (AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending)


async def active_alerts(db) -> list:
    """(AlertState, AlertRule) rows not yet resolved, latest transition first."""
    return list((await db.execute(
        select(AlertState, AlertRule)
        .join(AlertRule, AlertState.rule_id == AlertRule.id)
        .where(AlertState.state.in_(ACTIVE_STATES))
        .order_by(AlertState.last_transition_at.desc())
    )).all())


async def list_rules(db) -> list[AlertRule]:
    return list((await db.execute(select(AlertRule).order_by(AlertRule.name))).scalars())


async def list_maintenance_windows(db) -> list[MaintenanceWindow]:
    """Every window, latest start first."""
    return list((await db.execute(
        select(MaintenanceWindow).order_by(MaintenanceWindow.start_at.desc())
    )).scalars())


async def resources_for(db, source_module: str) -> list[str]:
    """Sorted distinct resource names for a source module.

    For monitor types (icmp/tcp/dns/http), supplements with monitor names so
    the list is populated even before any metrics have been recorded.
    """
    result = await db.execute(
        select(PluginMetric.resource_name)
        .where(PluginMetric.source_module == source_module)
        .distinct()
        .order_by(PluginMetric.resource_name)
    )
    resources: set[str] = {r for (r,) in result}
    if source_module in MONITOR_TYPE_VALUES:
        mon_result = await db.execute(
            select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name))
        resources.update(name for (name,) in mon_result)
    return sorted(resources)


async def metrics_for(db, source_module: str) -> list[str]:
    """The metric names a rule on this source can watch: the static catalog, or
    for dynamic sources (snmp) the names recorded so far."""
    metrics = METRIC_CATALOG.get(source_module, [])
    if source_module and not metrics:
        result = await db.execute(
            select(PluginMetric.metric_name)
            .where(PluginMetric.source_module == source_module)
            .distinct()
            .order_by(PluginMetric.metric_name))
        metrics = [r for (r,) in result]
    return metrics


def _choices(values: list[str], limit: int = 20) -> str:
    if not values:
        return "none yet"
    shown = ", ".join(values[:limit])
    return shown + (f" (and {len(values) - limit} more)" if len(values) > limit else "")


async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
                      operator: str, threshold, consecutive_failures_required=1,
                      consecutive_ok_required=1) -> dict:
    """Validated column values for a new or edited alert rule.

    The same choices the rule form offers: a known source, a resource that
    source has reported (or, for monitors, a monitor of that type), and a
    metric it emits. ValueError says what was wrong and what is valid.
    """
    name = (name or "").strip()
    if not name:
        raise ValueError("an alert rule needs a name.")
    source_module = (source_module or "").strip()
    if source_module not in METRIC_CATALOG:
        raise ValueError(f"source_module {source_module!r} is not one of {', '.join(SOURCE_MODULES)}.")
    resource_name = (resource_name or "").strip()
    resources = await resources_for(db, source_module)
    if resource_name not in resources:
        raise ValueError(f"{source_module} has no resource {resource_name!r}. "
                         f"Known: {_choices(resources)}.")
    metric_name = (metric_name or "").strip()
    metrics = await metrics_for(db, source_module)
    if metric_name not in metrics:
        raise ValueError(f"{source_module} does not report {metric_name!r}. "
                         f"Its metrics: {_choices(metrics)}.")
    try:
        op = AlertOperator((operator or "").strip())
    except ValueError:
        raise ValueError(f"operator {operator!r} is not one of "
                         f"{', '.join(o.value for o in AlertOperator)}.") from None
    try:
        threshold = float(threshold)
    except (TypeError, ValueError):
        raise ValueError(f"threshold must be a number, got {threshold!r}.") from None
    try:
        consecutive = (1 if consecutive_failures_required in (None, "")
                       else int(consecutive_failures_required))
    except (TypeError, ValueError):
        raise ValueError("consecutive checks must be a whole number.") from None
    if consecutive < 1:
        raise ValueError("consecutive checks must be at least 1.")
    try:
        ok_checks = (1 if consecutive_ok_required in (None, "")
                     else int(consecutive_ok_required))
    except (TypeError, ValueError):
        raise ValueError("checks to clear must be a whole number.") from None
    if ok_checks < 1:
        raise ValueError("checks to clear must be at least 1.")
    return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
            "metric_name": metric_name, "operator": op, "threshold": threshold,
            "consecutive_failures_required": consecutive,
            "consecutive_ok_required": ok_checks}


# Changing any of these changes what the rule is watching, so whatever state
# it had built up no longer means anything.
WATCHED_FIELDS = ("source_module", "resource_name", "metric_name", "operator", "threshold")


async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool | None = None) -> None:
    """Apply validated fields (and optionally enabled) to a rule, in the
    caller's transaction. A change to what it watches, or disabling it, puts
    its alert back to inactive (core.alerts.reset_rule_state)."""
    from steward.core.alerts import reset_rule_state
    changed = [k for k in WATCHED_FIELDS if k in fields and getattr(rule, k) != fields[k]]
    for k, v in fields.items():
        setattr(rule, k, v)
    disabling = enabled is False and rule.enabled
    if enabled is not None:
        rule.enabled = bool(enabled)
    if disabling:
        await reset_rule_state(db, rule, "rule disabled")
    elif changed:
        await reset_rule_state(db, rule, "rule changed: " + ", ".join(changed))


async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
                   ansible_action: dict | None = None) -> AlertRule:
    """Insert a rule and its inactive state row, in the caller's transaction."""
    rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
    db.add(rule)
    await db.flush()
    db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
                      last_transition_at=datetime.now(timezone.utc)))
    return rule


async def acknowledge(db, rule_id: str, user_id: str) -> AlertState:
    """Acknowledge a firing alert, in the caller's transaction.

    LookupError when the rule has no state; ValueError when it is not firing.
    """
    state = (await db.execute(
        select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one_or_none()
    if state is None:
        raise LookupError(rule_id)
    if state.state != AlertStateEnum.firing:
        raise ValueError(f"the alert is {state.state.value}, not firing; only a firing alert "
                         "can be acknowledged.")
    now = datetime.now(timezone.utc)
    state.state = AlertStateEnum.acknowledged
    state.acknowledged_by = user_id
    state.acknowledged_at = now
    state.last_transition_at = now
    return state


def window_fields(*, name: str, start_at: datetime, end_at: datetime,
                  scope_source: str | None, scope_resource: str | None) -> dict:
    """Validated columns for a maintenance window. A resource needs a source."""
    name = (name or "").strip()
    if not name:
        raise ValueError("a maintenance window needs a name.")
    if end_at <= start_at:
        raise ValueError("the window must end after it starts.")
    scope_source = (scope_source or "").strip() or None
    scope_resource = (scope_resource or "").strip() or None
    if scope_source is not None and scope_source not in METRIC_CATALOG:
        raise ValueError(f"source {scope_source!r} is not one of {', '.join(SOURCE_MODULES)}.")
    if scope_source is None:
        scope_resource = None  # can't have resource without source
    return {"name": name[:128], "start_at": start_at, "end_at": end_at,
            "scope_source": scope_source, "scope_resource": scope_resource}


def window_scope(w) -> str:
    if not w.scope_source:
        return "all"
    return f"{w.scope_source}/{w.scope_resource}" if w.scope_resource else f"{w.scope_source}/*"