FabledSteward/steward/mcp/tools/_events.py

"""What else was going on: events on a host, or one resource, in a window
(#5837), so an anomaly comes with its circumstances.

Core gathers alert transitions, Ansible runs and other metric anomalies. A
plugin adds its own through `mcp_events(db, host_id, host_name, since, until)`
in its __init__.py (the host agent its maintenance jobs and downtime, Docker
its container events); each returns a list of `event(...)`.
"""
from __future__ import annotations

import logging
from dataclasses import dataclass
from datetime import datetime

from sqlalchemy import and_, or_, select

from steward.alerts.service import watches_host
from steward.core.alerts import host_of_resource
from steward.mcp._context import mcp_app
from steward.models.alerts import AlertEvent, AlertRule
from steward.models.ansible import AnsibleRun
from steward.models.ansible_inventory import AnsibleTarget
from steward.models.hosts import Host
from steward.models.monitors import Monitor
from steward.models.norms import MetricAnomaly, MetricAnomalySpan

from ._shape import db_session, iso

logger = logging.getLogger(__name__)

PER_SOURCE = 40     # events any one source contributes to one window


def event(at: datetime | None, kind: str, what: str, *, until: datetime | None = None,
          subject: str | None = None, ids: dict | None = None) -> dict:
    """One thing that happened. `until` for something that lasted (a run, a
    reboot); `subject` names what it happened to when that isn't the host."""
    return {"at": iso(at), "until": iso(until), "kind": kind, "what": what,
            "subject": subject, "ids": ids or {}}


@dataclass(frozen=True)
class Place:
    """Where a metric's resource lives: its host, when it has one."""
    source: str
    resource: str
    host_id: str | None = None
    host_name: str | None = None


async def place_of(db, source: str, resource: str) -> Place:
    """The host behind a resource: an agent-fed series names it ("nas:/",
    "nas/web"), a monitor may be pinned to one; a Traefik router has none."""
    name = host_of_resource(source, resource)
    host = None
    if name:
        host = (await db.execute(select(Host).where(Host.name == name).limit(1))).scalar_one_or_none()
    else:
        monitor = (await db.execute(select(Monitor).where(
            Monitor.type == source, Monitor.name == resource).limit(1))).scalar_one_or_none()
        if monitor is not None and monitor.host_id:
            host = await db.get(Host, monitor.host_id)
    return Place(source, resource, host.id if host else None, host.name if host else None)


def _overlaps(start, end):
    """A row lasting from `start` to `end` (None: still going) overlaps the window."""
    def clause(since, until):
        return and_(start <= until, or_(end.is_(None), end >= since))
    return clause


async def alert_events(db, place: Place, since: datetime, until: datetime) -> list[dict]:
    """Alert transitions on the resource itself, or anything its host
    carries: its agent series, its containers, its monitors."""
    monitor_keys = set()
    if place.host_id:
        monitor_keys = {(t, n) for t, n in (await db.execute(
            select(Monitor.type, Monitor.name).where(Monitor.host_id == place.host_id))).all()}
    rows = (await db.execute(
        select(AlertEvent, AlertRule).join(AlertRule, AlertRule.id == AlertEvent.rule_id)
        .where(AlertEvent.transitioned_at >= since, AlertEvent.transitioned_at <= until)
        .order_by(AlertEvent.transitioned_at).limit(500))).all()
    out = []
    for ev, rule in rows:
        mine = (rule.source_module, rule.resource_name) == (place.source, place.resource)
        if not mine and not (place.host_name and watches_host(rule, place.host_name, monitor_keys)):
            continue
        note = f" ({ev.note})" if ev.note else ""
        out.append(event(ev.transitioned_at, "alert", f"{rule.name}: {ev.from_state} → {ev.to_state}{note}",
                         subject=rule.resource_name, ids={"rule_id": rule.id}))
    return out[:PER_SOURCE]


async def ansible_runs(db, place: Place, since: datetime, until: datetime) -> list[dict]:
    """Ansible runs that touched the host while the window was open (by the
    hosts in their recap: a run still going hasn't said yet)."""
    if not place.host_id:
        return []
    names = {place.host_name} | set((await db.execute(
        select(AnsibleTarget.name).where(AnsibleTarget.host_id == place.host_id))).scalars())
    runs = (await db.execute(
        select(AnsibleRun).where(_overlaps(AnsibleRun.started_at, AnsibleRun.finished_at)(since, until))
        .order_by(AnsibleRun.started_at).limit(200))).scalars()
    out = []
    for r in runs:
        if not names & set(((r.results or {}).get("hosts") or {}).keys()):
            continue
        check = " in check mode" if (r.params or {}).get("check") else ""
        out.append(event(r.started_at, "ansible_run",
                         f"Ansible {r.playbook_path}{check}: {r.status.value}",
                         until=r.finished_at, ids={"run_id": r.id}))
    return out[:PER_SOURCE]


def _same_host(source: str, resource: str, place: Place) -> bool:
    if (source, resource) == (place.source, place.resource):
        return True
    return bool(place.host_name) and host_of_resource(source, resource) == place.host_name


async def other_anomalies(db, place: Place, anomaly_id: str, since: datetime,
                          until: datetime) -> tuple[list[dict], int]:
    """Other anomalies going on in the window: those on the same host as
    events, and how many there were anywhere else (many at once points at
    something shared: the network, storage, Steward itself)."""
    rows = (await db.execute(
        select(MetricAnomalySpan, MetricAnomaly)
        .join(MetricAnomaly, MetricAnomaly.id == MetricAnomalySpan.anomaly_id)
        .where(MetricAnomaly.id != anomaly_id,
               _overlaps(MetricAnomalySpan.started_at, MetricAnomalySpan.ended_at)(since, until))
        .order_by(MetricAnomalySpan.started_at).limit(500))).all()
    here, elsewhere = [], 0
    for span, a in rows:
        if _same_host(a.source_module, a.resource_name, place):
            here.append(event(span.started_at, "anomaly",
                              f"{a.resource_name} {a.metric_name} anomaly (peak {round(span.peak_level, 3)}, "
                              f"usual {round(a.usual_hold, 3)}; {a.review})",
                              until=span.ended_at, subject=a.resource_name, ids={"anomaly_id": a.id}))
        else:
            elsewhere += 1
    return here[:PER_SOURCE], elsewhere


async def plugin_events(place: Place, since: datetime, until: datetime) -> list[dict]:
    """What enabled plugins know happened on the host (`mcp_events`), each
    in its own session so one that fails can't spoil the others' reads."""
    server = getattr(mcp_app(), "mcp_server", None)
    if server is None or not place.host_id:
        return []
    out = []
    for plugin, source in sorted((getattr(server, "event_sources", None) or {}).items()):
        if not server.plugin_enabled(plugin):
            continue
        try:
            async with db_session() as db:
                out.extend((await source(db, place.host_id, place.host_name, since, until))[:PER_SOURCE])
        except Exception:
            logger.exception("MCP events source of plugin %r failed", plugin)
    return out


async def around(db, place: Place, anomaly_id: str, since: datetime,
                 until: datetime) -> tuple[list[dict], int]:
    """Everything above for one window, oldest first, and the count of
    anomalies elsewhere."""
    anomalies, elsewhere = await other_anomalies(db, place, anomaly_id, since, until)
    found = (await alert_events(db, place, since, until) + await ansible_runs(db, place, since, until)
             + anomalies + await plugin_events(place, since, until))
    return sorted(found, key=lambda e: e["at"] or ""), elsewhere