FabledSteward/steward/mcp/tools/_events.py

"""What else was going on: events on a host, or one resource, in a window
(#5837), so an anomaly comes with its circumstances.

Core reads the timeline (alert episodes, Ansible runs, and what plugins
wrote there: maintenance, downtime, deploys, restarts), other metric
anomalies, and Steward's own gaps in recording. A plugin adds what the
timeline doesn't hold through `mcp_events(db, host_id, host_name, since,
until)` in its __init__.py (Docker its stops and health changes, syslog its
warnings), and for a resource with no host through
`mcp_resource_events(db, source, resource, since, until)` (Traefik: the
containers of the swarm service behind a router, #5862); each returns a list
of `event(...)`.
"""
from __future__ import annotations

import logging
from dataclasses import dataclass
from datetime import datetime, timedelta

from sqlalchemy import and_, func, or_, select

from steward.core.alerts import host_of_resource
from steward.core.formatting import human_duration
from steward.mcp._context import mcp_app
from steward.models.hosts import Host
from steward.models.metrics import PluginMetric
from steward.models.monitors import Monitor
from steward.models.norms import MetricAnomaly, MetricAnomalySpan

from ._shape import db_session, iso

logger = logging.getLogger(__name__)

PER_SOURCE = 40     # events any one source contributes to one window: the
                    # nearest the occurrence's start, the rest counted (#5861)
# Steward samples its own health once a minute (core/self_monitor.py); a
# longer break between two samples is a stretch it wasn't recording.
STEWARD_GAP_S = 90.0


def event(at: datetime | None, kind: str, what: str, *, until: datetime | None = None,
          subject: str | None = None, ids: dict | None = None) -> dict:
    """One thing that happened. `until` for something that lasted (a run, a
    reboot); `subject` names what it happened to when that isn't the host."""
    return {"at": iso(at), "until": iso(until), "kind": kind, "what": what,
            "subject": subject, "ids": ids or {}}


@dataclass(frozen=True)
class Place:
    """Where a metric's resource lives: its host, when it has one."""
    source: str
    resource: str
    host_id: str | None = None
    host_name: str | None = None


async def place_of(db, source: str, resource: str) -> Place:
    """The host behind a resource: an agent-fed series names it ("nas:/",
    "nas/web"), a monitor may be pinned to one; a Traefik router has none."""
    name = host_of_resource(source, resource)
    host = None
    if name:
        host = (await db.execute(select(Host).where(Host.name == name).limit(1))).scalar_one_or_none()
    else:
        monitor = (await db.execute(select(Monitor).where(
            Monitor.type == source, Monitor.name == resource).limit(1))).scalar_one_or_none()
        if monitor is not None and monitor.host_id:
            host = await db.get(Host, monitor.host_id)
    return Place(source, resource, host.id if host else None, host.name if host else None)


def _overlaps(start, end):
    """A row lasting from `start` to `end` (None: still going) overlaps the window."""
    def clause(since, until):
        return and_(start <= until, or_(end.is_(None), end >= since))
    return clause


async def timeline_events(db, place: Place, since: datetime, until: datetime) -> list[dict]:
    """What the timeline holds about the host, or the resource itself, while
    the window was open: alert episodes, playbook runs, maintenance, reboots
    and lost contact, deploys, restarts and crash-loops, settings changes.
    Hidden types are included: a dry run or a single restart can be what
    explains a reading. Anomalies come from other_anomalies, by occurrence."""
    from steward.models.timeline import TimelineEntry
    about = [TimelineEntry.subject == place.resource]
    if place.host_name:
        about.append(TimelineEntry.hosts.contains([place.host_name]))
    rows = (await db.execute(
        select(TimelineEntry)
        .where(or_(*about), _overlaps(TimelineEntry.at, func.coalesce(TimelineEntry.until, TimelineEntry.at))
               (since, until), or_(TimelineEntry.ref_type.is_(None), TimelineEntry.ref_type != "metric_anomaly"))
        .order_by(TimelineEntry.at).limit(500))).scalars()
    return [event(e.at, e.type, e.summary, until=e.until, subject=e.subject,
                  ids={e.ref_type: e.ref_id} if e.ref_type else None) for e in rows]


def _same_host(source: str, resource: str, place: Place) -> bool:
    if (source, resource) == (place.source, place.resource):
        return True
    return bool(place.host_name) and host_of_resource(source, resource) == place.host_name


async def other_anomalies(db, place: Place, anomaly_id: str, since: datetime,
                          until: datetime) -> tuple[list[dict], int]:
    """Other anomalies going on in the window: those on the same host as
    events, and how many there were anywhere else (many at once points at
    something shared: the network, storage, Steward itself)."""
    rows = (await db.execute(
        select(MetricAnomalySpan, MetricAnomaly)
        .join(MetricAnomaly, MetricAnomaly.id == MetricAnomalySpan.anomaly_id)
        .where(MetricAnomaly.id != anomaly_id,
               _overlaps(MetricAnomalySpan.started_at, MetricAnomalySpan.ended_at)(since, until))
        .order_by(MetricAnomalySpan.started_at).limit(500))).all()
    here, elsewhere = [], 0
    for span, a in rows:
        if _same_host(a.source_module, a.resource_name, place):
            here.append(event(span.started_at, "anomaly",
                              f"{a.resource_name} {a.metric_name} anomaly (peak {round(span.peak_level, 3)}, "
                              f"usual {round(a.usual_hold, 3)}; {a.review})",
                              until=span.ended_at, subject=a.resource_name, ids={"anomaly_id": a.id}))
        else:
            elsewhere += 1
    return here, elsewhere


def gaps_in(times: list[datetime], since: datetime, until: datetime,
            restarts: list[tuple[datetime, float]] = ()) -> list[dict]:
    """Each restart (its first sample's time and how long since the last
    one) and each other break longer than STEWARD_GAP_S between consecutive
    samples that overlaps the window, as a "steward" event lasting the break.
    A restart is shown however short: a quick redeploy never breaks the
    minute, and still restarts every count Steward keeps."""
    out = []
    for at, secs in restarts:
        if at >= since and at - timedelta(seconds=secs) <= until:
            out.append(event(at - timedelta(seconds=secs), "steward",
                             f"Steward restarted, not recording for {human_duration(secs)}: "
                             "readings either side of it can be wrong", until=at))
    restarted = {at for at, _ in restarts}
    for before, after in zip(times, times[1:]):
        secs = (after - before).total_seconds()
        if secs > STEWARD_GAP_S and after >= since and before <= until and after not in restarted:
            out.append(event(before, "steward",
                             f"Steward wasn't recording for {human_duration(secs)} (a restart, a deploy or a "
                             "stall): readings either side of a gap can be wrong", until=after))
    return out


async def steward_gaps(db, since: datetime, until: datetime) -> list[dict]:
    """When Steward itself wasn't recording, read from its own once-a-minute
    health sample. Shown beside every anomaly, placed or not: a restart
    distorts whatever Steward measures, a Traefik counter as much as a host's
    silence (#5862). Raw samples only, so a window older than their
    retention shows none."""
    rows = (await db.execute(
        select(PluginMetric.metric_name, PluginMetric.recorded_at, PluginMetric.value).where(
            PluginMetric.source_module == "steward", PluginMetric.resource_name == "process",
            PluginMetric.metric_name.in_(("db_roundtrip_ms", "restart_gap_secs")),
            PluginMetric.recorded_at >= since - timedelta(hours=1), PluginMetric.recorded_at <= until)
        .order_by(PluginMetric.recorded_at))).all()
    return gaps_in([at for name, at, _ in rows if name == "db_roundtrip_ms"], since, until,
                   [(at, value) for name, at, value in rows if name == "restart_gap_secs"])


async def plugin_events(place: Place, since: datetime, until: datetime) -> list[list[dict]]:
    """What enabled plugins know happened on the host (`mcp_events`), one
    list per plugin, each read in its own session so one that fails can't
    spoil the others' reads."""
    server = getattr(mcp_app(), "mcp_server", None)
    if server is None or not place.host_id:
        return []
    out = []
    for plugin, source in sorted((getattr(server, "event_sources", None) or {}).items()):
        if not server.plugin_enabled(plugin):
            continue
        try:
            async with db_session() as db:
                out.append(await source(db, place.host_id, place.host_name, since, until))
        except Exception:
            logger.exception("MCP events source of plugin %r failed", plugin)
    return out


async def resource_events(place: Place, since: datetime, until: datetime) -> list[list[dict]]:
    """What enabled plugins know happened to the resource itself
    (`mcp_resource_events`), for one with no host to read events by: one
    list per plugin."""
    server = getattr(mcp_app(), "mcp_server", None)
    if server is None:
        return []
    out = []
    for plugin, source in sorted((getattr(server, "resource_event_sources", None) or {}).items()):
        if not server.plugin_enabled(plugin):
            continue
        try:
            async with db_session() as db:
                out.append(await source(db, place.source, place.resource, since, until))
        except Exception:
            logger.exception("MCP resource events source of plugin %r failed", plugin)
    return out


def _at(e: dict) -> datetime | None:
    return datetime.fromisoformat(e["at"].replace("Z", "+00:00")) if e.get("at") else None


def nearest(events: list[dict], focus: datetime, keep: int = PER_SOURCE) -> tuple[list[dict], int]:
    """The `keep` events nearest `focus` (an occurrence's start), and how
    many were left out: a busy host's window is cut where it matters least."""
    if len(events) <= keep:
        return events, 0
    by_distance = sorted(events, key=lambda e: abs((_at(e) - focus).total_seconds()) if _at(e) else float("inf"))
    return by_distance[:keep], len(events) - keep


async def around(db, place: Place, anomaly_id: str, since: datetime, until: datetime,
                 focus: datetime) -> tuple[list[dict], int, int]:
    """Everything above for one window, oldest first; the count of anomalies
    elsewhere; and how many events were left out, each source keeping its
    PER_SOURCE nearest `focus`."""
    anomalies, elsewhere = await other_anomalies(db, place, anomaly_id, since, until)
    groups = [await timeline_events(db, place, since, until), anomalies,
              *await plugin_events(place, since, until),
              *await resource_events(place, since, until), await steward_gaps(db, since, until)]
    found, left_out = [], 0
    for group in groups:
        kept, cut = nearest(group, focus)
        found += kept
        left_out += cut
    return sorted(found, key=lambda e: e["at"] or ""), elsewhere, left_out