"""What else was going on: events on a host, or one resource, in a window (#5837), so an anomaly comes with its circumstances. Core gathers alert transitions, Ansible runs, other metric anomalies and Steward's own gaps in recording. A plugin adds its own through `mcp_events(db, host_id, host_name, since, until)` in its __init__.py (the host agent its maintenance jobs and downtime, Docker its container events), and for a resource with no host through `mcp_resource_events(db, source, resource, since, until)` (Traefik: the containers of the swarm service behind a router, #5862); each returns a list of `event(...)`. """ from __future__ import annotations import logging from dataclasses import dataclass from datetime import datetime, timedelta from sqlalchemy import and_, or_, select from steward.alerts.service import watches_host from steward.core.alerts import host_of_resource from steward.core.formatting import human_duration from steward.mcp._context import mcp_app from steward.models.alerts import AlertEvent, AlertRule from steward.models.ansible import AnsibleRun from steward.models.ansible_inventory import AnsibleTarget from steward.models.hosts import Host from steward.models.metrics import PluginMetric from steward.models.monitors import Monitor from steward.models.norms import MetricAnomaly, MetricAnomalySpan from ._shape import db_session, iso logger = logging.getLogger(__name__) PER_SOURCE = 40 # events any one source contributes to one window: the # nearest the occurrence's start, the rest counted (#5861) # Steward samples its own health once a minute (core/self_monitor.py); a # longer break between two samples is a stretch it wasn't recording. STEWARD_GAP_S = 90.0 def event(at: datetime | None, kind: str, what: str, *, until: datetime | None = None, subject: str | None = None, ids: dict | None = None) -> dict: """One thing that happened. `until` for something that lasted (a run, a reboot); `subject` names what it happened to when that isn't the host.""" return {"at": iso(at), "until": iso(until), "kind": kind, "what": what, "subject": subject, "ids": ids or {}} @dataclass(frozen=True) class Place: """Where a metric's resource lives: its host, when it has one.""" source: str resource: str host_id: str | None = None host_name: str | None = None async def place_of(db, source: str, resource: str) -> Place: """The host behind a resource: an agent-fed series names it ("nas:/", "nas/web"), a monitor may be pinned to one; a Traefik router has none.""" name = host_of_resource(source, resource) host = None if name: host = (await db.execute(select(Host).where(Host.name == name).limit(1))).scalar_one_or_none() else: monitor = (await db.execute(select(Monitor).where( Monitor.type == source, Monitor.name == resource).limit(1))).scalar_one_or_none() if monitor is not None and monitor.host_id: host = await db.get(Host, monitor.host_id) return Place(source, resource, host.id if host else None, host.name if host else None) def _overlaps(start, end): """A row lasting from `start` to `end` (None: still going) overlaps the window.""" def clause(since, until): return and_(start <= until, or_(end.is_(None), end >= since)) return clause async def alert_events(db, place: Place, since: datetime, until: datetime) -> list[dict]: """Alert transitions on the resource itself, or anything its host carries: its agent series, its containers, its monitors.""" monitor_keys = set() if place.host_id: monitor_keys = {(t, n) for t, n in (await db.execute( select(Monitor.type, Monitor.name).where(Monitor.host_id == place.host_id))).all()} rows = (await db.execute( select(AlertEvent, AlertRule).join(AlertRule, AlertRule.id == AlertEvent.rule_id) .where(AlertEvent.transitioned_at >= since, AlertEvent.transitioned_at <= until) .order_by(AlertEvent.transitioned_at).limit(500))).all() out = [] for ev, rule in rows: mine = (rule.source_module, rule.resource_name) == (place.source, place.resource) if not mine and not (place.host_name and watches_host(rule, place.host_name, monitor_keys)): continue note = f" ({ev.note})" if ev.note else "" out.append(event(ev.transitioned_at, "alert", f"{rule.name}: {ev.from_state} → {ev.to_state}{note}", subject=rule.resource_name, ids={"rule_id": rule.id})) return out async def ansible_runs(db, place: Place, since: datetime, until: datetime) -> list[dict]: """Ansible runs that touched the host while the window was open (by the hosts in their recap: a run still going hasn't said yet).""" if not place.host_id: return [] names = {place.host_name} | set((await db.execute( select(AnsibleTarget.name).where(AnsibleTarget.host_id == place.host_id))).scalars()) runs = (await db.execute( select(AnsibleRun).where(_overlaps(AnsibleRun.started_at, AnsibleRun.finished_at)(since, until)) .order_by(AnsibleRun.started_at).limit(200))).scalars() out = [] for r in runs: if not names & set(((r.results or {}).get("hosts") or {}).keys()): continue check = " in check mode" if (r.params or {}).get("check") else "" out.append(event(r.started_at, "ansible_run", f"Ansible {r.playbook_path}{check}: {r.status.value}", until=r.finished_at, ids={"run_id": r.id})) return out def _same_host(source: str, resource: str, place: Place) -> bool: if (source, resource) == (place.source, place.resource): return True return bool(place.host_name) and host_of_resource(source, resource) == place.host_name async def other_anomalies(db, place: Place, anomaly_id: str, since: datetime, until: datetime) -> tuple[list[dict], int]: """Other anomalies going on in the window: those on the same host as events, and how many there were anywhere else (many at once points at something shared: the network, storage, Steward itself).""" rows = (await db.execute( select(MetricAnomalySpan, MetricAnomaly) .join(MetricAnomaly, MetricAnomaly.id == MetricAnomalySpan.anomaly_id) .where(MetricAnomaly.id != anomaly_id, _overlaps(MetricAnomalySpan.started_at, MetricAnomalySpan.ended_at)(since, until)) .order_by(MetricAnomalySpan.started_at).limit(500))).all() here, elsewhere = [], 0 for span, a in rows: if _same_host(a.source_module, a.resource_name, place): here.append(event(span.started_at, "anomaly", f"{a.resource_name} {a.metric_name} anomaly (peak {round(span.peak_level, 3)}, " f"usual {round(a.usual_hold, 3)}; {a.review})", until=span.ended_at, subject=a.resource_name, ids={"anomaly_id": a.id})) else: elsewhere += 1 return here, elsewhere def gaps_in(times: list[datetime], since: datetime, until: datetime, restarts: list[tuple[datetime, float]] = ()) -> list[dict]: """Each restart (its first sample's time and how long since the last one) and each other break longer than STEWARD_GAP_S between consecutive samples that overlaps the window, as a "steward" event lasting the break. A restart is shown however short: a quick redeploy never breaks the minute, and still restarts every count Steward keeps.""" out = [] for at, secs in restarts: if at >= since and at - timedelta(seconds=secs) <= until: out.append(event(at - timedelta(seconds=secs), "steward", f"Steward restarted, not recording for {human_duration(secs)}: " "readings either side of it can be wrong", until=at)) restarted = {at for at, _ in restarts} for before, after in zip(times, times[1:]): secs = (after - before).total_seconds() if secs > STEWARD_GAP_S and after >= since and before <= until and after not in restarted: out.append(event(before, "steward", f"Steward wasn't recording for {human_duration(secs)} (a restart, a deploy or a " "stall): readings either side of a gap can be wrong", until=after)) return out async def steward_gaps(db, since: datetime, until: datetime) -> list[dict]: """When Steward itself wasn't recording, read from its own once-a-minute health sample. Shown beside every anomaly, placed or not: a restart distorts whatever Steward measures, a Traefik counter as much as a host's silence (#5862). Raw samples only, so a window older than their retention shows none.""" rows = (await db.execute( select(PluginMetric.metric_name, PluginMetric.recorded_at, PluginMetric.value).where( PluginMetric.source_module == "steward", PluginMetric.resource_name == "process", PluginMetric.metric_name.in_(("db_roundtrip_ms", "restart_gap_secs")), PluginMetric.recorded_at >= since - timedelta(hours=1), PluginMetric.recorded_at <= until) .order_by(PluginMetric.recorded_at))).all() return gaps_in([at for name, at, _ in rows if name == "db_roundtrip_ms"], since, until, [(at, value) for name, at, value in rows if name == "restart_gap_secs"]) async def plugin_events(place: Place, since: datetime, until: datetime) -> list[list[dict]]: """What enabled plugins know happened on the host (`mcp_events`), one list per plugin, each read in its own session so one that fails can't spoil the others' reads.""" server = getattr(mcp_app(), "mcp_server", None) if server is None or not place.host_id: return [] out = [] for plugin, source in sorted((getattr(server, "event_sources", None) or {}).items()): if not server.plugin_enabled(plugin): continue try: async with db_session() as db: out.append(await source(db, place.host_id, place.host_name, since, until)) except Exception: logger.exception("MCP events source of plugin %r failed", plugin) return out async def resource_events(place: Place, since: datetime, until: datetime) -> list[list[dict]]: """What enabled plugins know happened to the resource itself (`mcp_resource_events`), for one with no host to read events by: one list per plugin.""" server = getattr(mcp_app(), "mcp_server", None) if server is None: return [] out = [] for plugin, source in sorted((getattr(server, "resource_event_sources", None) or {}).items()): if not server.plugin_enabled(plugin): continue try: async with db_session() as db: out.append(await source(db, place.source, place.resource, since, until)) except Exception: logger.exception("MCP resource events source of plugin %r failed", plugin) return out def _at(e: dict) -> datetime | None: return datetime.fromisoformat(e["at"].replace("Z", "+00:00")) if e.get("at") else None def nearest(events: list[dict], focus: datetime, keep: int = PER_SOURCE) -> tuple[list[dict], int]: """The `keep` events nearest `focus` (an occurrence's start), and how many were left out: a busy host's window is cut where it matters least.""" if len(events) <= keep: return events, 0 by_distance = sorted(events, key=lambda e: abs((_at(e) - focus).total_seconds()) if _at(e) else float("inf")) return by_distance[:keep], len(events) - keep async def around(db, place: Place, anomaly_id: str, since: datetime, until: datetime, focus: datetime) -> tuple[list[dict], int, int]: """Everything above for one window, oldest first; the count of anomalies elsewhere; and how many events were left out, each source keeping its PER_SOURCE nearest `focus`.""" anomalies, elsewhere = await other_anomalies(db, place, anomaly_id, since, until) groups = [await alert_events(db, place, since, until), await ansible_runs(db, place, since, until), anomalies, *await plugin_events(place, since, until), *await resource_events(place, since, until), await steward_gaps(db, since, until)] found, left_out = [], 0 for group in groups: kept, cut = nearest(group, focus) found += kept left_out += cut return sorted(found, key=lambda e: e["at"] or ""), elsewhere, left_out