fix(mcp): get_anomaly shows Steward's own restarts beside every anomaly, and a router's containers (#5862)

All 16 Traefik 4xx anomalies on 2026-10-15 came back with nothing
around them, though 15 were Steward's own restarts. A router has no
host, and every events source was read by host.

- Core: each break of more than 90 s in Steward's once-a-minute health
  sample is a "steward" event beside every anomaly, placed or not.
- Plugins: an optional mcp_resource_events(db, source, resource, since,
  until) for a resource with no host. Traefik's gives the container
  events of the swarm service behind a router, on any host.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-15 15:58:28 -04:00
co-authored by Claude Opus 5.5
parent a24b9da69d
commit 91bb199d14
8 changed files with 139 additions and 10 deletions
+62 -6
View File
@@ -1,26 +1,32 @@
"""What else was going on: events on a host, or one resource, in a window
(#5837), so an anomaly comes with its circumstances.
Core gathers alert transitions, Ansible runs and other metric anomalies. A
plugin adds its own through `mcp_events(db, host_id, host_name, since, until)`
in its __init__.py (the host agent its maintenance jobs and downtime, Docker
its container events); each returns a list of `event(...)`.
Core gathers alert transitions, Ansible runs, other metric anomalies and
Steward's own gaps in recording. A plugin adds its own through
`mcp_events(db, host_id, host_name, since, until)` in its __init__.py (the
host agent its maintenance jobs and downtime, Docker its container events),
and for a resource with no host through
`mcp_resource_events(db, source, resource, since, until)` (Traefik: the
containers of the swarm service behind a router, #5862); each returns a list
of `event(...)`.
"""
from __future__ import annotations
import logging
from dataclasses import dataclass
from datetime import datetime
from datetime import datetime, timedelta
from sqlalchemy import and_, or_, select
from steward.alerts.service import watches_host
from steward.core.alerts import host_of_resource
from steward.core.formatting import human_duration
from steward.mcp._context import mcp_app
from steward.models.alerts import AlertEvent, AlertRule
from steward.models.ansible import AnsibleRun
from steward.models.ansible_inventory import AnsibleTarget
from steward.models.hosts import Host
from steward.models.metrics import PluginMetric
from steward.models.monitors import Monitor
from steward.models.norms import MetricAnomaly, MetricAnomalySpan
@@ -29,6 +35,9 @@ from ._shape import db_session, iso
logger = logging.getLogger(__name__)
PER_SOURCE = 40 # events any one source contributes to one window
# Steward samples its own health once a minute (core/self_monitor.py); a
# longer break between two samples is a stretch it wasn't recording.
STEWARD_GAP_S = 90.0
def event(at: datetime | None, kind: str, what: str, *, until: datetime | None = None,
@@ -142,6 +151,34 @@ async def other_anomalies(db, place: Place, anomaly_id: str, since: datetime,
return here[:PER_SOURCE], elsewhere
def gaps_in(times: list[datetime], since: datetime, until: datetime) -> list[dict]:
"""Each break longer than STEWARD_GAP_S between consecutive samples that
overlaps the window, as a "steward" event lasting the break."""
out = []
for before, after in zip(times, times[1:]):
secs = (after - before).total_seconds()
if secs > STEWARD_GAP_S and after >= since and before <= until:
out.append(event(before, "steward",
f"Steward wasn't recording for {human_duration(secs)} (a restart, a deploy or a "
"stall): readings either side of a gap can be wrong", until=after))
return out
async def steward_gaps(db, since: datetime, until: datetime) -> list[dict]:
"""When Steward itself wasn't recording, read from its own once-a-minute
health sample. Shown beside every anomaly, placed or not: a restart
distorts whatever Steward measures, a Traefik counter as much as a host's
silence (#5862). Raw samples only, so a window older than their
retention shows none."""
rows = (await db.execute(
select(PluginMetric.recorded_at).where(
PluginMetric.source_module == "steward", PluginMetric.resource_name == "process",
PluginMetric.metric_name == "db_roundtrip_ms",
PluginMetric.recorded_at >= since - timedelta(hours=1), PluginMetric.recorded_at <= until)
.order_by(PluginMetric.recorded_at))).scalars()
return gaps_in(list(rows), since, until)
async def plugin_events(place: Place, since: datetime, until: datetime) -> list[dict]:
"""What enabled plugins know happened on the host (`mcp_events`), each
in its own session so one that fails can't spoil the others' reads."""
@@ -160,11 +197,30 @@ async def plugin_events(place: Place, since: datetime, until: datetime) -> list[
return out
async def resource_events(place: Place, since: datetime, until: datetime) -> list[dict]:
"""What enabled plugins know happened to the resource itself
(`mcp_resource_events`), for one with no host to read events by."""
server = getattr(mcp_app(), "mcp_server", None)
if server is None:
return []
out = []
for plugin, source in sorted((getattr(server, "resource_event_sources", None) or {}).items()):
if not server.plugin_enabled(plugin):
continue
try:
async with db_session() as db:
out.extend((await source(db, place.source, place.resource, since, until))[:PER_SOURCE])
except Exception:
logger.exception("MCP resource events source of plugin %r failed", plugin)
return out
async def around(db, place: Place, anomaly_id: str, since: datetime,
until: datetime) -> tuple[list[dict], int]:
"""Everything above for one window, oldest first, and the count of
anomalies elsewhere."""
anomalies, elsewhere = await other_anomalies(db, place, anomaly_id, since, until)
found = (await alert_events(db, place, since, until) + await ansible_runs(db, place, since, until)
+ anomalies + await plugin_events(place, since, until))
+ anomalies + await plugin_events(place, since, until)
+ await resource_events(place, since, until) + await steward_gaps(db, since, until))
return sorted(found, key=lambda e: e["at"] or ""), elsewhere
+5 -2
View File
@@ -158,9 +158,12 @@ async def get_anomaly(anomaly_id: str, occurrences: int = 5) -> dict:
- `occurrences[]`: `started_at`, `ended_at`, `held_s`, `peak_level`,
and `usual` for that hour as Steward has learned it now (`usual_hold`,
`usual_peak`, `flag_above`), plus `around`: alert transitions, Ansible
runs, other anomalies on the host, and what plugins saw (container
runs, other anomalies on the host, what plugins saw (container
starts, deaths and OOM kills; maintenance jobs; reboots and agent
stops), from 30 minutes before to 30 minutes after; and
stops; for a Traefik router, its swarm service's containers on any
host), and kind "steward": a stretch Steward itself wasn't recording
(a restart or deploy), around which any reading can be wrong. From 30
minutes before to 30 minutes after; and
`anomalies_elsewhere`, how many anomalies on other hosts overlapped it.
Many at once points at something shared (network, storage, Steward).
- `series`: the newest shown occurrence with time either side (as long