fix(mcp): get_anomaly shows Steward's own restarts beside every anomaly, and a router's containers (#5862)
All 16 Traefik 4xx anomalies on 2026-10-15 came back with nothing around them, though 15 were Steward's own restarts. A router has no host, and every events source was read by host. - Core: each break of more than 90 s in Steward's once-a-minute health sample is a "steward" event beside every anomaly, placed or not. - Plugins: an optional mcp_resource_events(db, source, resource, since, until) for a resource with no host. Traefik's gives the container events of the swarm service behind a router, on any host. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+10
-1
@@ -82,6 +82,9 @@ class StewardMCPServer(MCPServer):
|
||||
# plugin name -> its `mcp_events(db, host_id, host_name, since, until)`:
|
||||
# what it knows happened on a host, for get_anomaly.
|
||||
self.event_sources: dict[str, object] = {}
|
||||
# plugin name -> its `mcp_resource_events(db, source, resource, since,
|
||||
# until)`: what it knows happened to a resource with no host.
|
||||
self.resource_event_sources: dict[str, object] = {}
|
||||
|
||||
# ── registration ─────────────────────────────────────────────────────────
|
||||
def register_tool(self, fn, *, role: UserRole, read_only: bool, plugin: str | None = None) -> None:
|
||||
@@ -108,7 +111,9 @@ class StewardMCPServer(MCPServer):
|
||||
"""Add a plugin's tools via its optional `register_mcp_tools(register)`,
|
||||
its optional `mcp_overview(db)` as a get_overview findings source, and
|
||||
its optional `mcp_events(db, host_id, host_name, since, until)` as a
|
||||
source of what happened on a host (get_anomaly).
|
||||
source of what happened on a host (get_anomaly), and its optional
|
||||
`mcp_resource_events(db, source, resource, since, until)` for what
|
||||
happened to a resource with no host.
|
||||
|
||||
Idempotent per plugin (hot-reload drops its previous tools first) and
|
||||
tolerant: a raising hook is logged and leaves the plugin with no tools,
|
||||
@@ -123,6 +128,10 @@ class StewardMCPServer(MCPServer):
|
||||
events = getattr(module, "mcp_events", None)
|
||||
if callable(events):
|
||||
self.event_sources[name] = events
|
||||
self.resource_event_sources.pop(name, None)
|
||||
resource_events = getattr(module, "mcp_resource_events", None)
|
||||
if callable(resource_events):
|
||||
self.resource_event_sources[name] = resource_events
|
||||
hook = getattr(module, "register_mcp_tools", None)
|
||||
if hook is None:
|
||||
return
|
||||
|
||||
@@ -1,26 +1,32 @@
|
||||
"""What else was going on: events on a host, or one resource, in a window
|
||||
(#5837), so an anomaly comes with its circumstances.
|
||||
|
||||
Core gathers alert transitions, Ansible runs and other metric anomalies. A
|
||||
plugin adds its own through `mcp_events(db, host_id, host_name, since, until)`
|
||||
in its __init__.py (the host agent its maintenance jobs and downtime, Docker
|
||||
its container events); each returns a list of `event(...)`.
|
||||
Core gathers alert transitions, Ansible runs, other metric anomalies and
|
||||
Steward's own gaps in recording. A plugin adds its own through
|
||||
`mcp_events(db, host_id, host_name, since, until)` in its __init__.py (the
|
||||
host agent its maintenance jobs and downtime, Docker its container events),
|
||||
and for a resource with no host through
|
||||
`mcp_resource_events(db, source, resource, since, until)` (Traefik: the
|
||||
containers of the swarm service behind a router, #5862); each returns a list
|
||||
of `event(...)`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from sqlalchemy import and_, or_, select
|
||||
|
||||
from steward.alerts.service import watches_host
|
||||
from steward.core.alerts import host_of_resource
|
||||
from steward.core.formatting import human_duration
|
||||
from steward.mcp._context import mcp_app
|
||||
from steward.models.alerts import AlertEvent, AlertRule
|
||||
from steward.models.ansible import AnsibleRun
|
||||
from steward.models.ansible_inventory import AnsibleTarget
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.metrics import PluginMetric
|
||||
from steward.models.monitors import Monitor
|
||||
from steward.models.norms import MetricAnomaly, MetricAnomalySpan
|
||||
|
||||
@@ -29,6 +35,9 @@ from ._shape import db_session, iso
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PER_SOURCE = 40 # events any one source contributes to one window
|
||||
# Steward samples its own health once a minute (core/self_monitor.py); a
|
||||
# longer break between two samples is a stretch it wasn't recording.
|
||||
STEWARD_GAP_S = 90.0
|
||||
|
||||
|
||||
def event(at: datetime | None, kind: str, what: str, *, until: datetime | None = None,
|
||||
@@ -142,6 +151,34 @@ async def other_anomalies(db, place: Place, anomaly_id: str, since: datetime,
|
||||
return here[:PER_SOURCE], elsewhere
|
||||
|
||||
|
||||
def gaps_in(times: list[datetime], since: datetime, until: datetime) -> list[dict]:
|
||||
"""Each break longer than STEWARD_GAP_S between consecutive samples that
|
||||
overlaps the window, as a "steward" event lasting the break."""
|
||||
out = []
|
||||
for before, after in zip(times, times[1:]):
|
||||
secs = (after - before).total_seconds()
|
||||
if secs > STEWARD_GAP_S and after >= since and before <= until:
|
||||
out.append(event(before, "steward",
|
||||
f"Steward wasn't recording for {human_duration(secs)} (a restart, a deploy or a "
|
||||
"stall): readings either side of a gap can be wrong", until=after))
|
||||
return out
|
||||
|
||||
|
||||
async def steward_gaps(db, since: datetime, until: datetime) -> list[dict]:
|
||||
"""When Steward itself wasn't recording, read from its own once-a-minute
|
||||
health sample. Shown beside every anomaly, placed or not: a restart
|
||||
distorts whatever Steward measures, a Traefik counter as much as a host's
|
||||
silence (#5862). Raw samples only, so a window older than their
|
||||
retention shows none."""
|
||||
rows = (await db.execute(
|
||||
select(PluginMetric.recorded_at).where(
|
||||
PluginMetric.source_module == "steward", PluginMetric.resource_name == "process",
|
||||
PluginMetric.metric_name == "db_roundtrip_ms",
|
||||
PluginMetric.recorded_at >= since - timedelta(hours=1), PluginMetric.recorded_at <= until)
|
||||
.order_by(PluginMetric.recorded_at))).scalars()
|
||||
return gaps_in(list(rows), since, until)
|
||||
|
||||
|
||||
async def plugin_events(place: Place, since: datetime, until: datetime) -> list[dict]:
|
||||
"""What enabled plugins know happened on the host (`mcp_events`), each
|
||||
in its own session so one that fails can't spoil the others' reads."""
|
||||
@@ -160,11 +197,30 @@ async def plugin_events(place: Place, since: datetime, until: datetime) -> list[
|
||||
return out
|
||||
|
||||
|
||||
async def resource_events(place: Place, since: datetime, until: datetime) -> list[dict]:
|
||||
"""What enabled plugins know happened to the resource itself
|
||||
(`mcp_resource_events`), for one with no host to read events by."""
|
||||
server = getattr(mcp_app(), "mcp_server", None)
|
||||
if server is None:
|
||||
return []
|
||||
out = []
|
||||
for plugin, source in sorted((getattr(server, "resource_event_sources", None) or {}).items()):
|
||||
if not server.plugin_enabled(plugin):
|
||||
continue
|
||||
try:
|
||||
async with db_session() as db:
|
||||
out.extend((await source(db, place.source, place.resource, since, until))[:PER_SOURCE])
|
||||
except Exception:
|
||||
logger.exception("MCP resource events source of plugin %r failed", plugin)
|
||||
return out
|
||||
|
||||
|
||||
async def around(db, place: Place, anomaly_id: str, since: datetime,
|
||||
until: datetime) -> tuple[list[dict], int]:
|
||||
"""Everything above for one window, oldest first, and the count of
|
||||
anomalies elsewhere."""
|
||||
anomalies, elsewhere = await other_anomalies(db, place, anomaly_id, since, until)
|
||||
found = (await alert_events(db, place, since, until) + await ansible_runs(db, place, since, until)
|
||||
+ anomalies + await plugin_events(place, since, until))
|
||||
+ anomalies + await plugin_events(place, since, until)
|
||||
+ await resource_events(place, since, until) + await steward_gaps(db, since, until))
|
||||
return sorted(found, key=lambda e: e["at"] or ""), elsewhere
|
||||
|
||||
@@ -158,9 +158,12 @@ async def get_anomaly(anomaly_id: str, occurrences: int = 5) -> dict:
|
||||
- `occurrences[]`: `started_at`, `ended_at`, `held_s`, `peak_level`,
|
||||
and `usual` for that hour as Steward has learned it now (`usual_hold`,
|
||||
`usual_peak`, `flag_above`), plus `around`: alert transitions, Ansible
|
||||
runs, other anomalies on the host, and what plugins saw (container
|
||||
runs, other anomalies on the host, what plugins saw (container
|
||||
starts, deaths and OOM kills; maintenance jobs; reboots and agent
|
||||
stops), from 30 minutes before to 30 minutes after; and
|
||||
stops; for a Traefik router, its swarm service's containers on any
|
||||
host), and kind "steward": a stretch Steward itself wasn't recording
|
||||
(a restart or deploy), around which any reading can be wrong. From 30
|
||||
minutes before to 30 minutes after; and
|
||||
`anomalies_elsewhere`, how many anomalies on other hosts overlapped it.
|
||||
Many at once points at something shared (network, storage, Steward).
|
||||
- `series`: the newest shown occurrence with time either side (as long
|
||||
|
||||
Reference in New Issue
Block a user