fix(mcp): get_anomaly shows Steward's own restarts beside every anomaly, and a router's containers (#5862)

All 16 Traefik 4xx anomalies on 2026-10-15 came back with nothing
around them, though 15 were Steward's own restarts. A router has no
host, and every events source was read by host.

- Core: each break of more than 90 s in Steward's once-a-minute health
  sample is a "steward" event beside every anomaly, placed or not.
- Plugins: an optional mcp_resource_events(db, source, resource, since,
  until) for a resource with no host. Traefik's gives the container
  events of the swarm service behind a router, on any host.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-15 15:58:28 -04:00
co-authored by Claude Opus 5.5
parent a24b9da69d
commit 91bb199d14
8 changed files with 139 additions and 10 deletions
+3 -1
View File
@@ -159,6 +159,8 @@ exposes `register_mcp_tools(register)`, optionally `mcp_overview(db)` to
contribute findings to `get_overview`, and optionally
`mcp_events(db, host_id, host_name, since, until)`: what it knows happened on a
host in a window, as a list of `steward.mcp.tools._events.event(...)`, which
`get_anomaly` shows beside an anomaly. See
`get_anomaly` shows beside an anomaly; for a resource with no host, optionally
`mcp_resource_events(db, source, resource, since, until)`. Core adds the
stretches Steward itself wasn't recording to every anomaly. See
[Writing a plugin](../plugins/writing-a-plugin.md). A tool that changes
anything calls `audit_mcp(...)` from `steward/mcp/tools/_audit.py`.
+4
View File
@@ -406,6 +406,10 @@ async def mcp_events(db, host_id, host_name, since, until) -> list[dict]:
- Return what happened between `since` and `until`, oldest first: something that lasted gives `until`. Keep it to tens of rows; the caller keeps the first 40.
- It is called only while the plugin is enabled, with its own session. If it raises, the anomaly is still returned without the plugin's events. The host agent (maintenance jobs, reboots) and Docker (container starts, deaths, OOM kills) are the examples to read.
A resource with no host (a Traefik router, say) has no host to read events by. A plugin that knows what happened to one of its own resources exports `mcp_resource_events(db, source, resource, since, until)` instead, same return shape, and returns `[]` for a source that isn't its own. Traefik's is the example: it gives a router's swarm service's container events, on whichever hosts they ran.
Core adds, beside every anomaly, the stretches Steward itself wasn't recording (kind `"steward"`: a restart, a deploy, a stall), read from its own once-a-minute health sample. A plugin doesn't need to report those.
---
## Shipping a Plugin
+7
View File
@@ -72,3 +72,10 @@ def register_mcp_tools(register) -> None:
async def mcp_overview(db) -> list[dict]:
from .mcp_tools import mcp_overview as findings
return await findings(db)
async def mcp_resource_events(db, source: str, resource: str, since, until) -> list[dict]:
if source != "traefik":
return []
from .mcp_tools import service_events
return await service_events(db, resource, since, until)
+24
View File
@@ -22,6 +22,7 @@ from steward.mcp.tools._shape import (
from steward.mcp.tools.overview import finding
from steward.models.users import UserRole
from . import meaning
from . import routes as page
from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat
from .scheduler import error_page_services, router_domains
@@ -316,6 +317,29 @@ async def mcp_overview(db) -> list[dict]:
return router_findings(current) + cert_findings(served, now, thresholds)
async def service_events(db, router: str, since, until) -> list[dict]:
"""What happened to the containers of the swarm service behind a router
(starts with their image, deaths, OOM kills, health changes), on every
host: a router anomaly has no host of its own to read events by (#5862).
[] without the Docker plugin, or when no single service matches."""
from steward.core.plugin_manager import loaded_plugin_submodule
from steward.mcp.tools._events import PER_SOURCE, event
from steward.models.hosts import Host
models = loaded_plugin_submodule("docker", "models")
if models is None:
return []
service = meaning.swarm_service(router, await page.swarm_service_names(db))
if service is None:
return []
e = models.DockerEvent
rows = (await db.execute(
select(e, Host.name).join(Host, Host.id == e.host_id)
.where(e.container_name.startswith(f"{service}.", autoescape=True), e.at >= since, e.at <= until)
.order_by(e.at).limit(PER_SOURCE))).all()
return [event(ev.at, "container", f"{ev.container_name} {ev.event}" + (f" ({ev.detail})" if ev.detail else ""),
subject=f"{host}/{ev.container_name}") for ev, host in rows]
def register_tools(register) -> None:
for fn in (list_routers, get_router, list_certificates, get_traffic):
register(fn, role=UserRole.viewer, read_only=True)
+10 -1
View File
@@ -82,6 +82,9 @@ class StewardMCPServer(MCPServer):
# plugin name -> its `mcp_events(db, host_id, host_name, since, until)`:
# what it knows happened on a host, for get_anomaly.
self.event_sources: dict[str, object] = {}
# plugin name -> its `mcp_resource_events(db, source, resource, since,
# until)`: what it knows happened to a resource with no host.
self.resource_event_sources: dict[str, object] = {}
# ── registration ─────────────────────────────────────────────────────────
def register_tool(self, fn, *, role: UserRole, read_only: bool, plugin: str | None = None) -> None:
@@ -108,7 +111,9 @@ class StewardMCPServer(MCPServer):
"""Add a plugin's tools via its optional `register_mcp_tools(register)`,
its optional `mcp_overview(db)` as a get_overview findings source, and
its optional `mcp_events(db, host_id, host_name, since, until)` as a
source of what happened on a host (get_anomaly).
source of what happened on a host (get_anomaly), and its optional
`mcp_resource_events(db, source, resource, since, until)` for what
happened to a resource with no host.
Idempotent per plugin (hot-reload drops its previous tools first) and
tolerant: a raising hook is logged and leaves the plugin with no tools,
@@ -123,6 +128,10 @@ class StewardMCPServer(MCPServer):
events = getattr(module, "mcp_events", None)
if callable(events):
self.event_sources[name] = events
self.resource_event_sources.pop(name, None)
resource_events = getattr(module, "mcp_resource_events", None)
if callable(resource_events):
self.resource_event_sources[name] = resource_events
hook = getattr(module, "register_mcp_tools", None)
if hook is None:
return
+62 -6
View File
@@ -1,26 +1,32 @@
"""What else was going on: events on a host, or one resource, in a window
(#5837), so an anomaly comes with its circumstances.
Core gathers alert transitions, Ansible runs and other metric anomalies. A
plugin adds its own through `mcp_events(db, host_id, host_name, since, until)`
in its __init__.py (the host agent its maintenance jobs and downtime, Docker
its container events); each returns a list of `event(...)`.
Core gathers alert transitions, Ansible runs, other metric anomalies and
Steward's own gaps in recording. A plugin adds its own through
`mcp_events(db, host_id, host_name, since, until)` in its __init__.py (the
host agent its maintenance jobs and downtime, Docker its container events),
and for a resource with no host through
`mcp_resource_events(db, source, resource, since, until)` (Traefik: the
containers of the swarm service behind a router, #5862); each returns a list
of `event(...)`.
"""
from __future__ import annotations
import logging
from dataclasses import dataclass
from datetime import datetime
from datetime import datetime, timedelta
from sqlalchemy import and_, or_, select
from steward.alerts.service import watches_host
from steward.core.alerts import host_of_resource
from steward.core.formatting import human_duration
from steward.mcp._context import mcp_app
from steward.models.alerts import AlertEvent, AlertRule
from steward.models.ansible import AnsibleRun
from steward.models.ansible_inventory import AnsibleTarget
from steward.models.hosts import Host
from steward.models.metrics import PluginMetric
from steward.models.monitors import Monitor
from steward.models.norms import MetricAnomaly, MetricAnomalySpan
@@ -29,6 +35,9 @@ from ._shape import db_session, iso
logger = logging.getLogger(__name__)
PER_SOURCE = 40 # events any one source contributes to one window
# Steward samples its own health once a minute (core/self_monitor.py); a
# longer break between two samples is a stretch it wasn't recording.
STEWARD_GAP_S = 90.0
def event(at: datetime | None, kind: str, what: str, *, until: datetime | None = None,
@@ -142,6 +151,34 @@ async def other_anomalies(db, place: Place, anomaly_id: str, since: datetime,
return here[:PER_SOURCE], elsewhere
def gaps_in(times: list[datetime], since: datetime, until: datetime) -> list[dict]:
"""Each break longer than STEWARD_GAP_S between consecutive samples that
overlaps the window, as a "steward" event lasting the break."""
out = []
for before, after in zip(times, times[1:]):
secs = (after - before).total_seconds()
if secs > STEWARD_GAP_S and after >= since and before <= until:
out.append(event(before, "steward",
f"Steward wasn't recording for {human_duration(secs)} (a restart, a deploy or a "
"stall): readings either side of a gap can be wrong", until=after))
return out
async def steward_gaps(db, since: datetime, until: datetime) -> list[dict]:
"""When Steward itself wasn't recording, read from its own once-a-minute
health sample. Shown beside every anomaly, placed or not: a restart
distorts whatever Steward measures, a Traefik counter as much as a host's
silence (#5862). Raw samples only, so a window older than their
retention shows none."""
rows = (await db.execute(
select(PluginMetric.recorded_at).where(
PluginMetric.source_module == "steward", PluginMetric.resource_name == "process",
PluginMetric.metric_name == "db_roundtrip_ms",
PluginMetric.recorded_at >= since - timedelta(hours=1), PluginMetric.recorded_at <= until)
.order_by(PluginMetric.recorded_at))).scalars()
return gaps_in(list(rows), since, until)
async def plugin_events(place: Place, since: datetime, until: datetime) -> list[dict]:
"""What enabled plugins know happened on the host (`mcp_events`), each
in its own session so one that fails can't spoil the others' reads."""
@@ -160,11 +197,30 @@ async def plugin_events(place: Place, since: datetime, until: datetime) -> list[
return out
async def resource_events(place: Place, since: datetime, until: datetime) -> list[dict]:
"""What enabled plugins know happened to the resource itself
(`mcp_resource_events`), for one with no host to read events by."""
server = getattr(mcp_app(), "mcp_server", None)
if server is None:
return []
out = []
for plugin, source in sorted((getattr(server, "resource_event_sources", None) or {}).items()):
if not server.plugin_enabled(plugin):
continue
try:
async with db_session() as db:
out.extend((await source(db, place.source, place.resource, since, until))[:PER_SOURCE])
except Exception:
logger.exception("MCP resource events source of plugin %r failed", plugin)
return out
async def around(db, place: Place, anomaly_id: str, since: datetime,
until: datetime) -> tuple[list[dict], int]:
"""Everything above for one window, oldest first, and the count of
anomalies elsewhere."""
anomalies, elsewhere = await other_anomalies(db, place, anomaly_id, since, until)
found = (await alert_events(db, place, since, until) + await ansible_runs(db, place, since, until)
+ anomalies + await plugin_events(place, since, until))
+ anomalies + await plugin_events(place, since, until)
+ await resource_events(place, since, until) + await steward_gaps(db, since, until))
return sorted(found, key=lambda e: e["at"] or ""), elsewhere
+5 -2
View File
@@ -158,9 +158,12 @@ async def get_anomaly(anomaly_id: str, occurrences: int = 5) -> dict:
- `occurrences[]`: `started_at`, `ended_at`, `held_s`, `peak_level`,
and `usual` for that hour as Steward has learned it now (`usual_hold`,
`usual_peak`, `flag_above`), plus `around`: alert transitions, Ansible
runs, other anomalies on the host, and what plugins saw (container
runs, other anomalies on the host, what plugins saw (container
starts, deaths and OOM kills; maintenance jobs; reboots and agent
stops), from 30 minutes before to 30 minutes after; and
stops; for a Traefik router, its swarm service's containers on any
host), and kind "steward": a stretch Steward itself wasn't recording
(a restart or deploy), around which any reading can be wrong. From 30
minutes before to 30 minutes after; and
`anomalies_elsewhere`, how many anomalies on other hosts overlapped it.
Many at once points at something shared (network, storage, Steward).
- `series`: the newest shown occurrence with time either side (as long
+24
View File
@@ -103,3 +103,27 @@ def test_a_plugin_adds_events_through_its_hook(app):
assert "events_test" not in mcp.event_sources
finally:
mcp.event_sources.pop("events_test", None)
def test_a_plugin_adds_a_hostless_resources_events_through_its_hook(app):
async def events(db, source, resource, since, until):
return []
mcp = app.mcp_server
mcp.register_plugin("events_test", NS(mcp_resource_events=events))
try:
assert mcp.resource_event_sources["events_test"] is events
mcp.register_plugin("events_test", NS())
assert "events_test" not in mcp.resource_event_sources
finally:
mcp.resource_event_sources.pop("events_test", None)
def test_steward_not_recording_is_an_event_beside_every_anomaly():
"""A Steward restart between two of its once-a-minute samples (#5862)."""
ticks = [NOW + timedelta(minutes=m) for m in (0, 1, 2)] + [NOW + timedelta(minutes=4, seconds=10)]
[gap] = _events.gaps_in(ticks, NOW, NOW + timedelta(hours=1))
assert gap["kind"] == "steward" and gap["at"] == "2026-10-15T12:02:00Z"
assert gap["until"] == "2026-10-15T12:04:10Z" and "wasn't recording" in gap["what"]
assert _events.gaps_in(ticks, NOW + timedelta(minutes=5), NOW + timedelta(hours=1)) == []
assert _events.gaps_in([NOW + timedelta(minutes=m) for m in range(5)], NOW, NOW + timedelta(hours=1)) == []