diff --git a/docs/core/mcp.md b/docs/core/mcp.md index a376623..03ddd0a 100644 --- a/docs/core/mcp.md +++ b/docs/core/mcp.md @@ -159,6 +159,8 @@ exposes `register_mcp_tools(register)`, optionally `mcp_overview(db)` to contribute findings to `get_overview`, and optionally `mcp_events(db, host_id, host_name, since, until)`: what it knows happened on a host in a window, as a list of `steward.mcp.tools._events.event(...)`, which -`get_anomaly` shows beside an anomaly. See +`get_anomaly` shows beside an anomaly; for a resource with no host, optionally +`mcp_resource_events(db, source, resource, since, until)`. Core adds the +stretches Steward itself wasn't recording to every anomaly. See [Writing a plugin](../plugins/writing-a-plugin.md). A tool that changes anything calls `audit_mcp(...)` from `steward/mcp/tools/_audit.py`. diff --git a/docs/plugins/writing-a-plugin.md b/docs/plugins/writing-a-plugin.md index 0921500..1999302 100644 --- a/docs/plugins/writing-a-plugin.md +++ b/docs/plugins/writing-a-plugin.md @@ -406,6 +406,10 @@ async def mcp_events(db, host_id, host_name, since, until) -> list[dict]: - Return what happened between `since` and `until`, oldest first: something that lasted gives `until`. Keep it to tens of rows; the caller keeps the first 40. - It is called only while the plugin is enabled, with its own session. If it raises, the anomaly is still returned without the plugin's events. The host agent (maintenance jobs, reboots) and Docker (container starts, deaths, OOM kills) are the examples to read. +A resource with no host (a Traefik router, say) has no host to read events by. A plugin that knows what happened to one of its own resources exports `mcp_resource_events(db, source, resource, since, until)` instead, same return shape, and returns `[]` for a source that isn't its own. Traefik's is the example: it gives a router's swarm service's container events, on whichever hosts they ran. + +Core adds, beside every anomaly, the stretches Steward itself wasn't recording (kind `"steward"`: a restart, a deploy, a stall), read from its own once-a-minute health sample. A plugin doesn't need to report those. + --- ## Shipping a Plugin diff --git a/plugins/traefik/__init__.py b/plugins/traefik/__init__.py index 612b6f9..335734a 100644 --- a/plugins/traefik/__init__.py +++ b/plugins/traefik/__init__.py @@ -72,3 +72,10 @@ def register_mcp_tools(register) -> None: async def mcp_overview(db) -> list[dict]: from .mcp_tools import mcp_overview as findings return await findings(db) + + +async def mcp_resource_events(db, source: str, resource: str, since, until) -> list[dict]: + if source != "traefik": + return [] + from .mcp_tools import service_events + return await service_events(db, resource, since, until) diff --git a/plugins/traefik/mcp_tools.py b/plugins/traefik/mcp_tools.py index ba9d6cf..41714ba 100644 --- a/plugins/traefik/mcp_tools.py +++ b/plugins/traefik/mcp_tools.py @@ -22,6 +22,7 @@ from steward.mcp.tools._shape import ( from steward.mcp.tools.overview import finding from steward.models.users import UserRole +from . import meaning from . import routes as page from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat from .scheduler import error_page_services, router_domains @@ -316,6 +317,29 @@ async def mcp_overview(db) -> list[dict]: return router_findings(current) + cert_findings(served, now, thresholds) +async def service_events(db, router: str, since, until) -> list[dict]: + """What happened to the containers of the swarm service behind a router + (starts with their image, deaths, OOM kills, health changes), on every + host: a router anomaly has no host of its own to read events by (#5862). + [] without the Docker plugin, or when no single service matches.""" + from steward.core.plugin_manager import loaded_plugin_submodule + from steward.mcp.tools._events import PER_SOURCE, event + from steward.models.hosts import Host + models = loaded_plugin_submodule("docker", "models") + if models is None: + return [] + service = meaning.swarm_service(router, await page.swarm_service_names(db)) + if service is None: + return [] + e = models.DockerEvent + rows = (await db.execute( + select(e, Host.name).join(Host, Host.id == e.host_id) + .where(e.container_name.startswith(f"{service}.", autoescape=True), e.at >= since, e.at <= until) + .order_by(e.at).limit(PER_SOURCE))).all() + return [event(ev.at, "container", f"{ev.container_name} {ev.event}" + (f" ({ev.detail})" if ev.detail else ""), + subject=f"{host}/{ev.container_name}") for ev, host in rows] + + def register_tools(register) -> None: for fn in (list_routers, get_router, list_certificates, get_traffic): register(fn, role=UserRole.viewer, read_only=True) diff --git a/steward/mcp/server.py b/steward/mcp/server.py index 962cb87..48856d9 100644 --- a/steward/mcp/server.py +++ b/steward/mcp/server.py @@ -82,6 +82,9 @@ class StewardMCPServer(MCPServer): # plugin name -> its `mcp_events(db, host_id, host_name, since, until)`: # what it knows happened on a host, for get_anomaly. self.event_sources: dict[str, object] = {} + # plugin name -> its `mcp_resource_events(db, source, resource, since, + # until)`: what it knows happened to a resource with no host. + self.resource_event_sources: dict[str, object] = {} # ── registration ───────────────────────────────────────────────────────── def register_tool(self, fn, *, role: UserRole, read_only: bool, plugin: str | None = None) -> None: @@ -108,7 +111,9 @@ class StewardMCPServer(MCPServer): """Add a plugin's tools via its optional `register_mcp_tools(register)`, its optional `mcp_overview(db)` as a get_overview findings source, and its optional `mcp_events(db, host_id, host_name, since, until)` as a - source of what happened on a host (get_anomaly). + source of what happened on a host (get_anomaly), and its optional + `mcp_resource_events(db, source, resource, since, until)` for what + happened to a resource with no host. Idempotent per plugin (hot-reload drops its previous tools first) and tolerant: a raising hook is logged and leaves the plugin with no tools, @@ -123,6 +128,10 @@ class StewardMCPServer(MCPServer): events = getattr(module, "mcp_events", None) if callable(events): self.event_sources[name] = events + self.resource_event_sources.pop(name, None) + resource_events = getattr(module, "mcp_resource_events", None) + if callable(resource_events): + self.resource_event_sources[name] = resource_events hook = getattr(module, "register_mcp_tools", None) if hook is None: return diff --git a/steward/mcp/tools/_events.py b/steward/mcp/tools/_events.py index 2e1ee18..1d6c53b 100644 --- a/steward/mcp/tools/_events.py +++ b/steward/mcp/tools/_events.py @@ -1,26 +1,32 @@ """What else was going on: events on a host, or one resource, in a window (#5837), so an anomaly comes with its circumstances. -Core gathers alert transitions, Ansible runs and other metric anomalies. A -plugin adds its own through `mcp_events(db, host_id, host_name, since, until)` -in its __init__.py (the host agent its maintenance jobs and downtime, Docker -its container events); each returns a list of `event(...)`. +Core gathers alert transitions, Ansible runs, other metric anomalies and +Steward's own gaps in recording. A plugin adds its own through +`mcp_events(db, host_id, host_name, since, until)` in its __init__.py (the +host agent its maintenance jobs and downtime, Docker its container events), +and for a resource with no host through +`mcp_resource_events(db, source, resource, since, until)` (Traefik: the +containers of the swarm service behind a router, #5862); each returns a list +of `event(...)`. """ from __future__ import annotations import logging from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timedelta from sqlalchemy import and_, or_, select from steward.alerts.service import watches_host from steward.core.alerts import host_of_resource +from steward.core.formatting import human_duration from steward.mcp._context import mcp_app from steward.models.alerts import AlertEvent, AlertRule from steward.models.ansible import AnsibleRun from steward.models.ansible_inventory import AnsibleTarget from steward.models.hosts import Host +from steward.models.metrics import PluginMetric from steward.models.monitors import Monitor from steward.models.norms import MetricAnomaly, MetricAnomalySpan @@ -29,6 +35,9 @@ from ._shape import db_session, iso logger = logging.getLogger(__name__) PER_SOURCE = 40 # events any one source contributes to one window +# Steward samples its own health once a minute (core/self_monitor.py); a +# longer break between two samples is a stretch it wasn't recording. +STEWARD_GAP_S = 90.0 def event(at: datetime | None, kind: str, what: str, *, until: datetime | None = None, @@ -142,6 +151,34 @@ async def other_anomalies(db, place: Place, anomaly_id: str, since: datetime, return here[:PER_SOURCE], elsewhere +def gaps_in(times: list[datetime], since: datetime, until: datetime) -> list[dict]: + """Each break longer than STEWARD_GAP_S between consecutive samples that + overlaps the window, as a "steward" event lasting the break.""" + out = [] + for before, after in zip(times, times[1:]): + secs = (after - before).total_seconds() + if secs > STEWARD_GAP_S and after >= since and before <= until: + out.append(event(before, "steward", + f"Steward wasn't recording for {human_duration(secs)} (a restart, a deploy or a " + "stall): readings either side of a gap can be wrong", until=after)) + return out + + +async def steward_gaps(db, since: datetime, until: datetime) -> list[dict]: + """When Steward itself wasn't recording, read from its own once-a-minute + health sample. Shown beside every anomaly, placed or not: a restart + distorts whatever Steward measures, a Traefik counter as much as a host's + silence (#5862). Raw samples only, so a window older than their + retention shows none.""" + rows = (await db.execute( + select(PluginMetric.recorded_at).where( + PluginMetric.source_module == "steward", PluginMetric.resource_name == "process", + PluginMetric.metric_name == "db_roundtrip_ms", + PluginMetric.recorded_at >= since - timedelta(hours=1), PluginMetric.recorded_at <= until) + .order_by(PluginMetric.recorded_at))).scalars() + return gaps_in(list(rows), since, until) + + async def plugin_events(place: Place, since: datetime, until: datetime) -> list[dict]: """What enabled plugins know happened on the host (`mcp_events`), each in its own session so one that fails can't spoil the others' reads.""" @@ -160,11 +197,30 @@ async def plugin_events(place: Place, since: datetime, until: datetime) -> list[ return out +async def resource_events(place: Place, since: datetime, until: datetime) -> list[dict]: + """What enabled plugins know happened to the resource itself + (`mcp_resource_events`), for one with no host to read events by.""" + server = getattr(mcp_app(), "mcp_server", None) + if server is None: + return [] + out = [] + for plugin, source in sorted((getattr(server, "resource_event_sources", None) or {}).items()): + if not server.plugin_enabled(plugin): + continue + try: + async with db_session() as db: + out.extend((await source(db, place.source, place.resource, since, until))[:PER_SOURCE]) + except Exception: + logger.exception("MCP resource events source of plugin %r failed", plugin) + return out + + async def around(db, place: Place, anomaly_id: str, since: datetime, until: datetime) -> tuple[list[dict], int]: """Everything above for one window, oldest first, and the count of anomalies elsewhere.""" anomalies, elsewhere = await other_anomalies(db, place, anomaly_id, since, until) found = (await alert_events(db, place, since, until) + await ansible_runs(db, place, since, until) - + anomalies + await plugin_events(place, since, until)) + + anomalies + await plugin_events(place, since, until) + + await resource_events(place, since, until) + await steward_gaps(db, since, until)) return sorted(found, key=lambda e: e["at"] or ""), elsewhere diff --git a/steward/mcp/tools/anomalies.py b/steward/mcp/tools/anomalies.py index 43178c4..d4660c1 100644 --- a/steward/mcp/tools/anomalies.py +++ b/steward/mcp/tools/anomalies.py @@ -158,9 +158,12 @@ async def get_anomaly(anomaly_id: str, occurrences: int = 5) -> dict: - `occurrences[]`: `started_at`, `ended_at`, `held_s`, `peak_level`, and `usual` for that hour as Steward has learned it now (`usual_hold`, `usual_peak`, `flag_above`), plus `around`: alert transitions, Ansible - runs, other anomalies on the host, and what plugins saw (container + runs, other anomalies on the host, what plugins saw (container starts, deaths and OOM kills; maintenance jobs; reboots and agent - stops), from 30 minutes before to 30 minutes after; and + stops; for a Traefik router, its swarm service's containers on any + host), and kind "steward": a stretch Steward itself wasn't recording + (a restart or deploy), around which any reading can be wrong. From 30 + minutes before to 30 minutes after; and `anomalies_elsewhere`, how many anomalies on other hosts overlapped it. Many at once points at something shared (network, storage, Steward). - `series`: the newest shown occurrence with time either side (as long diff --git a/tests/mcp/test_anomaly_review.py b/tests/mcp/test_anomaly_review.py index ba281ce..60b45dd 100644 --- a/tests/mcp/test_anomaly_review.py +++ b/tests/mcp/test_anomaly_review.py @@ -103,3 +103,27 @@ def test_a_plugin_adds_events_through_its_hook(app): assert "events_test" not in mcp.event_sources finally: mcp.event_sources.pop("events_test", None) + + +def test_a_plugin_adds_a_hostless_resources_events_through_its_hook(app): + async def events(db, source, resource, since, until): + return [] + + mcp = app.mcp_server + mcp.register_plugin("events_test", NS(mcp_resource_events=events)) + try: + assert mcp.resource_event_sources["events_test"] is events + mcp.register_plugin("events_test", NS()) + assert "events_test" not in mcp.resource_event_sources + finally: + mcp.resource_event_sources.pop("events_test", None) + + +def test_steward_not_recording_is_an_event_beside_every_anomaly(): + """A Steward restart between two of its once-a-minute samples (#5862).""" + ticks = [NOW + timedelta(minutes=m) for m in (0, 1, 2)] + [NOW + timedelta(minutes=4, seconds=10)] + [gap] = _events.gaps_in(ticks, NOW, NOW + timedelta(hours=1)) + assert gap["kind"] == "steward" and gap["at"] == "2026-10-15T12:02:00Z" + assert gap["until"] == "2026-10-15T12:04:10Z" and "wasn't recording" in gap["what"] + assert _events.gaps_in(ticks, NOW + timedelta(minutes=5), NOW + timedelta(hours=1)) == [] + assert _events.gaps_in([NOW + timedelta(minutes=m) for m in range(5)], NOW, NOW + timedelta(hours=1)) == []