feat(mcp): understand and judge a metric anomaly: what happened, what was usual, what else went on (#5837)

get_anomaly gives each occurrence against what is usual at that hour, the
metric around the newest one with its normal, what else happened on the
resource's host around each occurrence (alert transitions, Ansible runs,
other anomalies there and a count elsewhere, plus plugins' own events
through a new mcp_events hook: the host agent's maintenance jobs and
downtime, Docker's container starts, deaths and OOM kills), and how earlier
anomalies on the same metric were judged.

get_metric_normal reads the learned slots and the level that would be
flagged. review_anomaly takes one id or several, all or nothing, with a
note. get_anomaly_settings and (admin) set_anomaly_settings read and change
the review window and each normal's floor, as Settings → Thresholds does.

metric_series.points takes an optional `until`; core.alerts gains
host_of_resource; floor_setting names its floor.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/__init__.py

@@ -74,6 +74,11 @@ async def mcp_overview(db) -> list[dict]:
     return await findings(db)
 
 
+async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
+    from .mcp_tools import mcp_events as events
+    return await events(db, host_id, host_name, since, until)
+
+
 def get_nav() -> list[dict]:
     # Surfaces in the sidebar's Infrastructure group (fleet-wide container view).
     return [{"label": "Docker", "href": "/plugins/docker/"}]

plugins/docker/mcp_tools.py

@@ -27,7 +27,7 @@ from steward.models.users import UserRole
 from . import service as docker_svc
 from .service import PROBLEM_SAYS, PROBLEM_SEVERITY, PROBLEMS, container_problems, problem_since
 from .dedup import dedup_by_container_id
-from .models import DockerContainer, DockerDiskUsage, DockerSwarmNode, DockerSwarmService
+from .models import DockerContainer, DockerDiskUsage, DockerEvent, DockerSwarmNode, DockerSwarmService
 from .swarm_view import build_swarm_services, replaced_tasks, task_time
 
 CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS
@@ -751,6 +751,18 @@ async def prune_docker(host: str, what: str, check: bool = True) -> dict:
             "next": f'wait_for_run(run_id="{run.id}")'}
 
 
+async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
+    """Container lifecycle events on a host in a window (get_anomaly): starts
+    (with the image, so a deploy shows), deaths with their exit, OOM kills,
+    restarts and health changes."""
+    from steward.mcp.tools._events import event
+    rows = (await db.execute(select(DockerEvent).where(
+        DockerEvent.host_id == host_id, DockerEvent.at >= since, DockerEvent.at <= until)
+        .order_by(DockerEvent.at).limit(40))).scalars()
+    return [event(e.at, "container", f"{e.container_name} {e.event}" + (f" ({e.detail})" if e.detail else ""),
+                  subject=f"{host_name}/{e.container_name}") for e in rows]
+
+
 def register_tools(register) -> None:
     for fn in (list_containers, get_container, list_swarm_services,
                search_container_logs, get_docker_disk_usage):

plugins/host_agent/__init__.py

@@ -62,6 +62,11 @@ async def mcp_overview(db) -> list[dict]:
     return await findings(db)
 
 
+async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
+    from .mcp_tools import mcp_events as events
+    return await events(db, host_id, host_name, since, until)
+
+
 def get_alert_metrics() -> dict[str, list[str]]:
     """Host metrics ingest records for alert rules (resource = the host name;
     disk_used_pct per mount, resource "<host>:<mount>"). seconds_since_seen

plugins/host_agent/mcp_tools.py

@@ -15,8 +15,8 @@ from steward.mcp.tools.overview import finding
 from steward.models.hosts import Host
 from steward.models.users import UserRole
 
-from . import cadence, maintenance
-from .models import (HostAgentRegistration, HostMaintenanceJob, HostMaintenancePlan,
+from . import cadence, downtime, maintenance
+from .models import (HostAgentRegistration, HostDowntime, HostMaintenanceJob, HostMaintenancePlan,
                      HostMaintenanceRun)
 
 # A registration this old that has never checked in is a failed install, not one in progress.
@@ -295,6 +295,31 @@ async def mcp_overview(db) -> list[dict]:
     return out
 
 
+async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
+    """What the host agent knows happened on a host in a window (get_anomaly):
+    maintenance jobs, and downtime (reboots, agent stops, lost contact)."""
+    from steward.mcp.tools._events import event
+    out = []
+    jobs = (await db.execute(select(HostMaintenanceJob).where(
+        HostMaintenanceJob.host_id == host_id, HostMaintenanceJob.created_at <= until,
+        (HostMaintenanceJob.finished_at.is_(None)) | (HostMaintenanceJob.finished_at >= since))
+        .order_by(HostMaintenanceJob.created_at).limit(20))).scalars()
+    for job in jobs:
+        out.append(event(job.created_at, "maintenance", f"Maintenance: {maintenance.summary(job)}",
+                         until=job.finished_at, ids={"job_id": job.id}))
+    now = now_utc()
+    rows = (await db.execute(select(HostDowntime).where(
+        HostDowntime.host_id == host_id, HostDowntime.started_at <= until,
+        (HostDowntime.ended_at.is_(None)) | (HostDowntime.ended_at >= since))
+        .order_by(HostDowntime.started_at).limit(20))).scalars()
+    for row in rows:
+        seen = downtime.describe(row, now)
+        out.append(event(row.started_at, "downtime",
+                         f"{seen['label']} ({int(seen['seconds'])} s{', still going' if seen['ongoing'] else ''})",
+                         until=row.ended_at))
+    return out
+
+
 def register_tools(register) -> None:
     register(list_agents, role=UserRole.viewer, read_only=True)
     register(list_maintenance_plans, role=UserRole.viewer, read_only=True)