feat(mcp): understand and judge a metric anomaly: what happened, what was usual, what else went on (#5837)

get_anomaly gives each occurrence against what is usual at that hour, the
metric around the newest one with its normal, what else happened on the
resource's host around each occurrence (alert transitions, Ansible runs,
other anomalies there and a count elsewhere, plus plugins' own events
through a new mcp_events hook: the host agent's maintenance jobs and
downtime, Docker's container starts, deaths and OOM kills), and how earlier
anomalies on the same metric were judged.

get_metric_normal reads the learned slots and the level that would be
flagged. review_anomaly takes one id or several, all or nothing, with a
note. get_anomaly_settings and (admin) set_anomaly_settings read and change
the review window and each normal's floor, as Settings → Thresholds does.

metric_series.points takes an optional `until`; core.alerts gains
host_of_resource; floor_setting names its floor.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_mcp_anomaly_review.py

@@ -0,0 +1,174 @@
+"""Integration (#5837): understanding and judging a metric anomaly through
+MCP, against real Postgres.
+
+A host's CPU anomaly with two occurrences. Around the newest one, the host
+had an alert, a container death, a reboot, an Ansible run and a memory
+anomaly; an earlier anomaly on the same metric was marked normal with a note.
+get_anomaly brings all of that back; review_anomaly marks two at once;
+get_metric_normal reads the learned slots; set_anomaly_settings changes a
+floor and the review window, and the running app follows.
+
+Plugin rows are seeded with plain SQL (a second import of a plugin's models
+raises "Table already defined").
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def test_an_anomaly_with_its_circumstances_judged_through_mcp(app):
+    from sqlalchemy import select, text
+    from steward.core.api_tokens import Caller
+    from steward.core.norms import slot_of
+    from steward.mcp import _context
+    from steward.mcp.tools import anomalies as tools
+    from steward.models.alerts import AlertEvent, AlertOperator, AlertRule
+    from steward.models.ansible import AnsibleRun, AnsibleRunStatus
+    from steward.models.audit import AuditEvent
+    from steward.models.hosts import Host
+    from steward.models.metrics import PluginMetric
+    from steward.models.norms import MetricAnomaly, MetricAnomalySpan, MetricNorm
+    from steward.models.users import User, UserRole
+
+    tag = uuid.uuid4().hex[:8]
+    host_name, other_host = f"an-{tag}", f"an-other-{tag}"
+    now = datetime.now(timezone.utc).replace(second=0, microsecond=0)
+    new_start, new_end = now - timedelta(hours=3), now - timedelta(hours=2)
+    old_start, old_end = now - timedelta(hours=26), now - timedelta(hours=25, minutes=30)
+    ids = {}
+
+    def anomaly(resource, metric, start, end, **kw):
+        return MetricAnomaly(source_module="host_agent", resource_name=resource, metric_name=metric,
+                             started_at=start, last_started_at=start, ended_at=end,
+                             occurrences=kw.pop("occurrences", 1), peak_level=kw.pop("peak", 90.0), peak_at=start, usual_hold=20.0,
+                             usual_peak=35.0, **kw)
+
+    async def _seed():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                user = User(username=f"an-{tag}", email=f"an-{tag}@test", password_hash="x",
+                            role=UserRole.admin)
+                host, other = Host(name=host_name, address="10.9.9.7"), Host(name=other_host, address="10.9.9.8")
+                s.add_all([user, host, other])
+                await s.flush()
+                ids.update(user=user.id, host=host.id)
+
+                cpu = anomaly(host_name, "cpu_pct", old_start, new_end, occurrences=2, peak=93.0)
+                cpu.last_started_at = new_start
+                before = anomaly(host_name, "cpu_pct", now - timedelta(days=9), now - timedelta(days=9, hours=-1),
+                                 review="normal", reviewed_by="op", note="nightly backup", peak=70.0)
+                mem = anomaly(host_name, "mem_used_pct", new_start + timedelta(minutes=10), new_end, peak=95.0)
+                far = anomaly(other_host, "cpu_pct", new_start, new_end)
+                s.add_all([cpu, before, mem, far])
+                await s.flush()
+                ids.update(cpu=cpu.id, before=before.id, mem=mem.id)
+                s.add_all([
+                    MetricAnomalySpan(anomaly_id=cpu.id, started_at=old_start, ended_at=old_end, peak_level=88.0),
+                    MetricAnomalySpan(anomaly_id=cpu.id, started_at=new_start, ended_at=new_end, peak_level=93.0),
+                    MetricAnomalySpan(anomaly_id=mem.id, started_at=new_start + timedelta(minutes=10),
+                                      ended_at=new_end, peak_level=95.0),
+                    MetricAnomalySpan(anomaly_id=far.id, started_at=new_start, ended_at=new_end, peak_level=90.0),
+                    MetricNorm(source_module="host_agent", resource_name=host_name, metric_name="cpu_pct",
+                               slot=slot_of(new_start, "day"), kind="day", usual_hold=20.0, usual_peak=35.0,
+                               days_seen=12, updated_at=now),
+                ])
+                for minutes in range(-240, -60, 5):
+                    at = now + timedelta(minutes=minutes)
+                    s.add(PluginMetric(source_module="host_agent", resource_name=host_name,
+                                       metric_name="cpu_pct", recorded_at=at,
+                                       value=90.0 if new_start <= at < new_end else 18.0))
+
+                rule = AlertRule(name=f"CPU high {tag}", source_module="host_agent", resource_name=host_name,
+                                 metric_name="cpu_pct", operator=AlertOperator.gt, threshold=85,
+                                 created_by=user.id)
+                s.add(rule)
+                await s.flush()
+                s.add(AlertEvent(rule_id=rule.id, from_state="ok", to_state="firing", metric_value=91.0,
+                                 transitioned_at=new_start + timedelta(minutes=20)))
+                s.add(AnsibleRun(playbook_path="maintenance/backup.yml", source_name="steward-builtin",
+                                 status=AnsibleRunStatus.success, started_at=new_start - timedelta(minutes=10),
+                                 finished_at=new_start + timedelta(minutes=5), params={"check": False},
+                                 results={"hosts": {host_name: {"ok": 3}}, "failures": []}))
+                await s.execute(text(
+                    "INSERT INTO docker_events (id, host_id, container_name, event, detail, at) "
+                    "VALUES (:id, :h, 'web', 'die', 'exit 137', :at)"),
+                    {"id": str(uuid.uuid4()), "h": host.id, "at": new_start + timedelta(minutes=15)})
+                await s.execute(text(
+                    "INSERT INTO host_downtime (id, host_id, kind, started_at, ended_at, created_at) "
+                    "VALUES (:id, :h, 'reboot', :s, :e, now())"),
+                    {"id": str(uuid.uuid4()), "h": host.id, "s": new_end - timedelta(minutes=5),
+                     "e": new_end + timedelta(minutes=1)})
+
+    async def _go():
+        await _seed()
+        _context._app_ctx.set(app)
+        _context._caller_ctx.set(Caller(user_id=ids["user"], username=f"an-{tag}",
+                                        role=UserRole.admin, token_id=9))
+
+        got = await tools.get_anomaly(ids["cpu"])
+        assert got["place"] == {"host": host_name, "host_id": ids["host"]}
+        assert got["occurrences_total"] == 2 and got["truncated"] is False
+        newest, oldest = got["occurrences"]
+        assert newest["held_s"] == 3600 and newest["peak_level"] == 93.0
+        assert newest["usual"]["usual_hold"] == 20.0 and newest["usual"]["flag_above"] >= 25.0
+        kinds = {e["kind"] for e in newest["around"]}
+        assert {"alert", "ansible_run", "container", "downtime", "anomaly"} <= kinds
+        whats = " | ".join(e["what"] for e in newest["around"])
+        assert "web die (exit 137)" in whats and "maintenance/backup.yml" in whats
+        assert any(e["kind"] == "anomaly" and e["ids"]["anomaly_id"] == ids["mem"] for e in newest["around"])
+        assert newest["anomalies_elsewhere"] >= 1
+        assert not [e for e in oldest["around"] if e["kind"] in ("container", "ansible_run")]
+        points = got["series"]["points"]
+        assert max(p[1] for p in points) == 90.0 and min(p[1] for p in points) == 18.0
+        assert any(p[2] == 20.0 for p in points)
+        [prior] = got["precedent"]
+        assert prior["anomaly_id"] == ids["before"] and prior["note"] == "nightly backup"
+
+        normal = await tools.get_metric_normal("host_agent", host_name, "cpu_pct")
+        assert normal["learned"] is True and normal["kind"] == "day"
+        assert normal["floor_setting"] == "host_cpu_pct"
+        [slot] = normal["slots"]
+        assert slot["usual_hold"] == 20.0 and slot["label"].endswith(":00 UTC")
+        with pytest.raises(ValueError, match="doesn't learn a normal"):
+            await tools.get_metric_normal("host_agent", host_name, "nope_pct")
+
+        with pytest.raises(ValueError, match="Nothing was marked"):
+            await tools.review_anomaly([ids["cpu"], "no-such-id"], "real")
+        marked = await tools.review_anomaly([ids["cpu"], ids["mem"]], "real", note="the backup job hung")
+        assert {m["anomaly_id"] for m in marked["marked"]} == {ids["cpu"], ids["mem"]}
+        assert marked["reviewed_by"] == f"an-{tag} via MCP"
+
+        try:
+            changed = await tools.set_anomaly_settings(review_days=21, floors={"host_cpu_pct": 15})
+            assert changed["review_days"] == 21
+            floor = {f["name"]: f for f in changed["floors"]}["host_cpu_pct"]
+            assert floor["value"] == 15.0 and "host_agent/cpu_pct" in floor["metrics"]
+            assert app.config["THRESHOLDS"]["floors"]["host_cpu_pct"] == 15.0
+        finally:
+            await tools.set_anomaly_settings(review_days=14, floors={"host_cpu_pct": 10})
+
+        async with app.db_sessionmaker() as s:
+            audits = list((await s.execute(select(AuditEvent.action).where(
+                AuditEvent.user_id == ids["user"]).order_by(AuditEvent.timestamp))).scalars())
+            rows = {a.id: a for a in (await s.execute(select(MetricAnomaly).where(
+                MetricAnomaly.id.in_([ids["cpu"], ids["mem"]])))).scalars()}
+        assert audits.count("metric_anomaly.marked_real") == 2 and audits.count("settings.saved") == 2
+        assert {a.review for a in rows.values()} == {"real"}
+        assert {a.note for a in rows.values()} == {"the backup job hung"}
+
+    asyncio.run(_go())

tests/mcp/test_anomaly_review.py

@@ -0,0 +1,105 @@
+"""The anomaly review tools' pure parts and refusals (#5837): slot names, the
+line an anomaly opens at, how far a series reaches, the plugin events hook,
+and calls refused before anything is read or changed.
+
+get_anomaly's circumstances, a bulk review and a settings change run against
+Postgres in tests/integration/test_mcp_anomaly_review.py.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace as NS
+
+import pytest
+
+from steward.core import alerts
+from steward.mcp.tools import _events, anomalies
+from steward.models.users import UserRole
+
+NOW = datetime(2026, 10, 15, 12, 0, tzinfo=timezone.utc)
+
+
+def test_slots_are_named_as_a_reader_would():
+    assert anomalies.slot_label("day", 3) == "03:00 UTC"
+    assert anomalies.slot_label("week", 0) == "Mon 00:00 UTC"
+    assert anomalies.slot_label("week", 6 * 24 + 23) == "Sun 23:00 UTC"
+
+
+def test_an_anomaly_opens_above_the_larger_of_the_factor_and_the_floor():
+    assert anomalies.flag_above(20.0, 10.0) == 30.0       # 20 + floor beats 20 × 1.25
+    assert anomalies.flag_above(100.0, 10.0) == 125.0     # the factor beats the floor
+
+
+def test_a_series_reaches_as_far_as_the_occurrence_lasted_within_one_to_six_hours():
+    assert anomalies.series_pad(NOW - timedelta(minutes=20), NOW, NOW) == timedelta(hours=1)
+    assert anomalies.series_pad(NOW - timedelta(hours=3), NOW, NOW) == timedelta(hours=3)
+    assert anomalies.series_pad(NOW - timedelta(days=2), None, NOW) == timedelta(hours=6)
+
+
+def test_an_event_says_when_what_and_until():
+    e = _events.event(NOW, "downtime", "Rebooted (60 s)", until=NOW + timedelta(minutes=1))
+    assert e == {"at": "2026-10-15T12:00:00Z", "until": "2026-10-15T12:01:00Z", "kind": "downtime",
+                 "what": "Rebooted (60 s)", "subject": None, "ids": {}}
+
+
+def test_a_resource_names_its_host_through_the_source_that_feeds_it():
+    alerts.agent_fed("test_src", lambda r: r.split(":", 1)[0])
+    try:
+        assert alerts.host_of_resource("test_src", "nas:/var") == "nas"
+        assert alerts.host_of_resource("no_such_source", "nas") is None
+    finally:
+        alerts._AGENT_FED.pop("test_src", None)
+
+
+def test_same_host_covers_the_resource_itself_and_its_hosts_other_series():
+    alerts.agent_fed("test_src", lambda r: r.split(":", 1)[0])
+    try:
+        place = _events.Place("test_src", "nas", "h1", "nas")
+        assert _events._same_host("test_src", "nas:/var", place)
+        assert not _events._same_host("test_src", "other", place)
+        bare = _events.Place("traefik", "router-a")
+        assert _events._same_host("traefik", "router-a", bare)
+        assert not _events._same_host("traefik", "router-b", bare)
+    finally:
+        alerts._AGENT_FED.pop("test_src", None)
+
+
+async def test_refusals_change_nothing():
+    with pytest.raises(ValueError, match="at least one anomaly"):
+        await anomalies.review_anomaly([], "normal")
+    with pytest.raises(ValueError, match="At most"):
+        await anomalies.review_anomaly([f"id-{i}" for i in range(anomalies.REVIEW_MAX + 1)], "real")
+    with pytest.raises(ValueError, match="No floor named bogus"):
+        await anomalies.set_anomaly_settings(floors={"bogus": 1})
+    with pytest.raises(ValueError, match="0 or more"):
+        await anomalies.set_anomaly_settings(floors={"host_cpu_pct": -1})
+    with pytest.raises(ValueError, match="Nothing to change"):
+        await anomalies.set_anomaly_settings()
+    with pytest.raises(ValueError, match="doesn't learn a normal"):
+        await anomalies.get_metric_normal("no_source", "x", "no_metric")
+
+
+def test_anomaly_tools_are_registered_with_their_roles(app):
+    mcp = app.mcp_server
+    for name, role, read_only in (("list_anomalies", UserRole.viewer, True),
+                                  ("get_anomaly", UserRole.viewer, True),
+                                  ("get_metric_normal", UserRole.viewer, True),
+                                  ("get_anomaly_settings", UserRole.viewer, True),
+                                  ("review_anomaly", UserRole.operator, False),
+                                  ("set_anomaly_settings", UserRole.admin, False)):
+        assert mcp.tool_roles[name] is role, name
+        assert mcp._tool_manager.get_tool(name).annotations.read_only_hint is read_only, name
+
+
+def test_a_plugin_adds_events_through_its_hook(app):
+    async def events(db, host_id, host_name, since, until):
+        return []
+
+    mcp = app.mcp_server
+    mcp.register_plugin("events_test", NS(mcp_events=events))
+    try:
+        assert mcp.event_sources["events_test"] is events
+        mcp.register_plugin("events_test", NS())
+        assert "events_test" not in mcp.event_sources
+    finally:
+        mcp.event_sources.pop("events_test", None)