feat(mcp): diagnostic tools — overview, hosts, monitors, alerts, metrics, audit

MCP-4 of milestone 461 "MCP server: Steward as an LLM-inspectable,
LLM-operable hub". Twelve read-only viewer tools:

- get_overview: what is wrong now, ranked critical-then-oldest. Failing
  monitors with their error and when the run of failures began, firing,
  pending and still-breaching acknowledged alerts, TLS certificates expiring
  within 14 days, agents that stopped reporting, hosts over the configured
  CPU/memory/disk thresholds (Settings > Thresholds), failed Ansible runs in
  the last 24h, and findings from enabled plugins through a new optional
  `mcp_overview(db)` plugin hook. Each finding names the call that shows
  its evidence.
- list_hosts, get_host, get_host_metrics
- list_monitors, get_monitor (HTTP header values and URL passwords are
  redacted)
- list_alerts, get_alert_history, list_alert_rules, list_maintenance_windows
- query_metrics: the whole metric store, narrowed one argument at a time
  (sources > resources > latest values > a series summary)
- list_audit_events

Shared shaping in steward/mcp/tools/_shape.py: UTC ISO times with age_s,
bounded lists that say when and how they were cut, series as
min/avg/max/p95/last plus a ≤60-point series, lookup by id or
case-insensitive name with a did-you-mean, and a range argument that
refuses an unknown key instead of reading it as 24h.

New core reads in steward/core/metric_series.py (sources and resources
via loose index scans, latest values via DISTINCT ON, series across the
raw/hourly-rollup seam). monitors/service.py gains monitor_states,
down_since and recent_failures; agent_overview_by_host now carries the
fullest disk and the last sample time.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_mcp_tools.py

@@ -0,0 +1,201 @@
+"""Integration: the MCP diagnostic tools against real Postgres.
+
+Seeds one small, uniquely named slice of a fleet — a host with a failing ping
+monitor, an agent reporting high CPU, a firing alert, an HTTPS monitor whose
+certificate is about to expire, a failed Ansible run, an active maintenance
+window and an audit event — then calls each tool as a viewer and checks that
+its slice comes back shaped as the tool promises. The database is shared with
+other tests, so assertions look for the seeded rows rather than exact totals.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+_NEEDS_DB = pytest.mark.skipif(
+    not os.environ.get("STEWARD_DATABASE_URL"),
+    reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)",
+)
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+@_NEEDS_DB
+def test_diagnostic_tools_over_a_seeded_fleet(app):
+    from steward.core.api_tokens import Caller
+    from steward.core.audit import log_audit
+    from steward.mcp import _context
+    from steward.mcp.tools import alerts, audit, hosts, metrics, monitors, overview
+    from steward.models.alerts import (
+        AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
+    )
+    from steward.models.ansible import AnsibleRun, AnsibleRunStatus
+    from steward.models.hosts import Host
+    from steward.models.metrics import PluginMetric
+    from steward.models.monitors import Monitor, MonitorResult
+    from steward.models.users import User, UserRole
+
+    now = datetime.now(timezone.utc)
+    tag = uuid.uuid4().hex[:8]
+    uid, hid, rule_id = str(uuid.uuid4()), str(uuid.uuid4()), str(uuid.uuid4())
+    ping_id, web_id, run_id = str(uuid.uuid4()), str(uuid.uuid4()), str(uuid.uuid4())
+    host_name, username = f"mcp-host-{tag}", f"mcp-{tag}"
+
+    async def _seed():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add(User(id=uid, username=username, email=f"{tag}@e.test",
+                           password_hash="x", role=UserRole.admin, is_active=True))
+                s.add(Host(id=hid, name=host_name, address="10.9.9.9"))
+            async with s.begin():
+                s.add(Monitor(id=ping_id, name=f"ping-{tag}", type="icmp",
+                              target="10.9.9.9", host_id=hid, config_json="{}", enabled=True))
+                s.add(Monitor(id=web_id, name=f"web-{tag}", type="http",
+                              target="https://bob:hunter2@web.test/health",
+                              config_json=json.dumps({"headers": {"Authorization": "Bearer s3cret"}}),
+                              enabled=True))
+            async with s.begin():
+                s.add(MonitorResult(monitor_id=ping_id, checked_at=now - timedelta(minutes=10),
+                                    is_up=True, response_ms=1.0))
+                s.add(MonitorResult(monitor_id=ping_id, checked_at=now - timedelta(minutes=5),
+                                    is_up=False, error_msg="timeout"))
+                s.add(MonitorResult(monitor_id=ping_id, checked_at=now - timedelta(minutes=1),
+                                    is_up=False, error_msg="timeout"))
+                s.add(MonitorResult(monitor_id=web_id, checked_at=now - timedelta(minutes=1),
+                                    is_up=True, status_code=200, response_ms=40.0,
+                                    tls_expires_at=now + timedelta(days=5)))
+                for i, v in enumerate((10.0, 20.0, 95.0)):
+                    s.add(PluginMetric(source_module="host_agent", resource_name=host_name,
+                                       metric_name="cpu_pct", value=v,
+                                       recorded_at=now - timedelta(minutes=40 - i * 20, seconds=30)))
+                s.add(PluginMetric(source_module="host_agent", resource_name=host_name,
+                                   metric_name="mem_used_pct", value=50.0,
+                                   recorded_at=now - timedelta(seconds=30)))
+                s.add(PluginMetric(source_module="host_agent", resource_name=host_name + ":/",
+                                   metric_name="disk_used_pct", value=41.0,
+                                   recorded_at=now - timedelta(seconds=30)))
+                s.add(AlertRule(id=rule_id, name=f"cpu hot {tag}", source_module="host_agent",
+                                resource_name=host_name, metric_name="cpu_pct",
+                                operator=AlertOperator.gt, threshold=90.0,
+                                consecutive_failures_required=1, enabled=True, created_by=uid))
+            async with s.begin():
+                s.add(AlertState(rule_id=rule_id, state=AlertStateEnum.firing,
+                                 last_transition_at=now - timedelta(minutes=2)))
+                s.add(AlertEvent(rule_id=rule_id, from_state="inactive", to_state="firing",
+                                 metric_value=95.0, transitioned_at=now - timedelta(minutes=2)))
+                s.add(MaintenanceWindow(name=f"patching {tag}", start_at=now - timedelta(hours=1),
+                                        end_at=now + timedelta(hours=1), scope_source="host_agent",
+                                        scope_resource=host_name, created_by=uid))
+                s.add(AnsibleRun(id=run_id, playbook_path=f"site-{tag}.yml", source_name="t",
+                                 status=AnsibleRunStatus.failed, started_at=now - timedelta(hours=1),
+                                 finished_at=now - timedelta(minutes=55),
+                                 results={"failures": ["web01: apt lock held"]}))
+        await log_audit(app, uid, username, "host.created", "host", hid, {"name": host_name})
+
+    async def _go():
+        await _seed()
+        _context._app_ctx.set(app)
+        _context._caller_ctx.set(Caller(user_id=uid, username=username,
+                                        role=UserRole.viewer, token_id=0))
+
+        # ── get_overview ───────────────────────────────────────────────────
+        ov = await overview.get_overview()
+        assert ov["all_clear"] is False
+        mine = {(f["kind"], f["subject"]): f for f in ov["findings"]
+                if tag in f["subject"] or f["subject"] == host_name}
+        down = mine[("monitor_down", f"ping-{tag}")]
+        assert down["severity"] == "critical" and "timeout" in down["summary"]
+        assert down["ids"]["monitor_id"] == ping_id
+        # down_since is the first failure after the last success, not the latest check.
+        assert 250 <= down["age_s"] <= 400
+        assert mine[("tls_expiring", f"web-{tag}")]["severity"] == "warning"
+        assert mine[("alert_firing", f"cpu hot {tag}")]["ids"]["rule_id"] == rule_id
+        assert mine[("host_cpu_high", host_name)]["severity"] == "critical"
+        assert "apt lock held" in mine[("ansible_run_failed", f"site-{tag}.yml")]["summary"]
+        assert any(w["name"] == f"patching {tag}" for w in ov["active_maintenance"])
+        sev = [f["severity"] for f in ov["findings"]]
+        assert sev == sorted(sev, key=lambda s: {"critical": 0, "warning": 1}[s])
+
+        # ── hosts ──────────────────────────────────────────────────────────
+        listed = {h["host_name"]: h for h in (await hosts.list_hosts(limit=500))["hosts"]}
+        h = listed[host_name]
+        assert h["status"] == "down" and h["agent"]["reporting"] is True
+        assert h["agent"]["cpu_pct"] == 95.0
+
+        detail = await hosts.get_host(host_name.upper())  # names match case-insensitively
+        assert detail["host_id"] == hid
+        assert [m["state"] for m in detail["monitors"]] == ["down"]
+        assert detail["monitors"][0]["error"] == "timeout"
+        assert detail["agent"]["parts"]["/"]["disk_used_pct"] == 41.0
+        assert [a["rule_id"] for a in detail["active_alerts"]] == [rule_id]
+        assert detail["ansible_target"] is None
+
+        series = await hosts.get_host_metrics(hid, "cpu_pct", "1h")
+        assert series["max"] == 95.0 and series["points"] >= 1 and series["series"]
+        listing = await hosts.get_host_metrics(host_name)
+        assert {m["metric"] for m in listing["metrics"]} == {"cpu_pct", "mem_used_pct"}
+        with pytest.raises(ValueError, match="no metric 'nope'"):
+            await hosts.get_host_metrics(host_name, "nope")
+        with pytest.raises(ValueError, match="list_hosts"):
+            await hosts.get_host(f"no-such-{tag}")
+
+        # ── monitors ───────────────────────────────────────────────────────
+        down_list = await monitors.list_monitors(state="down", limit=500)
+        ping = next(m for m in down_list["monitors"] if m["monitor_id"] == ping_id)
+        assert ping["host_name"] == host_name and ping["down_since"]
+        web = await monitors.get_monitor(f"web-{tag}")
+        assert "hunter2" not in web["target"] and "bob:***@" in web["target"]
+        assert web["config"]["headers"] == {"Authorization": "<redacted>"}
+        assert 4 <= web["tls_expires_in_days"] <= 5.1
+        ping_detail = await monitors.get_monitor(ping_id, failures=5)
+        assert ping_detail["heartbeat"] == "+--"
+        assert [f["error"] for f in ping_detail["recent_failures"]] == ["timeout", "timeout"]
+        with pytest.raises(ValueError, match="is not one of"):
+            await monitors.list_monitors(state="broken")
+
+        # ── alerts ─────────────────────────────────────────────────────────
+        firing = [a for a in (await alerts.list_alerts())["alerts"] if a["rule_id"] == rule_id]
+        assert firing[0]["state"] == "firing" and firing[0]["condition"] == "cpu_pct > 90"
+        hist = await alerts.get_alert_history(rule=f"cpu hot {tag}", range="24h")
+        assert [(e["from_state"], e["to_state"]) for e in hist["events"]] == [("inactive", "firing")]
+        rules = {r["rule_id"]: r for r in (await alerts.list_alert_rules(limit=500))["rules"]}
+        assert rules[rule_id]["state"] == "firing"
+        windows = (await alerts.list_maintenance_windows(limit=500))["windows"]
+        assert any(w["name"] == f"patching {tag}" and w["active"] for w in windows)
+
+        # ── query_metrics, one argument at a time ──────────────────────────
+        assert "host_agent" in (await metrics.query_metrics())["source_modules"]
+        res = (await metrics.query_metrics("host_agent", limit=1000))["resources"]
+        assert host_name in res and host_name + ":/" in res
+        latest = await metrics.query_metrics("host_agent", host_name)
+        assert {m["metric"] for m in latest["metrics"]} == {"cpu_pct", "mem_used_pct"}
+        cpu = await metrics.query_metrics("host_agent", host_name, "cpu_pct", "1h")
+        assert cpu["last"] == 95.0
+        with pytest.raises(ValueError, match="no metrics recorded"):
+            await metrics.query_metrics(f"nothing-{tag}")
+        with pytest.raises(ValueError, match="is not one of"):
+            await metrics.query_metrics("host_agent", host_name, "cpu_pct", "2w")
+
+        # ── audit ──────────────────────────────────────────────────────────
+        events = (await audit.list_audit_events(action="host.", user=username))["events"]
+        assert [(e["action"], e["entity_id"]) for e in events] == [("host.created", hid)]
+        assert events[0]["detail"] == {"name": host_name}
+
+        # ── through the SDK: arguments validate and results serialise ──────
+        await app.mcp_server.call_tool("list_hosts", {"limit": 5})
+        await app.mcp_server.call_tool("get_overview", {})
+
+    asyncio.run(_go())

tests/mcp/test_tools.py

@@ -0,0 +1,236 @@
+"""The diagnostic tools' pure parts: response shaping, lookup, redaction, ranking.
+
+The queries behind each tool run against real Postgres in
+tests/integration/test_mcp_tools.py; these cover the logic that decides what
+an agent is told.
+"""
+from __future__ import annotations
+
+import json
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace as NS
+from unittest.mock import MagicMock
+
+import pytest
+
+from steward.mcp.tools import _shape, overview
+from steward.mcp.tools.monitors import _config
+from steward.models.alerts import AlertOperator
+from steward.models.ansible import AnsibleRunStatus
+from steward.models.users import UserRole
+
+NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc)
+
+MCP4_TOOLS = {
+    "get_overview", "list_hosts", "get_host", "get_host_metrics", "query_metrics",
+    "list_monitors", "get_monitor", "list_alerts", "get_alert_history",
+    "list_alert_rules", "list_maintenance_windows", "list_audit_events",
+}
+
+
+# ── shaping ──────────────────────────────────────────────────────────────────
+
+def test_iso_is_utc_with_z_and_naive_times_are_read_as_utc():
+    assert _shape.iso(NOW) == "2026-10-06T12:00:00Z"
+    assert _shape.iso(datetime(2026, 10, 6, 12, 0)) == "2026-10-06T12:00:00Z"
+    assert _shape.iso(None) is None
+
+
+def test_age_s_never_negative():
+    assert _shape.age_s(NOW - timedelta(seconds=90), NOW) == 90
+    assert _shape.age_s(NOW + timedelta(seconds=5), NOW) == 0
+
+
+def test_clamp_limit():
+    assert _shape.clamp_limit(0, 50, 500) == 50
+    assert _shape.clamp_limit(-3, 50, 500) == 50
+    assert _shape.clamp_limit(10, 50, 500) == 10
+    assert _shape.clamp_limit(9999, 50, 500) == 500
+
+
+def test_bounded_says_when_and_how_it_cut():
+    whole = _shape.bounded([1, 2], 5, key="xs")
+    assert whole == {"xs": [1, 2], "total": 2, "truncated": False}
+    cut = _shape.bounded(list(range(10)), 3, key="xs", more="Filter by `state`.")
+    assert cut["xs"] == [0, 1, 2] and cut["total"] == 10 and cut["truncated"] is True
+    assert cut["more"] == "Showing 3 of 10. Filter by `state`."
+
+
+def test_lookback_refuses_an_unknown_range_instead_of_guessing():
+    since, key = _shape.lookback("7d")
+    assert key == "7d"
+    with pytest.raises(ValueError, match="'2w' is not one of"):
+        _shape.lookback("2w")
+
+
+def test_summarize_and_downsample():
+    pts = [(NOW + timedelta(minutes=i), float(i)) for i in range(1, 101)]
+    s = _shape.summarize(pts, n=10)
+    assert (s["points"], s["min"], s["max"], s["last"]) == (100, 1.0, 100.0, 100.0)
+    assert s["avg"] == 50.5 and s["p95"] == 95.0
+    assert len(s["series"]) == 10
+    assert s["series"][0] == ["2026-10-06T12:01:00Z", 5.5]
+    assert _shape.summarize([]) == {"points": 0, "series": []}
+
+
+def _rows(*names):
+    return [NS(id=f"id-{i}", name=n) for i, n in enumerate(names)]
+
+
+def test_pick_by_id_then_case_insensitive_name():
+    rows = _rows("nas", "web01")
+    assert _shape.pick("id-1", rows, kind="host", list_tool="list_hosts").name == "web01"
+    assert _shape.pick("NAS", rows, kind="host", list_tool="list_hosts").id == "id-0"
+
+
+def test_pick_explains_a_miss_and_an_ambiguity():
+    with pytest.raises(ValueError, match=r"Did you mean 'web01'\?.*list_hosts"):
+        _shape.pick("web1", _rows("nas", "web01"), kind="host", list_tool="list_hosts")
+    with pytest.raises(ValueError, match="2 hosts are named 'nas'.*id-0, id-1"):
+        _shape.pick("nas", _rows("nas", "NAS"), kind="host", list_tool="list_hosts")
+    with pytest.raises(ValueError, match="pass a host name or id"):
+        _shape.pick("", _rows("nas"), kind="host", list_tool="list_hosts")
+
+
+def test_redact_url_hides_only_the_password():
+    assert _shape.redact_url("https://bob:pw@h.test:8443/x?q=1") == "https://bob:***@h.test:8443/x?q=1"
+    assert _shape.redact_url("https://h.test/x") == "https://h.test/x"
+    assert _shape.redact_url("10.0.0.1") == "10.0.0.1"
+
+
+def test_monitor_config_redacts_header_values():
+    m = NS(config_json=json.dumps({"method": "GET", "headers": {"Authorization": "Bearer x"}}))
+    assert _config(m) == {"method": "GET", "headers": {"Authorization": "<redacted>"}}
+    assert _config(NS(config_json="not json")) == {}
+
+
+# ── overview findings ────────────────────────────────────────────────────────
+
+def _monitor(mid, *, enabled=True, host_id=None):
+    return NS(id=mid, name=f"mon-{mid}", type="http", target="https://x.test",
+              enabled=enabled, host_id=host_id)
+
+
+def _result(*, up, error=None, tls_days=None):
+    return NS(is_up=up, error_msg=error, checked_at=NOW - timedelta(minutes=1),
+              tls_expires_at=NOW + timedelta(days=tls_days) if tls_days is not None else None)
+
+
+def test_monitor_findings():
+    mons = [_monitor("a", host_id="h1"), _monitor("b"), _monitor("c"), _monitor("d", enabled=False),
+            _monitor("e")]
+    states = {
+        "a": {"latest": _result(up=False, error="503"), "down_since": NOW - timedelta(hours=2)},
+        "b": {"latest": _result(up=True, tls_days=2)},
+        "c": {"latest": _result(up=True, tls_days=10)},
+        "d": {"latest": _result(up=False)},
+        "e": {"latest": _result(up=True, tls_days=60)},
+    }
+    out = {f["subject"]: f for f in overview.monitor_findings(mons, states, {"h1": "nas"}, NOW)}
+    assert set(out) == {"mon-a", "mon-b", "mon-c"}  # disabled and healthy left out
+    down = out["mon-a"]
+    assert down["severity"] == "critical" and down["kind"] == "monitor_down"
+    assert "on nas" in down["summary"] and "503" in down["summary"]
+    assert down["since"] == "2026-10-06T10:00:00Z"
+    assert down["next"] == 'get_monitor(monitor="a")'
+    assert (out["mon-b"]["severity"], out["mon-c"]["severity"]) == ("critical", "warning")
+
+
+def test_alert_findings_severity_by_state():
+    rule = NS(id="r1", name="cpu", metric_name="cpu_pct", operator=AlertOperator.gt,
+              threshold=90.0, source_module="host_agent", resource_name="nas")
+    rows = [(NS(state=NS(value=s), last_transition_at=NOW), rule)
+            for s in ("firing", "pending", "acknowledged")]
+    sev = {f["kind"]: f["severity"] for f in overview.alert_findings(rows)}
+    assert sev == {"alert_firing": "critical", "alert_pending": "warning",
+                   "alert_acknowledged": "warning"}
+
+
+THRESHOLDS = {"cpu": {"warn": 80, "crit": 90}, "mem": {"warn": 80, "crit": 90},
+              "disk": {"warn": 80, "crit": 90}}
+
+
+def test_agent_findings_use_the_configured_thresholds():
+    hosts = [NS(id="h1", name="nas"), NS(id="h2", name="web"), NS(id="h3", name="db")]
+    agent = {
+        "nas": {"fresh": True, "cpu_pct": 95.0, "mem_used_pct": 85.0, "disk_used_pct_worst": 10.0},
+        "web": {"fresh": False, "cpu_pct": 99.0, "last_at": NOW - timedelta(minutes=30)},
+    }
+    out = overview.agent_findings(hosts, agent, THRESHOLDS)
+    got = {(f["subject"], f["kind"]): f["severity"] for f in out}
+    assert got == {
+        ("nas", "host_cpu_high"): "critical",
+        ("nas", "host_mem_high"): "warning",
+        # A stale agent's old numbers are not reported as current load.
+        ("web", "agent_stale"): "warning",
+    }
+
+
+def test_run_findings_quote_the_failures():
+    run = NS(id="r", playbook_path="site.yml", source_name="infra",
+             status=AnsibleRunStatus.failed, started_at=NOW, finished_at=NOW,
+             results={"failures": ["web01: apt lock held"]})
+    f = overview.run_findings([run])[0]
+    assert f["kind"] == "ansible_run_failed" and "apt lock held" in f["summary"]
+
+
+def test_rank_puts_critical_first_then_oldest():
+    fs = [overview.finding("warning", "k", "w-old", "", since=NOW - timedelta(days=1)),
+          overview.finding("critical", "k", "c-new", "", since=NOW - timedelta(minutes=1)),
+          overview.finding("critical", "k", "c-old", "", since=NOW - timedelta(hours=1))]
+    assert [f["subject"] for f in overview.rank(fs)] == ["c-old", "c-new", "w-old"]
+
+
+async def test_plugin_overview_sources_skip_disabled_plugins_and_survive_failures(monkeypatch):
+    calls = []
+
+    async def good(db):
+        calls.append("good")
+        return [overview.finding("warning", "container_unhealthy", "redis", "unhealthy")]
+
+    async def broken(db):
+        raise RuntimeError("boom")
+
+    async def off(db):
+        calls.append("off")
+        return []
+
+    class _CM:
+        async def __aenter__(self):
+            return object()
+
+        async def __aexit__(self, *a):
+            return False
+
+    monkeypatch.setattr(overview, "db_session", lambda: _CM())
+    server = MagicMock()
+    server.overview_sources = {"docker": good, "snmp": broken, "unifi": off}
+    server.plugin_enabled = lambda p: p != "unifi"
+    found, unavailable = await overview._plugin_findings(server)
+    assert [f["subject"] for f in found] == ["redis"]
+    assert unavailable == ["snmp"] and calls == ["good"]
+
+
+def test_register_plugin_picks_up_an_overview_source():
+    from steward.mcp.server import StewardMCPServer
+    app = MagicMock()
+    app.config = {"PLUGINS": {}}
+    mcp = StewardMCPServer(app)
+
+    async def mcp_overview(db):
+        return []
+    mcp.register_plugin("docker", NS(mcp_overview=mcp_overview))
+    assert mcp.overview_sources == {"docker": mcp_overview}
+    mcp.register_plugin("docker", NS())  # reloaded without it
+    assert mcp.overview_sources == {}
+
+
+# ── registration ─────────────────────────────────────────────────────────────
+
+def test_diagnostic_tools_are_registered_read_only_for_viewers(app):
+    mcp = app.mcp_server
+    assert MCP4_TOOLS <= set(mcp.tool_roles)
+    for name in MCP4_TOOLS:
+        assert mcp.tool_roles[name] is UserRole.viewer, name
+        assert mcp._tool_manager.get_tool(name).annotations.read_only_hint is True, name
+        assert (mcp._tool_manager.get_tool(name).description or "").strip(), f"{name} has no docstring"