bvandeusenandClaude Opus 5.5
b8f73270e7
The dashboard's host widgets took 18-60 s. Every "latest reading" query (hosts overview, host-agent fleet, host page, vitals, SNMP, MCP get_host) searched plugin_metrics for the newest row per metric. Postgres has no skip scan, so each one read every raw sample in the 7-day window. - plugin_metric_latest: one row per (source, resource, metric). record_metrics upserts it in the same transaction, moving forward only and in sorted key order. Migration 0030 seeds it by stepping the existing index. - Retention drops a latest row once its sample leaves the raw window, so a resource that stopped reporting disappears when its history does, as before. Renames carry it along. - A quiet host keeps its last values, marked stale. A time-bounded query would have shown it as missing. - Vitals sparklines span the 6 h they claim, averaged in SQL to ~120 points, instead of fetching 6 h raw and drawing the last 120 samples. - Dashboard and share-view cells: "Loading…" until the first answer; a 20 s deadline per fetch; "Couldn't load (why)" with Retry now and an automatic retry for cells that don't poll; "as of HH:MM" in the header, and "couldn't refresh · as of" when a refresh fails over shown data. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
210 lines
13 KiB
Python
210 lines
13 KiB
Python
"""Integration: the MCP diagnostic tools against real Postgres.
|
|
|
|
Seeds one small, uniquely named slice of a fleet — a host with a failing ping
|
|
monitor, an agent reporting high CPU, a firing alert, an HTTPS monitor whose
|
|
certificate is about to expire, a failed Ansible run, an active maintenance
|
|
window and an audit event — then calls each tool as a viewer and checks that
|
|
its slice comes back shaped as the tool promises. The database is shared with
|
|
other tests, so assertions look for the seeded rows rather than exact totals.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
_NEEDS_DB = pytest.mark.skipif(
|
|
not os.environ.get("STEWARD_DATABASE_URL"),
|
|
reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)",
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_diagnostic_tools_over_a_seeded_fleet(app):
|
|
from steward.core.api_tokens import Caller
|
|
from steward.core.audit import log_audit
|
|
from steward.mcp import _context
|
|
from steward.mcp.tools import alerts, audit, hosts, metrics, monitors, overview
|
|
from steward.models.alerts import (
|
|
AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
|
|
)
|
|
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
|
from steward.models.hosts import Host
|
|
from steward.core.alerts import _keep_latest
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.monitors import Monitor, MonitorResult
|
|
from steward.models.users import User, UserRole
|
|
|
|
now = datetime.now(timezone.utc)
|
|
tag = uuid.uuid4().hex[:8]
|
|
uid, hid, rule_id = str(uuid.uuid4()), str(uuid.uuid4()), str(uuid.uuid4())
|
|
ping_id, web_id, run_id = str(uuid.uuid4()), str(uuid.uuid4()), str(uuid.uuid4())
|
|
host_name, username = f"mcp-host-{tag}", f"mcp-{tag}"
|
|
|
|
async def _seed():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
s.add(User(id=uid, username=username, email=f"{tag}@e.test",
|
|
password_hash="x", role=UserRole.admin, is_active=True))
|
|
s.add(Host(id=hid, name=host_name, address="10.9.9.9"))
|
|
async with s.begin():
|
|
s.add(Monitor(id=ping_id, name=f"ping-{tag}", type="icmp",
|
|
target="10.9.9.9", host_id=hid, config_json="{}", enabled=True))
|
|
s.add(Monitor(id=web_id, name=f"web-{tag}", type="http",
|
|
target="https://bob:hunter2@web.test/health",
|
|
config_json=json.dumps({"headers": {"Authorization": "Bearer s3cret"}}),
|
|
enabled=True))
|
|
async with s.begin():
|
|
s.add(MonitorResult(monitor_id=ping_id, checked_at=now - timedelta(minutes=10),
|
|
is_up=True, response_ms=1.0))
|
|
s.add(MonitorResult(monitor_id=ping_id, checked_at=now - timedelta(minutes=5),
|
|
is_up=False, error_msg="timeout"))
|
|
s.add(MonitorResult(monitor_id=ping_id, checked_at=now - timedelta(minutes=1),
|
|
is_up=False, error_msg="timeout"))
|
|
s.add(MonitorResult(monitor_id=web_id, checked_at=now - timedelta(minutes=1),
|
|
is_up=True, status_code=200, response_ms=40.0,
|
|
tls_expires_at=now + timedelta(days=5)))
|
|
samples = [PluginMetric(source_module="host_agent", resource_name=host_name,
|
|
metric_name="cpu_pct", value=v,
|
|
recorded_at=now - timedelta(minutes=40 - i * 20, seconds=30))
|
|
for i, v in enumerate((10.0, 20.0, 95.0))]
|
|
samples.append(PluginMetric(source_module="host_agent", resource_name=host_name,
|
|
metric_name="mem_used_pct", value=50.0,
|
|
recorded_at=now - timedelta(seconds=30)))
|
|
samples.append(PluginMetric(source_module="host_agent",
|
|
resource_name=host_name + ":/",
|
|
metric_name="disk_used_pct", value=41.0,
|
|
recorded_at=now - timedelta(seconds=30)))
|
|
s.add_all(samples)
|
|
# What record_metrics keeps beside the rows, without evaluating
|
|
# the rule this seed sets up by hand.
|
|
await _keep_latest(s, samples)
|
|
s.add(AlertRule(id=rule_id, name=f"cpu hot {tag}", source_module="host_agent",
|
|
resource_name=host_name, metric_name="cpu_pct",
|
|
operator=AlertOperator.gt, threshold=90.0,
|
|
consecutive_failures_required=1, enabled=True, created_by=uid))
|
|
async with s.begin():
|
|
s.add(AlertState(rule_id=rule_id, state=AlertStateEnum.firing,
|
|
last_transition_at=now - timedelta(minutes=2)))
|
|
s.add(AlertEvent(rule_id=rule_id, from_state="inactive", to_state="firing",
|
|
metric_value=95.0, transitioned_at=now - timedelta(minutes=2)))
|
|
s.add(MaintenanceWindow(name=f"patching {tag}", start_at=now - timedelta(hours=1),
|
|
end_at=now + timedelta(hours=1), scope_source="host_agent",
|
|
scope_resource=host_name, created_by=uid))
|
|
s.add(AnsibleRun(id=run_id, playbook_path=f"site-{tag}.yml", source_name="t",
|
|
status=AnsibleRunStatus.failed, started_at=now - timedelta(hours=1),
|
|
finished_at=now - timedelta(minutes=55),
|
|
results={"failures": ["web01: apt lock held"]}))
|
|
await log_audit(app, uid, username, "host.created", "host", hid, {"name": host_name})
|
|
|
|
async def _go():
|
|
await _seed()
|
|
_context._app_ctx.set(app)
|
|
_context._caller_ctx.set(Caller(user_id=uid, username=username,
|
|
role=UserRole.viewer, token_id=0))
|
|
|
|
# ── get_overview ───────────────────────────────────────────────────
|
|
ov = await overview.get_overview()
|
|
assert ov["all_clear"] is False
|
|
mine = {(f["kind"], f["subject"]): f for f in ov["findings"]
|
|
if tag in f["subject"] or f["subject"] == host_name}
|
|
down = mine[("monitor_down", f"ping-{tag}")]
|
|
assert down["severity"] == "critical" and "timeout" in down["summary"]
|
|
assert down["ids"]["monitor_id"] == ping_id
|
|
# down_since is the first failure after the last success, not the latest check.
|
|
assert 250 <= down["age_s"] <= 400
|
|
assert mine[("tls_expiring", f"web-{tag}")]["severity"] == "warning"
|
|
assert mine[("alert_firing", f"cpu hot {tag}")]["ids"]["rule_id"] == rule_id
|
|
assert mine[("host_cpu_high", host_name)]["severity"] == "critical"
|
|
assert "apt lock held" in mine[("ansible_run_failed", f"site-{tag}.yml")]["summary"]
|
|
assert any(w["name"] == f"patching {tag}" for w in ov["active_maintenance"])
|
|
sev = [f["severity"] for f in ov["findings"]]
|
|
assert sev == sorted(sev, key=lambda s: {"critical": 0, "warning": 1}[s])
|
|
|
|
# ── hosts ──────────────────────────────────────────────────────────
|
|
listed = {h["host_name"]: h for h in (await hosts.list_hosts(limit=500))["hosts"]}
|
|
h = listed[host_name]
|
|
assert h["status"] == "down" and h["agent"]["reporting"] is True
|
|
assert h["agent"]["cpu_pct"] == 95.0
|
|
|
|
detail = await hosts.get_host(host_name.upper()) # names match case-insensitively
|
|
assert detail["host_id"] == hid
|
|
assert [m["state"] for m in detail["monitors"]] == ["down"]
|
|
assert detail["monitors"][0]["error"] == "timeout"
|
|
mount = detail["agent"]["mounts"]
|
|
assert mount["items"][0]["mount"] == "/" and mount["items"][0]["disk_used_pct"] == 41.0
|
|
assert mount["total"] == 1 and mount["truncated"] is False
|
|
assert [a["rule_id"] for a in detail["active_alerts"]] == [rule_id]
|
|
assert detail["ansible_target"] is None
|
|
|
|
series = await hosts.get_host_metrics(hid, "cpu_pct", "1h")
|
|
assert series["max"] == 95.0 and series["points"] >= 1 and series["series"]
|
|
listing = await hosts.get_host_metrics(host_name)
|
|
assert {m["metric"] for m in listing["metrics"]} == {"cpu_pct", "mem_used_pct"}
|
|
with pytest.raises(ValueError, match="no metric 'nope'"):
|
|
await hosts.get_host_metrics(host_name, "nope")
|
|
with pytest.raises(ValueError, match="list_hosts"):
|
|
await hosts.get_host(f"no-such-{tag}")
|
|
|
|
# ── monitors ───────────────────────────────────────────────────────
|
|
down_list = await monitors.list_monitors(state="down", limit=500)
|
|
ping = next(m for m in down_list["monitors"] if m["monitor_id"] == ping_id)
|
|
assert ping["host_name"] == host_name and ping["down_since"]
|
|
web = await monitors.get_monitor(f"web-{tag}")
|
|
assert "hunter2" not in web["target"] and "bob:***@" in web["target"]
|
|
assert web["config"]["headers"] == {"Authorization": "<redacted>"}
|
|
assert 4 <= web["tls_expires_in_days"] <= 5.1
|
|
ping_detail = await monitors.get_monitor(ping_id, failures=5)
|
|
assert ping_detail["heartbeat"] == "+--"
|
|
assert [f["error"] for f in ping_detail["recent_failures"]] == ["timeout", "timeout"]
|
|
with pytest.raises(ValueError, match="is not one of"):
|
|
await monitors.list_monitors(state="broken")
|
|
|
|
# ── alerts ─────────────────────────────────────────────────────────
|
|
firing = [a for a in (await alerts.list_alerts())["alerts"] if a["rule_id"] == rule_id]
|
|
assert firing[0]["state"] == "firing" and firing[0]["condition"] == "cpu_pct > 90"
|
|
hist = await alerts.get_alert_history(rule=f"cpu hot {tag}", range="24h")
|
|
assert [(e["from_state"], e["to_state"]) for e in hist["events"]] == [("inactive", "firing")]
|
|
rules = {r["rule_id"]: r for r in (await alerts.list_alert_rules(limit=500))["rules"]}
|
|
assert rules[rule_id]["state"] == "firing"
|
|
windows = (await alerts.list_maintenance_windows(limit=500))["windows"]
|
|
assert any(w["name"] == f"patching {tag}" and w["active"] for w in windows)
|
|
|
|
# ── query_metrics, one argument at a time ──────────────────────────
|
|
assert "host_agent" in (await metrics.query_metrics())["source_modules"]
|
|
res = (await metrics.query_metrics("host_agent", limit=1000))["resources"]
|
|
assert host_name in res and host_name + ":/" in res
|
|
latest = await metrics.query_metrics("host_agent", host_name)
|
|
assert {m["metric"] for m in latest["metrics"]} == {"cpu_pct", "mem_used_pct"}
|
|
cpu = await metrics.query_metrics("host_agent", host_name, "cpu_pct", "1h")
|
|
assert cpu["last"] == 95.0
|
|
with pytest.raises(ValueError, match="no metrics recorded"):
|
|
await metrics.query_metrics(f"nothing-{tag}")
|
|
with pytest.raises(ValueError, match="is not one of"):
|
|
await metrics.query_metrics("host_agent", host_name, "cpu_pct", "2w")
|
|
|
|
# ── audit ──────────────────────────────────────────────────────────
|
|
events = (await audit.list_audit_events(action="host.", user=username))["events"]
|
|
assert [(e["action"], e["entity_id"]) for e in events] == [("host.created", hid)]
|
|
assert events[0]["detail"] == {"name": host_name}
|
|
|
|
# ── through the SDK: arguments validate and results serialise ──────
|
|
await app.mcp_server.call_tool("list_hosts", {"limit": 5})
|
|
await app.mcp_server.call_tool("get_overview", {})
|
|
|
|
asyncio.run(_go())
|