bvandeusenandClaude Opus 5.5
21338dbcd9
MCP-5 of milestone 461 "MCP server: Steward as an LLM-inspectable, LLM-operable hub". Contributed through each plugin's register_mcp_tools hook, so they exist only while the plugin is enabled. docker (viewer, read-only): - list_containers: filter by host, status, compose project, or problem (restarting, crash_looping, unhealthy, oom_killed, exited_nonzero); problems sort first and each row names what is wrong. - get_container: state, CPU/memory history across the raw/hourly seam, lifecycle events. A unique part of a swarm task name is enough. - list_swarm_services: replicas per node (agent-less placements as ghosts) and the swarm's nodes. - search_container_logs: text (% and _ literal), stream, range, context lines, paging by a full-precision `before` cursor; over-long lines keep both ends and say how much of the middle was cut. - get_docker_disk_usage: bytes per host, reclaimable first. - mcp_overview: failing containers and under-replicated swarm services join get_overview. host_agent: list_agents (reporting / stale / pending, version, OS), and an overview finding for agents registered over 30 minutes ago that never checked in. Settings > Thresholds gains the TLS-expiry warn/crit days (were 14/3 in code) and the crash-loop deaths-per-hour count (3), which get_overview and the docker tools now read. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
212 lines
12 KiB
Python
212 lines
12 KiB
Python
"""Integration: the docker and host_agent plugins' MCP tools against real Postgres.
|
|
|
|
Rows are seeded with plain SQL so this file never imports a plugin's models
|
|
(a second import of them after the app loaded the plugin raises "Table already
|
|
defined"). The tool modules come from the loaded plugin when there is one,
|
|
and from the source tree when the plugin is not loaded.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import importlib
|
|
import os
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
_NEEDS_DB = pytest.mark.skipif(
|
|
not os.environ.get("STEWARD_DATABASE_URL"),
|
|
reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)",
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
def _plugin(name):
|
|
from steward.core.plugin_manager import get_loaded_plugin_modules
|
|
loaded = get_loaded_plugin_modules().get(name)
|
|
return loaded if loaded is not None else importlib.import_module(f"plugins.{name}")
|
|
|
|
|
|
def _sub(plugin_module, sub):
|
|
return importlib.import_module(f"{plugin_module.__name__}.{sub}")
|
|
|
|
|
|
@_NEEDS_DB
|
|
def test_docker_and_agent_tools_over_seeded_rows(app):
|
|
from sqlalchemy import text
|
|
from steward.core.api_tokens import Caller
|
|
from steward.mcp import _context
|
|
from steward.models.hosts import Host
|
|
from steward.models.users import UserRole
|
|
|
|
docker, agent = _plugin("docker"), _plugin("host_agent")
|
|
dtools, atools = _sub(docker, "mcp_tools"), _sub(agent, "mcp_tools")
|
|
|
|
now = datetime.now(timezone.utc)
|
|
tag = uuid.uuid4().hex[:8]
|
|
hid, hid2 = str(uuid.uuid4()), str(uuid.uuid4())
|
|
host_name, quiet_host = f"dock-{tag}", f"quiet-{tag}"
|
|
web, db_, job, svc = f"web-{tag}", f"db-{tag}", f"job-{tag}", f"svc-{tag}"
|
|
|
|
container_sql = text("""
|
|
INSERT INTO docker_containers
|
|
(host_id, name, container_id, image, status, cpu_pct, mem_pct, mem_usage_bytes,
|
|
mem_limit_bytes, restart_count, ports_json, started_at, scraped_at, health,
|
|
exit_code, oom_killed, compose_project, service_name)
|
|
VALUES (:h, :n, :cid, 'img:1', :st, 5.0, 10.0, 1000, 10000, :rc, '[]', :started, :now,
|
|
:health, :exit, false, :proj, :svc)
|
|
""")
|
|
|
|
async def _seed():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
s.add(Host(id=hid, name=host_name, address="10.8.8.8"))
|
|
s.add(Host(id=hid2, name=quiet_host, address="10.8.8.9"))
|
|
async with s.begin():
|
|
for name, st, health, exit_code, proj, service in (
|
|
(web, "running", "healthy", None, f"stack-{tag}", None),
|
|
(db_, "running", "unhealthy", None, f"stack-{tag}", None),
|
|
(job, "exited", None, 1, None, None),
|
|
(f"{svc}.1.abc", "running", None, None, None, svc),
|
|
):
|
|
await s.execute(container_sql, {
|
|
"h": hid, "n": name, "cid": uuid.uuid4().hex, "st": st, "rc": 0,
|
|
"started": now - timedelta(hours=2), "now": now, "health": health,
|
|
"exit": exit_code, "proj": proj, "svc": service})
|
|
await s.execute(text(
|
|
"INSERT INTO docker_events (id, host_id, container_name, event, detail, at) "
|
|
"VALUES (:id, :h, :n, 'die', 'exit 1', :at)"),
|
|
{"id": str(uuid.uuid4()), "h": hid, "n": job, "at": now - timedelta(minutes=10)})
|
|
for i, (stream, line) in enumerate((
|
|
("stdout", "starting up"), ("stdout", "connected to db"),
|
|
("stderr", "ERROR: 100% of disk_used"), ("stdout", "retrying"),
|
|
("stdout", "x" * 5000))):
|
|
await s.execute(text(
|
|
"INSERT INTO docker_logs (id, host_id, container_name, ts, stream, line) "
|
|
"VALUES (:id, :h, :n, :ts, :st, :line)"),
|
|
{"id": str(uuid.uuid4()), "h": hid, "n": web,
|
|
"ts": now - timedelta(minutes=5 - i), "st": stream, "line": line})
|
|
for i, cpu in enumerate((10.0, 30.0, 50.0)):
|
|
await s.execute(text(
|
|
"INSERT INTO docker_metrics (id, host_id, container_name, scraped_at, "
|
|
"cpu_pct, mem_pct, mem_usage_bytes) VALUES (:id, :h, :n, :t, :c, 20.0, 1000)"),
|
|
{"id": str(uuid.uuid4()), "h": hid, "n": web,
|
|
"t": now - timedelta(minutes=30 - i * 10), "c": cpu})
|
|
await s.execute(text(
|
|
"INSERT INTO docker_swarm_services (host_id, service_name, mode, desired, running, "
|
|
"placement_json, updated_at) VALUES (:h, :n, 'replicated', 3, 1, '[]', :now)"),
|
|
{"h": hid, "n": svc, "now": now})
|
|
await s.execute(text(
|
|
"INSERT INTO docker_swarm_nodes (host_id, node_id, hostname, role, availability, "
|
|
"status, leader, updated_at) VALUES (:h, :nid, :hn, 'manager', 'active', 'ready', "
|
|
"true, :now)"), {"h": hid, "nid": f"node-{tag}", "hn": host_name, "now": now})
|
|
await s.execute(text(
|
|
"INSERT INTO docker_disk_usage (host_id, layers_size, images_size, images_reclaimable, "
|
|
"containers_size, volumes_size, build_cache_size, images_total, images_active, "
|
|
"containers_count, volumes_count, updated_at) "
|
|
"VALUES (:h, 1, 5000, 3000, 10, 20, 30, 4, 1, 4, 2, :now)"), {"h": hid, "now": now})
|
|
await s.execute(text(
|
|
"INSERT INTO host_agent_registrations (id, host_id, token_hash, token_created_at, "
|
|
"agent_version, last_seen_at, created_at, updated_at) "
|
|
"VALUES (:id, :h, :th, :t, '1.5.0', :seen, :t, :t)"),
|
|
{"id": str(uuid.uuid4()), "h": hid, "th": uuid.uuid4().hex + uuid.uuid4().hex,
|
|
"t": now - timedelta(days=1), "seen": now - timedelta(seconds=20)})
|
|
await s.execute(text(
|
|
"INSERT INTO host_agent_registrations (id, host_id, token_hash, token_created_at, "
|
|
"created_at, updated_at) VALUES (:id, :h, :th, :t, :t, :t)"),
|
|
{"id": str(uuid.uuid4()), "h": hid2, "th": uuid.uuid4().hex + uuid.uuid4().hex,
|
|
"t": now - timedelta(hours=2)})
|
|
|
|
async def _go():
|
|
await _seed()
|
|
_context._app_ctx.set(app)
|
|
_context._caller_ctx.set(Caller(user_id="u", username="viewer", role=UserRole.viewer, token_id=0))
|
|
|
|
# ── list_containers: problems first, filters ───────────────────────
|
|
mine = [c for c in (await dtools.list_containers(host=host_name, limit=500))["containers"]]
|
|
names = [c["container_name"] for c in mine]
|
|
assert set(names) >= {web, db_, job}
|
|
problems = {c["container_name"]: c["problems"] for c in mine}
|
|
assert problems[db_] == ["unhealthy"] and problems[job] == ["exited_nonzero"]
|
|
assert problems[web] == []
|
|
assert names.index(db_) < names.index(web) and names.index(job) < names.index(web)
|
|
only = (await dtools.list_containers(host=hid, problem="unhealthy"))["containers"]
|
|
assert [c["container_name"] for c in only] == [db_]
|
|
proj = (await dtools.list_containers(compose_project=f"stack-{tag}"))["containers"]
|
|
assert {c["container_name"] for c in proj} == {web, db_}
|
|
with pytest.raises(ValueError, match="is not 'any' or one of"):
|
|
await dtools.list_containers(problem="sad")
|
|
|
|
# ── get_container ──────────────────────────────────────────────────
|
|
c = await dtools.get_container(web, range="1h")
|
|
assert c["host_name"] == host_name and c["status"] == "running"
|
|
assert c["cpu_pct_history"]["max"] == 50.0 and c["cpu_pct_history"]["points"] == 3
|
|
crashed = await dtools.get_container(job, host=host_name)
|
|
assert [e["event"] for e in crashed["events"]] == ["die"]
|
|
# A unique part of a long swarm task name is enough.
|
|
assert (await dtools.get_container(f"{svc}.1"))["swarm_service"] == svc
|
|
with pytest.raises(ValueError, match="list_containers"):
|
|
await dtools.get_container(f"nope-{tag}")
|
|
|
|
# ── search_container_logs ──────────────────────────────────────────
|
|
hit = await dtools.search_container_logs(container=web, text="100%", context=1)
|
|
assert [m["line"] for m in hit["matches"]] == ["ERROR: 100% of disk_used"]
|
|
assert hit["matches"][0]["before"] == ["connected to db"]
|
|
assert hit["matches"][0]["after"] == ["retrying"]
|
|
# % and _ match literally, not as wildcards.
|
|
assert (await dtools.search_container_logs(container=web, text="o_n"))["matches"] == []
|
|
page1 = await dtools.search_container_logs(container=web, limit=2)
|
|
assert page1["truncated"] and len(page1["matches"]) == 2
|
|
assert "characters omitted" in page1["matches"][0]["line"]
|
|
assert len(page1["matches"][0]["line"]) < 2100
|
|
page2 = await dtools.search_container_logs(container=web, limit=2, before=page1["next_before"])
|
|
assert [m["line"] for m in page2["matches"]] == ["ERROR: 100% of disk_used", "connected to db"]
|
|
errs = await dtools.search_container_logs(host=host_name, stream="stderr")
|
|
assert [m["container_name"] for m in errs["matches"]] == [web]
|
|
with pytest.raises(ValueError, match="context needs `container`"):
|
|
await dtools.search_container_logs(text="x", context=2)
|
|
|
|
# ── swarm, disk ────────────────────────────────────────────────────
|
|
sw = await dtools.list_swarm_services(limit=500)
|
|
s = next(x for x in sw["services"] if x["service_name"] == svc)
|
|
assert (s["state"], s["running"], s["desired"]) == ("degraded", 1, 3)
|
|
assert any(n["node_id"] == f"node-{tag}" and n["leader"] for n in sw["nodes"])
|
|
disk = (await dtools.get_docker_disk_usage(host=host_name))["hosts"]
|
|
assert disk[0]["images_reclaimable_bytes"] == 3000
|
|
|
|
# ── overview contributions ─────────────────────────────────────────
|
|
async with app.db_sessionmaker() as db:
|
|
found = {(f["kind"], f["subject"]) for f in await dtools.mcp_overview(db)}
|
|
agent_found = {(f["kind"], f["subject"]) for f in await atools.mcp_overview(db)}
|
|
assert ("container_unhealthy", db_) in found
|
|
assert ("container_exited_nonzero", job) in found
|
|
assert ("swarm_service_degraded", svc) in found
|
|
assert not any(subject == web for _, subject in found)
|
|
assert ("agent_never_reported", quiet_host) in agent_found
|
|
assert not any(subject == host_name for _, subject in agent_found)
|
|
|
|
# ── host agents ────────────────────────────────────────────────────
|
|
agents = {a["host_name"]: a for a in (await atools.list_agents())["agents"]}
|
|
assert agents[host_name]["state"] == "reporting" and agents[host_name]["agent_version"] == "1.5.0"
|
|
assert agents[quiet_host]["state"] == "pending"
|
|
|
|
# ── registration through the plugin hook, gated on enablement ─────
|
|
mcp = app.mcp_server
|
|
mcp.register_plugin("docker", docker)
|
|
mcp.register_plugin("host_agent", agent)
|
|
assert {"list_containers", "search_container_logs", "list_agents"} <= set(mcp.tool_roles)
|
|
assert mcp.tool_plugins["list_containers"] == "docker"
|
|
assert set(mcp.overview_sources) >= {"docker", "host_agent"}
|
|
|
|
asyncio.run(_go())
|