diff --git a/plugins/docker/mcp_tools.py b/plugins/docker/mcp_tools.py index 9f45276..494a91f 100644 --- a/plugins/docker/mcp_tools.py +++ b/plugins/docker/mcp_tools.py @@ -7,6 +7,7 @@ the plugin's pages use. from __future__ import annotations import json +from dataclasses import dataclass from datetime import datetime, timedelta from sqlalchemy import select @@ -294,31 +295,69 @@ async def list_containers(host: str = "", status: str = "", problem: str = "", more="Pass a larger `limit` (max 500), or filter by `host`, `status` or `problem`.") -async def _pick_container(db, container: str, host: str): +@dataclass +class RemovedContainer: + """A container Docker has removed, known only by its retained logs and events.""" + host_id: str + name: str + last_seen: datetime + + +_REMOVED_HINT = ("It may have been removed: Swarm replaces a dead task with a new container under " + "a new name. list_containers(status=\"exited\") lists stopped containers, and " + "search_container_logs(host=...) searches a whole host.") + + +async def _pick_container(db, container: str, host: str, *, removed: bool = False): + """The container named `container`: a live one, else (with `removed`) one + Docker has removed whose logs or events Steward still holds, returned as a + RemovedContainer. Exact names beat partial ones, and live beats removed.""" containers = await docker_svc.list_containers(db) + host_id = None if host: hosts = list((await db.execute(select(Host))).scalars()) - h = pick(host, hosts, kind="host", list_tool="list_hosts") - containers = [c for c in containers if c.host_id == h.id] + host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id + containers = [c for c in containers if c.host_id == host_id] want = (container or "").strip() if not want: raise ValueError("pass a container name; list_containers lists them.") + live = {(c.host_id, c.name) for c in containers} + + def _one(found, what): + if len(found) > 1: + raise ValueError(f"{len(found)} hosts have {what} named {want!r}; pass `host` " + f"(one of: {', '.join(sorted(f[0] for f in found))}).") + return found[0] if found else None + same = [c for c in containers if c.name == want] or \ [c for c in containers if c.name.lower() == want.lower()] - if len(same) == 1: - return same[0] if same: - raise ValueError(f"{len(same)} hosts run a container named {want!r}; pass `host` " - f"(one of: {', '.join(sorted(c.host_id for c in same))}).") + return _one([(c.host_id, c) for c in same], "a container")[1] + if removed: + gone = [r for r in await docker_svc.retained_names(db, want, exact=True, host_id=host_id) + if (r[0], r[1]) not in live] + if gone: + return RemovedContainer(*_one(gone, "a removed container")) # One more chance: a unique container whose name contains it (swarm task # names look like "stack_svc.1.k2j…"). partial = [c for c in containers if want.lower() in c.name.lower()] if len(partial) == 1: return partial[0] - hint = (f" Containers whose names contain it: {', '.join(sorted(c.name for c in partial)[:10])}." - if partial else "") + gone = [] + if removed and not partial: + gone = [r for r in await docker_svc.retained_names(db, want, exact=False, host_id=host_id) + if (r[0], r[1]) not in live] + if len(gone) == 1: + return RemovedContainer(*gone[0]) + names = sorted(c.name for c in partial)[:10] + [f"{r[1]} (removed)" for r in gone[:10]] + hint = f" Containers whose names contain it: {', '.join(names)}." if names else "" raise ValueError(f"no container named {want!r}{' on that host' if host else ''}.{hint} " - "list_containers lists them.") + f"{'' if names else _REMOVED_HINT + ' '}list_containers lists them.") + + +def _removed_note(c: RemovedContainer) -> str: + return (f"{c.name} no longer exists: Docker removed it (Swarm replaces a failed task with a new " + f"container). This is what Steward retained; it was last seen {iso(c.last_seen)}.") async def get_container(container: str, host: str = "", range: str = "24h", events: int = 20) -> dict: @@ -333,12 +372,20 @@ async def get_container(container: str, host: str = "", range: str = "24h", even "7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values. events: how many lifecycle events (start, stop, die, oom, health_change) to return, newest first (default 20, max 100). + + A container Docker has since removed (a dead Swarm task, a `docker rm`) + still resolves by its exact name, or by a part of it that matches one + removed container, while Steward retains its logs or events: the reply + has `removed: true`, `last_seen_at`, its history and its events, and no + live state. A live container has `removed: false`. """ since, key = lookback(range) events = clamp_limit(events, 20, 100) now = now_utc() async with db_session() as db: - c = await _pick_container(db, container, host) + c = await _pick_container(db, container, host, removed=True) + if isinstance(c, RemovedContainer): + return await _removed_container(db, c, since, key, events, now) gone = (c.host_id, c.name) in await _replaced(db, await docker_svc.list_containers(db)) failures = (await docker_svc.recent_failures_by_container( db, now - timedelta(hours=24))).get((c.host_id, c.name), []) @@ -357,6 +404,7 @@ async def get_container(container: str, host: str = "", range: str = "24h", even began = problem_since(problems, failures, hour_ago, sick.get((c.host_id, c.name))) return { **container_row(c, host_names, problems, now, replaced=gone, since=began), + "removed": False, "container_id": c.container_id, "ports": ports, "net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes, @@ -364,12 +412,35 @@ async def get_container(container: str, host: str = "", range: str = "24h", even "range": key, "cpu_pct_history": summarize(cpu), "mem_pct_history": summarize(mem), - "events": [{"event": e.event, "detail": e.detail, "at": iso(e.at), "age_s": age_s(e.at, now)} - for e in evs], + "events": _event_rows(evs, now), "next": f'search_container_logs(container="{c.name}", host="{c.host_id}")', } +def _event_rows(evs, now: datetime) -> list[dict]: + return [{"event": e.event, "detail": e.detail, "at": iso(e.at), "age_s": age_s(e.at, now)} + for e in evs] + + +async def _removed_container(db, c: RemovedContainer, since, key: str, events: int, + now: datetime) -> dict: + """get_container for a container Docker has removed: what Steward kept.""" + host_names = await _host_names(db) + cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since) + mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since) + evs = await docker_svc.container_events(db, c.host_id, c.name, events) + return { + "container_name": c.name, "host_id": c.host_id, "host_name": host_names.get(c.host_id), + "removed": True, "note": _removed_note(c), + "last_seen_at": iso(c.last_seen), "last_seen_age_s": age_s(c.last_seen, now), + "range": key, + "cpu_pct_history": summarize(cpu), + "mem_pct_history": summarize(mem), + "events": _event_rows(evs, now), + "next": f'search_container_logs(container="{c.name}", host="{c.host_id}", range="24h")', + } + + async def list_swarm_services(limit: int = 100) -> dict: """Docker Swarm services with their replicas per node, and the swarm's nodes. @@ -455,7 +526,9 @@ async def search_container_logs(container: str = "", host: str = "", text: str = """Search the container log lines Steward has retained, newest first. container: one container's name (as list_containers shows); empty searches - every container (narrow with `host`). + every container (narrow with `host`). A container Docker has removed + still works while its lines are retained: the reply then has + `container_removed: true` and a `note` saying when it was last seen. host: a host's name or id; empty for all hosts. text: case-insensitive substring to match; empty matches every line. range: how far back — "1h" (default), "6h", "24h", "7d" or "30d". Logs are @@ -484,10 +557,11 @@ async def search_container_logs(container: str = "", host: str = "", text: str = except ValueError as exc: raise ValueError(f"before {before!r} is not an ISO-8601 time; pass a response's next_before.") from exc async with db_session() as db: - host_id = name = None + host_id = name = removed = None if container: - c = await _pick_container(db, container, host) + c = await _pick_container(db, container, host, removed=True) host_id, name = c.host_id, c.name + removed = c if isinstance(c, RemovedContainer) else None elif host: hosts = list((await db.execute(select(Host))).scalars()) host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id @@ -506,6 +580,11 @@ async def search_container_logs(container: str = "", host: str = "", text: str = m["after"] = [cut_middle(x.line) for x in post] matches.append(m) out = {"range": key, "matches": matches, "returned": len(matches), "truncated": more} + if removed is not None: + out["container_removed"] = True + out["note"] = _removed_note(removed) + if removed.last_seen < since: + out["note"] += " That is before this range; pass a longer `range`." if more: # Full precision: `at` is cut to the second for reading, and a cursor # cut the same way would skip lines logged earlier in that second. diff --git a/plugins/docker/service.py b/plugins/docker/service.py index 0b938bb..531e294 100644 --- a/plugins/docker/service.py +++ b/plugins/docker/service.py @@ -218,6 +218,34 @@ async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list] return out +async def retained_names(db, want: str, *, exact: bool, host_id: str | None = None, + limit: int = 20) -> list[tuple[str, str, datetime]]: + """(host_id, container_name, last seen) of containers Steward still holds + logs or events for, newest first — including ones Docker has removed. + + exact: the name itself, from both logs and events. Otherwise names that + contain `want` (case-insensitive), from events only: a substring match + over every retained log line's name would scan the log table, and a + container Steward watched die has a die or stop event. + """ + seen: dict[tuple[str, str], datetime] = {} + sources = ((DockerLog, DockerLog.ts), (DockerEvent, DockerEvent.at)) if exact \ + else ((DockerEvent, DockerEvent.at),) + for model, at in sources: + match = (model.container_name == want if exact + else model.container_name.ilike(f"%{escape_like(want)}%", escape="\\")) + stmt = (select(model.host_id, model.container_name, func.max(at)) + .where(match).group_by(model.host_id, model.container_name)) + if host_id: + stmt = stmt.where(model.host_id == host_id) + for hid, name, last in (await db.execute(stmt)).all(): + key = (hid, name) + if key not in seen or last > seen[key]: + seen[key] = last + rows = sorted(((h, n, t) for (h, n), t in seen.items()), key=lambda r: r[2], reverse=True) + return rows[:limit] + + async def unhealthy_since(db, keys) -> dict[tuple[str, str], datetime]: """{(host_id, container_name): when it last turned unhealthy} for the given containers, from their health_change events; absent when none is retained.""" diff --git a/tests/integration/test_mcp_plugin_tools.py b/tests/integration/test_mcp_plugin_tools.py index 09125e5..c1e6b85 100644 --- a/tests/integration/test_mcp_plugin_tools.py +++ b/tests/integration/test_mcp_plugin_tools.py @@ -59,6 +59,7 @@ def test_docker_and_agent_tools_over_seeded_rows(app): host_name, quiet_host = f"dock-{tag}", f"quiet-{tag}" web, db_, job, svc = f"web-{tag}", f"db-{tag}", f"job-{tag}", f"svc-{tag}" replaced = f"{svc}.1.old" + removed = f"{svc}.2.gone{tag}" # a dead task Docker removed: logs and events only def _ago(minutes): return (now - timedelta(minutes=minutes)).isoformat().replace("+00:00", "Z") @@ -95,6 +96,14 @@ def test_docker_and_agent_tools_over_seeded_rows(app): "INSERT INTO docker_events (id, host_id, container_name, event, detail, at) " "VALUES (:id, :h, :n, 'die', 'exit 1', :at)"), {"id": str(uuid.uuid4()), "h": hid, "n": name, "at": now - timedelta(minutes=10)}) + await s.execute(text( + "INSERT INTO docker_events (id, host_id, container_name, event, detail, at) " + "VALUES (:id, :h, :n, 'die', 'exit 1', :at)"), + {"id": str(uuid.uuid4()), "h": hid, "n": removed, "at": now - timedelta(minutes=12)}) + await s.execute(text( + "INSERT INTO docker_logs (id, host_id, container_name, ts, stream, line) " + "VALUES (:id, :h, :n, :ts, 'stderr', 'FATAL: cannot open database')"), + {"id": str(uuid.uuid4()), "h": hid, "n": removed, "ts": now - timedelta(minutes=13)}) for i, (stream, line) in enumerate(( ("stdout", "starting up"), ("stdout", "connected to db"), ("stderr", "ERROR: 100% of disk_used"), ("stdout", "retrying"), @@ -194,6 +203,19 @@ def test_docker_and_agent_tools_over_seeded_rows(app): with pytest.raises(ValueError, match="context needs `container`"): await dtools.search_container_logs(text="x", context=2) + # ── a container Docker removed is still reachable by name ───────── + gone = await dtools.get_container(removed) + assert gone["removed"] is True and gone["host_name"] == host_name + assert [e["event"] for e in gone["events"]] == ["die"] and "no longer exists" in gone["note"] + assert (await dtools.get_container(web))["removed"] is False + lines = await dtools.search_container_logs(container=removed, range="1h") + assert lines["container_removed"] is True + assert [m["line"] for m in lines["matches"]] == ["FATAL: cannot open database"] + # A unique part of its name finds it through its events. + assert (await dtools.get_container(f"2.gone{tag}"))["container_name"] == removed + with pytest.raises(ValueError, match="may have been removed"): + await dtools.get_container(f"nothing-{tag}") + # ── swarm, disk ──────────────────────────────────────────────────── sw = await dtools.list_swarm_services(limit=500) s = next(x for x in sw["services"] if x["service_name"] == svc)