feat(docker): a removed container's logs and events stay reachable by name

#5010 "Logs of a removed container cannot be reached by name"

- get_container and search_container_logs fall back to the names Steward
  still holds logs or events for when no live container matches: the exact
  name from logs and events, or a part of it that matches one removed
  container's events.
- get_container answers with removed: true, last_seen_at, its history and
  events; search_container_logs marks container_removed and notes when it
  was last seen, and says when that is older than the range.
- A miss now says the container may have been removed, that Swarm replaces a
  dead task under a new name, and where to look instead.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/mcp_tools.py

@@ -7,6 +7,7 @@ the plugin's pages use.
 from __future__ import annotations
 
 import json
+from dataclasses import dataclass
 from datetime import datetime, timedelta
 
 from sqlalchemy import select
@@ -294,31 +295,69 @@ async def list_containers(host: str = "", status: str = "", problem: str = "",
                    more="Pass a larger `limit` (max 500), or filter by `host`, `status` or `problem`.")
 
 
-async def _pick_container(db, container: str, host: str):
+@dataclass
+class RemovedContainer:
+    """A container Docker has removed, known only by its retained logs and events."""
+    host_id: str
+    name: str
+    last_seen: datetime
+
+
+_REMOVED_HINT = ("It may have been removed: Swarm replaces a dead task with a new container under "
+                 "a new name. list_containers(status=\"exited\") lists stopped containers, and "
+                 "search_container_logs(host=...) searches a whole host.")
+
+
+async def _pick_container(db, container: str, host: str, *, removed: bool = False):
+    """The container named `container`: a live one, else (with `removed`) one
+    Docker has removed whose logs or events Steward still holds, returned as a
+    RemovedContainer. Exact names beat partial ones, and live beats removed."""
     containers = await docker_svc.list_containers(db)
+    host_id = None
     if host:
         hosts = list((await db.execute(select(Host))).scalars())
-        h = pick(host, hosts, kind="host", list_tool="list_hosts")
-        containers = [c for c in containers if c.host_id == h.id]
+        host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
+        containers = [c for c in containers if c.host_id == host_id]
     want = (container or "").strip()
     if not want:
         raise ValueError("pass a container name; list_containers lists them.")
+    live = {(c.host_id, c.name) for c in containers}
+
+    def _one(found, what):
+        if len(found) > 1:
+            raise ValueError(f"{len(found)} hosts have {what} named {want!r}; pass `host` "
+                             f"(one of: {', '.join(sorted(f[0] for f in found))}).")
+        return found[0] if found else None
+
     same = [c for c in containers if c.name == want] or \
            [c for c in containers if c.name.lower() == want.lower()]
-    if len(same) == 1:
-        return same[0]
     if same:
-        raise ValueError(f"{len(same)} hosts run a container named {want!r}; pass `host` "
-                         f"(one of: {', '.join(sorted(c.host_id for c in same))}).")
+        return _one([(c.host_id, c) for c in same], "a container")[1]
+    if removed:
+        gone = [r for r in await docker_svc.retained_names(db, want, exact=True, host_id=host_id)
+                if (r[0], r[1]) not in live]
+        if gone:
+            return RemovedContainer(*_one(gone, "a removed container"))
     # One more chance: a unique container whose name contains it (swarm task
     # names look like "stack_svc.1.k2j…").
     partial = [c for c in containers if want.lower() in c.name.lower()]
     if len(partial) == 1:
         return partial[0]
-    hint = (f" Containers whose names contain it: {', '.join(sorted(c.name for c in partial)[:10])}."
-            if partial else "")
+    gone = []
+    if removed and not partial:
+        gone = [r for r in await docker_svc.retained_names(db, want, exact=False, host_id=host_id)
+                if (r[0], r[1]) not in live]
+        if len(gone) == 1:
+            return RemovedContainer(*gone[0])
+    names = sorted(c.name for c in partial)[:10] + [f"{r[1]} (removed)" for r in gone[:10]]
+    hint = f" Containers whose names contain it: {', '.join(names)}." if names else ""
     raise ValueError(f"no container named {want!r}{' on that host' if host else ''}.{hint} "
-                     "list_containers lists them.")
+                     f"{'' if names else _REMOVED_HINT + ' '}list_containers lists them.")
+
+
+def _removed_note(c: RemovedContainer) -> str:
+    return (f"{c.name} no longer exists: Docker removed it (Swarm replaces a failed task with a new "
+            f"container). This is what Steward retained; it was last seen {iso(c.last_seen)}.")
 
 
 async def get_container(container: str, host: str = "", range: str = "24h", events: int = 20) -> dict:
@@ -333,12 +372,20 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
       "7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values.
     events: how many lifecycle events (start, stop, die, oom, health_change)
       to return, newest first (default 20, max 100).
+
+    A container Docker has since removed (a dead Swarm task, a `docker rm`)
+    still resolves by its exact name, or by a part of it that matches one
+    removed container, while Steward retains its logs or events: the reply
+    has `removed: true`, `last_seen_at`, its history and its events, and no
+    live state. A live container has `removed: false`.
     """
     since, key = lookback(range)
     events = clamp_limit(events, 20, 100)
     now = now_utc()
     async with db_session() as db:
-        c = await _pick_container(db, container, host)
+        c = await _pick_container(db, container, host, removed=True)
+        if isinstance(c, RemovedContainer):
+            return await _removed_container(db, c, since, key, events, now)
         gone = (c.host_id, c.name) in await _replaced(db, await docker_svc.list_containers(db))
         failures = (await docker_svc.recent_failures_by_container(
             db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
@@ -357,6 +404,7 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
     began = problem_since(problems, failures, hour_ago, sick.get((c.host_id, c.name)))
     return {
         **container_row(c, host_names, problems, now, replaced=gone, since=began),
+        "removed": False,
         "container_id": c.container_id,
         "ports": ports,
         "net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
@@ -364,12 +412,35 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
         "range": key,
         "cpu_pct_history": summarize(cpu),
         "mem_pct_history": summarize(mem),
-        "events": [{"event": e.event, "detail": e.detail, "at": iso(e.at), "age_s": age_s(e.at, now)}
-                   for e in evs],
+        "events": _event_rows(evs, now),
         "next": f'search_container_logs(container="{c.name}", host="{c.host_id}")',
     }
 
 
+def _event_rows(evs, now: datetime) -> list[dict]:
+    return [{"event": e.event, "detail": e.detail, "at": iso(e.at), "age_s": age_s(e.at, now)}
+            for e in evs]
+
+
+async def _removed_container(db, c: RemovedContainer, since, key: str, events: int,
+                             now: datetime) -> dict:
+    """get_container for a container Docker has removed: what Steward kept."""
+    host_names = await _host_names(db)
+    cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
+    mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
+    evs = await docker_svc.container_events(db, c.host_id, c.name, events)
+    return {
+        "container_name": c.name, "host_id": c.host_id, "host_name": host_names.get(c.host_id),
+        "removed": True, "note": _removed_note(c),
+        "last_seen_at": iso(c.last_seen), "last_seen_age_s": age_s(c.last_seen, now),
+        "range": key,
+        "cpu_pct_history": summarize(cpu),
+        "mem_pct_history": summarize(mem),
+        "events": _event_rows(evs, now),
+        "next": f'search_container_logs(container="{c.name}", host="{c.host_id}", range="24h")',
+    }
+
+
 async def list_swarm_services(limit: int = 100) -> dict:
     """Docker Swarm services with their replicas per node, and the swarm's nodes.
 
@@ -455,7 +526,9 @@ async def search_container_logs(container: str = "", host: str = "", text: str =
     """Search the container log lines Steward has retained, newest first.
 
     container: one container's name (as list_containers shows); empty searches
-      every container (narrow with `host`).
+      every container (narrow with `host`). A container Docker has removed
+      still works while its lines are retained: the reply then has
+      `container_removed: true` and a `note` saying when it was last seen.
     host: a host's name or id; empty for all hosts.
     text: case-insensitive substring to match; empty matches every line.
     range: how far back — "1h" (default), "6h", "24h", "7d" or "30d". Logs are
@@ -484,10 +557,11 @@ async def search_container_logs(container: str = "", host: str = "", text: str =
         except ValueError as exc:
             raise ValueError(f"before {before!r} is not an ISO-8601 time; pass a response's next_before.") from exc
     async with db_session() as db:
-        host_id = name = None
+        host_id = name = removed = None
         if container:
-            c = await _pick_container(db, container, host)
+            c = await _pick_container(db, container, host, removed=True)
             host_id, name = c.host_id, c.name
+            removed = c if isinstance(c, RemovedContainer) else None
         elif host:
             hosts = list((await db.execute(select(Host))).scalars())
             host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
@@ -506,6 +580,11 @@ async def search_container_logs(container: str = "", host: str = "", text: str =
                 m["after"] = [cut_middle(x.line) for x in post]
             matches.append(m)
     out = {"range": key, "matches": matches, "returned": len(matches), "truncated": more}
+    if removed is not None:
+        out["container_removed"] = True
+        out["note"] = _removed_note(removed)
+        if removed.last_seen < since:
+            out["note"] += " That is before this range; pass a longer `range`."
     if more:
         # Full precision: `at` is cut to the second for reading, and a cursor
         # cut the same way would skip lines logged earlier in that second.

plugins/docker/service.py

@@ -218,6 +218,34 @@ async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]
     return out
 
 
+async def retained_names(db, want: str, *, exact: bool, host_id: str | None = None,
+                         limit: int = 20) -> list[tuple[str, str, datetime]]:
+    """(host_id, container_name, last seen) of containers Steward still holds
+    logs or events for, newest first — including ones Docker has removed.
+
+    exact: the name itself, from both logs and events. Otherwise names that
+    contain `want` (case-insensitive), from events only: a substring match
+    over every retained log line's name would scan the log table, and a
+    container Steward watched die has a die or stop event.
+    """
+    seen: dict[tuple[str, str], datetime] = {}
+    sources = ((DockerLog, DockerLog.ts), (DockerEvent, DockerEvent.at)) if exact \
+        else ((DockerEvent, DockerEvent.at),)
+    for model, at in sources:
+        match = (model.container_name == want if exact
+                 else model.container_name.ilike(f"%{escape_like(want)}%", escape="\\"))
+        stmt = (select(model.host_id, model.container_name, func.max(at))
+                .where(match).group_by(model.host_id, model.container_name))
+        if host_id:
+            stmt = stmt.where(model.host_id == host_id)
+        for hid, name, last in (await db.execute(stmt)).all():
+            key = (hid, name)
+            if key not in seen or last > seen[key]:
+                seen[key] = last
+    rows = sorted(((h, n, t) for (h, n), t in seen.items()), key=lambda r: r[2], reverse=True)
+    return rows[:limit]
+
+
 async def unhealthy_since(db, keys) -> dict[tuple[str, str], datetime]:
     """{(host_id, container_name): when it last turned unhealthy} for the given
     containers, from their health_change events; absent when none is retained."""

tests/integration/test_mcp_plugin_tools.py

@@ -59,6 +59,7 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
     host_name, quiet_host = f"dock-{tag}", f"quiet-{tag}"
     web, db_, job, svc = f"web-{tag}", f"db-{tag}", f"job-{tag}", f"svc-{tag}"
     replaced = f"{svc}.1.old"
+    removed = f"{svc}.2.gone{tag}"  # a dead task Docker removed: logs and events only
 
     def _ago(minutes):
         return (now - timedelta(minutes=minutes)).isoformat().replace("+00:00", "Z")
@@ -95,6 +96,14 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
                         "INSERT INTO docker_events (id, host_id, container_name, event, detail, at) "
                         "VALUES (:id, :h, :n, 'die', 'exit 1', :at)"),
                         {"id": str(uuid.uuid4()), "h": hid, "n": name, "at": now - timedelta(minutes=10)})
+                await s.execute(text(
+                    "INSERT INTO docker_events (id, host_id, container_name, event, detail, at) "
+                    "VALUES (:id, :h, :n, 'die', 'exit 1', :at)"),
+                    {"id": str(uuid.uuid4()), "h": hid, "n": removed, "at": now - timedelta(minutes=12)})
+                await s.execute(text(
+                    "INSERT INTO docker_logs (id, host_id, container_name, ts, stream, line) "
+                    "VALUES (:id, :h, :n, :ts, 'stderr', 'FATAL: cannot open database')"),
+                    {"id": str(uuid.uuid4()), "h": hid, "n": removed, "ts": now - timedelta(minutes=13)})
                 for i, (stream, line) in enumerate((
                         ("stdout", "starting up"), ("stdout", "connected to db"),
                         ("stderr", "ERROR: 100% of disk_used"), ("stdout", "retrying"),
@@ -194,6 +203,19 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
         with pytest.raises(ValueError, match="context needs `container`"):
             await dtools.search_container_logs(text="x", context=2)
 
+        # ── a container Docker removed is still reachable by name ─────────
+        gone = await dtools.get_container(removed)
+        assert gone["removed"] is True and gone["host_name"] == host_name
+        assert [e["event"] for e in gone["events"]] == ["die"] and "no longer exists" in gone["note"]
+        assert (await dtools.get_container(web))["removed"] is False
+        lines = await dtools.search_container_logs(container=removed, range="1h")
+        assert lines["container_removed"] is True
+        assert [m["line"] for m in lines["matches"]] == ["FATAL: cannot open database"]
+        # A unique part of its name finds it through its events.
+        assert (await dtools.get_container(f"2.gone{tag}"))["container_name"] == removed
+        with pytest.raises(ValueError, match="may have been removed"):
+            await dtools.get_container(f"nothing-{tag}")
+
         # ── swarm, disk ────────────────────────────────────────────────────
         sw = await dtools.list_swarm_services(limit=500)
         s = next(x for x in sw["services"] if x["service_name"] == svc)