fix(docker): a container that crashes between two samples leaves a die event and its last log lines (#5049)

Agent 1.11.0 sends finished_at (State.FinishedAt) and reads a stopped
container's logs once per stop: since its cursor if it was being read, or
its last 200 lines when first seen already crashed (non-zero exit, stopped
in the last 10 min). Cursors and finals are forgotten once a container is no
longer listed.

Ingest gives a container first seen already exited non-zero a die event, and
dates die/stop events by finished_at when the agent sends it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/agent.py

@@ -20,7 +20,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor, wait as wait_futures
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.10.0"
+AGENT_VERSION = "1.11.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -456,6 +456,8 @@ DOCKER_DF_INTERVAL_SECONDS = 600.0
 # kept per container in the agent's rate-state.
 DOCKER_LOG_BATCH_MAX_BYTES = 262144   # 256 KiB of log text per push, all containers
 DOCKER_LOG_FIRST_TAIL = 50            # lines seeded on a container's first interval
+DOCKER_LOG_FINAL_TAIL = 200           # lines read once from a container found already crashed
+DOCKER_LOG_FINAL_WINDOW_SECONDS = 600  # ... when it stopped this recently
 
 
 def _dechunk(body: bytes) -> bytes:
@@ -599,12 +601,12 @@ def _docker_inspect_fields(insp: dict):
     return health, restart_count, exit_code, oom_killed
 
 
-def _docker_started_at(insp: dict):
+def _docker_started_at(insp: dict, key: str = "StartedAt"):
     """State.StartedAt as ISO-8601, or None for a container never started
     (Docker's zero time) or an inspect that failed. When the container last
     started, not when it was created: a restarted container's uptime counts
-    from its restart."""
-    raw = (((insp or {}).get("State") or {}).get("StartedAt") or "")
+    from its restart. key="FinishedAt" reads when it last stopped."""
+    raw = (((insp or {}).get("State") or {}).get(key) or "")
     if not raw or raw.startswith("0001-"):
         return None
     ts = _parse_log_ts(raw)
@@ -665,6 +667,9 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
             pass
 
     started_at = _docker_started_at(insp)
+    # When it last stopped (a running container's is its previous stop, or
+    # None): dates a die event, and tells the log collector a stop is new.
+    finished_at = None if state == "running" else _docker_started_at(insp, "FinishedAt")
     record = {
         "name": name,
         "container_id": cid[:12],
@@ -676,6 +681,7 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
         "mem_pct": mem_pct,
         "ports": _docker_ports(c.get("Ports", [])),
         "started_at": started_at,
+        "finished_at": finished_at,
         "health": health,
         "restart_count": restart_count,
         "exit_code": exit_code,
@@ -872,39 +878,64 @@ def _parse_container_logs(raw: bytes) -> list:
 
 def collect_docker_logs(socket_path: str, containers: list, state: dict,
                         exclude=None,
-                        max_bytes: int = DOCKER_LOG_BATCH_MAX_BYTES) -> list:
-    """New container-log lines since the last interval, for running containers.
+                        max_bytes: int = DOCKER_LOG_BATCH_MAX_BYTES,
+                        now: float | None = None) -> list:
+    """New container-log lines since the last interval.
 
     Keeps a per-container since-cursor in `state["docker_log_cursors"]`
     (name → unix ts) so each interval fetches only what's new; a container's
-    first interval seeds from a short tail. The whole batch is capped at
-    `max_bytes` of log text — once exceeded a truncation marker is appended and
-    collection stops (logs must never balloon a push). `exclude` container names
-    are skipped entirely (a per-host bandwidth opt-out). Best-effort: a
-    per-container transport error just contributes nothing this interval.
+    first interval seeds from a short tail. A running container is read every
+    interval. A stopped one is read once per stop (`finished_at`, remembered in
+    `state["docker_log_finals"]`), so the lines it wrote on its way down reach
+    Steward (#5049): since its cursor if it was being read, or, for one first
+    seen already stopped with a non-zero exit in the last
+    DOCKER_LOG_FINAL_WINDOW_SECONDS, its last DOCKER_LOG_FINAL_TAIL lines. That
+    is how a task that crashes between two samples is explained by its own
+    output. Both maps forget a container once it is no longer listed.
+
+    The whole batch is capped at `max_bytes` of log text — once exceeded a
+    truncation marker is appended and collection stops (logs must never balloon
+    a push). `exclude` container names are skipped entirely (a per-host
+    bandwidth opt-out). Best-effort: a per-container transport error just
+    contributes nothing this interval.
     Returns [{"container", "stream", "ts", "line"}, …].
     """
     exclude_set = set(exclude or ())
     cursors = state.setdefault("docker_log_cursors", {})
-    running = {c.get("name") for c in (containers or [])
-               if c.get("name") and c.get("status") == "running"}
-    # Forget cursors for containers no longer running so state can't grow without
-    # bound over the agent's lifetime.
-    for gone in [k for k in cursors if k not in running]:
-        del cursors[gone]
+    finals = state.setdefault("docker_log_finals", {})
+    now = time.time() if now is None else now
+    listed = {c.get("name"): c for c in (containers or []) if c.get("name")}
+    # Forget containers no longer listed so state can't grow without bound
+    # over the agent's lifetime.
+    for book in (cursors, finals):
+        for gone in [k for k in book if k not in listed]:
+            del book[gone]
 
     out: list = []
     total = 0
     truncated = False
-    for c in (containers or []):
-        name = c.get("name")
-        if not name or name not in running or name in exclude_set:
+    for name, c in listed.items():
+        if name in exclude_set:
             continue
         prev = cursors.get(name)
         base = f"/containers/{name}/logs?stdout=1&stderr=1&timestamps=1"
-        # since accepts a fractional Unix ts; we still dedup the boundary line.
-        path = (f"{base}&tail={DOCKER_LOG_FIRST_TAIL}" if prev is None
-                else f"{base}&since={prev:.9f}")
+        if c.get("status") == "running":
+            # since accepts a fractional Unix ts; we still dedup the boundary line.
+            path = (f"{base}&tail={DOCKER_LOG_FIRST_TAIL}" if prev is None
+                    else f"{base}&since={prev:.9f}")
+        else:
+            stopped = c.get("finished_at")
+            if not stopped or finals.get(name) == stopped:
+                continue
+            if prev is not None:
+                path = f"{base}&since={prev:.9f}"
+            else:
+                at = _parse_log_ts(stopped)
+                if (c.get("exit_code") in (None, 0) or at is None
+                        or now - at.timestamp() > DOCKER_LOG_FINAL_WINDOW_SECONDS):
+                    continue
+                path = f"{base}&tail={DOCKER_LOG_FINAL_TAIL}"
+            finals[name] = stopped
         try:
             raw = _docker_request_raw(socket_path, path)
         except (OSError, ValueError):