fix(docker): a container that crashes between two samples leaves a die event and its last log lines (#5049)
CI / lint (push) Successful in 2s
CI / integration (push) Successful in 46s
CI / unit (push) Successful in 1m10s
CI / publish (push) Successful in 8s

Agent 1.11.0 sends finished_at (State.FinishedAt) and reads a stopped
container's logs once per stop: since its cursor if it was being read, or
its last 200 lines when first seen already crashed (non-zero exit, stopped
in the last 10 min). Cursors and finals are forgotten once a container is no
longer listed.

Ingest gives a container first seen already exited non-zero a die event, and
dates die/stop events by finished_at when the agent sends it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-06 20:58:46 -04:00
co-authored by Claude Opus 5.5
parent 4978d28a51
commit 326a9cd254
4 changed files with 151 additions and 26 deletions
+12 -3
View File
@@ -40,6 +40,8 @@ def _derive_events(old_state: dict, new_containers: list) -> list:
start — a container transitions into running (or a genuinely new
container appears already running)
die — also for a new container first seen already exited
non-zero: a task that crashed between two samples (#5049)
restart — Docker restarted it (its restart count went up); said
instead of a start, since that is what happened
stop — running → not-running, or a running container is removed
@@ -64,10 +66,12 @@ def _derive_events(old_state: dict, new_containers: list) -> list:
old = old_state.get(name)
if old is None:
# Newly observed container — only a start is meaningful (we have no
# prior state to diff a death against).
# Newly observed container: a start, or, when it is already down
# with a non-zero exit, the death that happened between samples.
if new_running:
events.append((name, "start", c.get("image") or None))
elif new_status in _STOPPED_STATES and new_exit not in (None, 0):
events.append((name, "die", f"exit {new_exit}"))
continue
old_running = (old.get("status") or "").lower() == "running"
@@ -365,10 +369,15 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
for r in old_rows
}
if old_state:
# A stop is dated when Docker says it happened (agent ≥ 1.11), not when
# the sample that noticed it was taken.
finished = {c.get("name"): _parse_started_at(c.get("finished_at"))
for c in latest_containers}
for name, event, detail in _derive_events(old_state, latest_containers):
at = finished.get(name) if event in ("die", "stop") else None
session.add(DockerEvent(
host_id=host.id, container_name=name,
event=event, detail=detail, at=latest_at,
event=event, detail=detail, at=at or latest_at,
))
await mark("events")