fix(docker): a container that crashes between two samples leaves a die event and its last log lines (#5049)
Agent 1.11.0 sends finished_at (State.FinishedAt) and reads a stopped container's logs once per stop: since its cursor if it was being read, or its last 200 lines when first seen already crashed (non-zero exit, stopped in the last 10 min). Cursors and finals are forgotten once a container is no longer listed. Ingest gives a container first seen already exited non-zero a die event, and dates die/stop events by finished_at when the agent sends it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -40,6 +40,8 @@ def _derive_events(old_state: dict, new_containers: list) -> list:
|
||||
|
||||
start — a container transitions into running (or a genuinely new
|
||||
container appears already running)
|
||||
die — also for a new container first seen already exited
|
||||
non-zero: a task that crashed between two samples (#5049)
|
||||
restart — Docker restarted it (its restart count went up); said
|
||||
instead of a start, since that is what happened
|
||||
stop — running → not-running, or a running container is removed
|
||||
@@ -64,10 +66,12 @@ def _derive_events(old_state: dict, new_containers: list) -> list:
|
||||
old = old_state.get(name)
|
||||
|
||||
if old is None:
|
||||
# Newly observed container — only a start is meaningful (we have no
|
||||
# prior state to diff a death against).
|
||||
# Newly observed container: a start, or, when it is already down
|
||||
# with a non-zero exit, the death that happened between samples.
|
||||
if new_running:
|
||||
events.append((name, "start", c.get("image") or None))
|
||||
elif new_status in _STOPPED_STATES and new_exit not in (None, 0):
|
||||
events.append((name, "die", f"exit {new_exit}"))
|
||||
continue
|
||||
|
||||
old_running = (old.get("status") or "").lower() == "running"
|
||||
@@ -365,10 +369,15 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
for r in old_rows
|
||||
}
|
||||
if old_state:
|
||||
# A stop is dated when Docker says it happened (agent ≥ 1.11), not when
|
||||
# the sample that noticed it was taken.
|
||||
finished = {c.get("name"): _parse_started_at(c.get("finished_at"))
|
||||
for c in latest_containers}
|
||||
for name, event, detail in _derive_events(old_state, latest_containers):
|
||||
at = finished.get(name) if event in ("die", "stop") else None
|
||||
session.add(DockerEvent(
|
||||
host_id=host.id, container_name=name,
|
||||
event=event, detail=detail, at=latest_at,
|
||||
event=event, detail=detail, at=at or latest_at,
|
||||
))
|
||||
await mark("events")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user