fix(docker): a container that crashes between two samples leaves a die event and its last log lines (#5049)
CI / lint (push) Successful in 2s
CI / integration (push) Successful in 46s
CI / unit (push) Successful in 1m10s
CI / publish (push) Successful in 8s

Agent 1.11.0 sends finished_at (State.FinishedAt) and reads a stopped
container's logs once per stop: since its cursor if it was being read, or
its last 200 lines when first seen already crashed (non-zero exit, stopped
in the last 10 min). Cursors and finals are forgotten once a container is no
longer listed.

Ingest gives a container first seen already exited non-zero a die event, and
dates die/stop events by finished_at when the agent sends it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-06 20:58:46 -04:00
co-authored by Claude Opus 5.5
parent 4978d28a51
commit 326a9cd254
4 changed files with 151 additions and 26 deletions
@@ -634,6 +634,75 @@ def test_collect_docker_logs_forgets_gone_container_cursors(monkeypatch):
assert "db" in state["docker_log_cursors"]
def _final_raw(seen):
def fake_raw(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
seen.append(path)
return (_frame(1, b"2026-10-06T12:00:00.000000000Z booting\n")
+ _frame(2, b"2026-10-06T12:00:01.000000000Z fatal: no config\n"))
return fake_raw
def test_collect_docker_logs_reads_a_stopped_container_once(monkeypatch):
# #5049: a container that was being read and has stopped is read one last
# time since its cursor, then not again until it stops again.
seen: list = []
monkeypatch.setattr(a, "_docker_request_raw", _final_raw(seen))
state = {"docker_log_cursors": {"web": 1791288000.5}} # 2026-10-06T12:00:00.5Z
down = [{"name": "web", "status": "exited", "exit_code": 1,
"finished_at": "2026-10-06T12:00:02+00:00"}]
out = a.collect_docker_logs("/sock", down, state)
assert [r["line"] for r in out] == ["fatal: no config"] # since the cursor
assert "since=" in seen[0]
assert a.collect_docker_logs("/sock", down, state) == [] # same stop: not again
assert len(seen) == 1
again = [{**down[0], "finished_at": "2026-10-06T12:05:00+00:00"}]
a.collect_docker_logs("/sock", again, state) # a new stop is read
assert len(seen) == 2
@pytest.mark.parametrize("exit_code,stopped_ago,read", [
(1, 30, True), # crashed between two samples: its last lines
(0, 30, False), # finished cleanly: nothing to explain
(None, 30, False), # inspect failed: no claim
(1, 3600, False), # an old crash already on the host when the agent came up
])
def test_collect_docker_logs_tails_a_container_first_seen_crashed(monkeypatch, exit_code,
stopped_ago, read):
seen: list = []
monkeypatch.setattr(a, "_docker_request_raw", _final_raw(seen))
stopped = 1791288000.0
down = [{"name": "codex_db.1.x", "status": "exited", "exit_code": exit_code,
"finished_at": "2026-10-06T12:00:00+00:00"}]
out = a.collect_docker_logs("/sock", down, {}, now=stopped + stopped_ago)
assert bool(out) is read
if read:
assert f"tail={a.DOCKER_LOG_FINAL_TAIL}" in seen[0]
assert out[-1]["line"] == "fatal: no config"
def test_collect_docker_logs_forgets_finals_of_unlisted_containers(monkeypatch):
seen: list = []
monkeypatch.setattr(a, "_docker_request_raw", _final_raw(seen))
state: dict = {}
down = [{"name": "job", "status": "exited", "exit_code": 2,
"finished_at": "2026-10-06T12:00:00+00:00"}]
a.collect_docker_logs("/sock", down, state, now=1791288010.0)
assert "job" in state["docker_log_finals"]
a.collect_docker_logs("/sock", [], state, now=1791288020.0)
assert state["docker_log_finals"] == {} and state["docker_log_cursors"] == {}
def test_finished_at_only_for_a_stopped_container(monkeypatch):
insp = {"State": {"StartedAt": "2026-10-06T11:00:00Z", "FinishedAt": "2026-10-06T12:00:00Z",
"ExitCode": 1}}
monkeypatch.setattr(a, "_docker_request", lambda s, p, timeout=a.DOCKER_API_TIMEOUT: insp)
rec = a._collect_one_container("/sock", {"Id": "abc", "Names": ["/job"], "State": "exited"})
assert rec["finished_at"].startswith("2026-10-06T12:00:00")
insp["State"]["FinishedAt"] = "0001-01-01T00:00:00Z" # never stopped
rec = a._collect_one_container("/sock", {"Id": "abc", "Names": ["/job"], "State": "exited"})
assert rec["finished_at"] is None
def test_build_sample_includes_logs_when_present(monkeypatch):
monkeypatch.setattr(a, "collect_docker",
lambda _s: ("ok", None, [{"name": "web", "status": "running"}]))