fix(docker): a container that crashes between two samples leaves a die event and its last log lines (#5049)
Agent 1.11.0 sends finished_at (State.FinishedAt) and reads a stopped container's logs once per stop: since its cursor if it was being read, or its last 200 lines when first seen already crashed (non-zero exit, stopped in the last 10 min). Cursors and finals are forgotten once a container is no longer listed. Ingest gives a container first seen already exited non-zero a die event, and dates die/stop events by finished_at when the agent sends it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -634,6 +634,75 @@ def test_collect_docker_logs_forgets_gone_container_cursors(monkeypatch):
|
||||
assert "db" in state["docker_log_cursors"]
|
||||
|
||||
|
||||
def _final_raw(seen):
|
||||
def fake_raw(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
|
||||
seen.append(path)
|
||||
return (_frame(1, b"2026-10-06T12:00:00.000000000Z booting\n")
|
||||
+ _frame(2, b"2026-10-06T12:00:01.000000000Z fatal: no config\n"))
|
||||
return fake_raw
|
||||
|
||||
|
||||
def test_collect_docker_logs_reads_a_stopped_container_once(monkeypatch):
|
||||
# #5049: a container that was being read and has stopped is read one last
|
||||
# time since its cursor, then not again until it stops again.
|
||||
seen: list = []
|
||||
monkeypatch.setattr(a, "_docker_request_raw", _final_raw(seen))
|
||||
state = {"docker_log_cursors": {"web": 1791288000.5}} # 2026-10-06T12:00:00.5Z
|
||||
down = [{"name": "web", "status": "exited", "exit_code": 1,
|
||||
"finished_at": "2026-10-06T12:00:02+00:00"}]
|
||||
out = a.collect_docker_logs("/sock", down, state)
|
||||
assert [r["line"] for r in out] == ["fatal: no config"] # since the cursor
|
||||
assert "since=" in seen[0]
|
||||
assert a.collect_docker_logs("/sock", down, state) == [] # same stop: not again
|
||||
assert len(seen) == 1
|
||||
again = [{**down[0], "finished_at": "2026-10-06T12:05:00+00:00"}]
|
||||
a.collect_docker_logs("/sock", again, state) # a new stop is read
|
||||
assert len(seen) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exit_code,stopped_ago,read", [
|
||||
(1, 30, True), # crashed between two samples: its last lines
|
||||
(0, 30, False), # finished cleanly: nothing to explain
|
||||
(None, 30, False), # inspect failed: no claim
|
||||
(1, 3600, False), # an old crash already on the host when the agent came up
|
||||
])
|
||||
def test_collect_docker_logs_tails_a_container_first_seen_crashed(monkeypatch, exit_code,
|
||||
stopped_ago, read):
|
||||
seen: list = []
|
||||
monkeypatch.setattr(a, "_docker_request_raw", _final_raw(seen))
|
||||
stopped = 1791288000.0
|
||||
down = [{"name": "codex_db.1.x", "status": "exited", "exit_code": exit_code,
|
||||
"finished_at": "2026-10-06T12:00:00+00:00"}]
|
||||
out = a.collect_docker_logs("/sock", down, {}, now=stopped + stopped_ago)
|
||||
assert bool(out) is read
|
||||
if read:
|
||||
assert f"tail={a.DOCKER_LOG_FINAL_TAIL}" in seen[0]
|
||||
assert out[-1]["line"] == "fatal: no config"
|
||||
|
||||
|
||||
def test_collect_docker_logs_forgets_finals_of_unlisted_containers(monkeypatch):
|
||||
seen: list = []
|
||||
monkeypatch.setattr(a, "_docker_request_raw", _final_raw(seen))
|
||||
state: dict = {}
|
||||
down = [{"name": "job", "status": "exited", "exit_code": 2,
|
||||
"finished_at": "2026-10-06T12:00:00+00:00"}]
|
||||
a.collect_docker_logs("/sock", down, state, now=1791288010.0)
|
||||
assert "job" in state["docker_log_finals"]
|
||||
a.collect_docker_logs("/sock", [], state, now=1791288020.0)
|
||||
assert state["docker_log_finals"] == {} and state["docker_log_cursors"] == {}
|
||||
|
||||
|
||||
def test_finished_at_only_for_a_stopped_container(monkeypatch):
|
||||
insp = {"State": {"StartedAt": "2026-10-06T11:00:00Z", "FinishedAt": "2026-10-06T12:00:00Z",
|
||||
"ExitCode": 1}}
|
||||
monkeypatch.setattr(a, "_docker_request", lambda s, p, timeout=a.DOCKER_API_TIMEOUT: insp)
|
||||
rec = a._collect_one_container("/sock", {"Id": "abc", "Names": ["/job"], "State": "exited"})
|
||||
assert rec["finished_at"].startswith("2026-10-06T12:00:00")
|
||||
insp["State"]["FinishedAt"] = "0001-01-01T00:00:00Z" # never stopped
|
||||
rec = a._collect_one_container("/sock", {"Id": "abc", "Names": ["/job"], "State": "exited"})
|
||||
assert rec["finished_at"] is None
|
||||
|
||||
|
||||
def test_build_sample_includes_logs_when_present(monkeypatch):
|
||||
monkeypatch.setattr(a, "collect_docker",
|
||||
lambda _s: ("ok", None, [{"name": "web", "status": "running"}]))
|
||||
|
||||
Reference in New Issue
Block a user