fix(docker): a restarted agent's re-sent log lines are stored once (#5809)

An agent keeps its log cursors in memory, so after a restart it seeds each
container from its last 50 lines, which Steward already holds; docker_logs
has no unique key, so every update or reboot doubled them. The ingest now
reads what it already has for each container over the batch's own span,
in one query, and skips those lines. A push of new lines overlaps nothing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-15 10:08:38 -04:00
co-authored by Claude Opus 5.5
parent a68990db70
commit 423493e2bf
2 changed files with 75 additions and 2 deletions
+49
View File
@@ -221,6 +221,55 @@ def test_persist_logs_stores_lines(app):
]
@_NEEDS_DB
def test_persist_logs_skips_lines_a_restarted_agent_sends_again(app):
"""#5809: a restarted agent has lost its cursors and re-sends each
container's last lines. Those are stored once; a new line, or another line
at the same instant, is still kept."""
from types import SimpleNamespace
from sqlalchemy import text
from steward.models.hosts import Host
persist = _persist_fn(app)
now = datetime.now(timezone.utc)
def line(container, at, text_, stream="stdout"):
return {"container": container, "stream": stream,
"ts": f"2026-01-01T00:00:0{at}+00:00", "line": text_}
first = [(now, [line("web", 1, "started"), line("web", 2, "listening"),
line("db", 1, "ready")])]
again = [(now, [line("web", 1, "started"), line("web", 2, "listening"),
line("web", 2, "worker 2 up"), line("web", 3, "GET /"),
line("db", 1, "ready"), line("db", 1, "ready", stream="stderr")])]
hid = str(uuid.uuid4())
host = SimpleNamespace(id=hid, name="loghost-again") # as below: no expiry across commits
async def _go():
async with app.db_sessionmaker() as s:
async with s.begin():
s.add(Host(id=hid, name="loghost-again", address="10.0.0.19"))
await s.flush()
await persist(s, host, [], None, None, first)
async with s.begin():
await persist(s, host, [], None, None, again)
rows = (await s.execute(text(
"SELECT container_name, stream, line FROM docker_logs "
"WHERE host_id = :h ORDER BY container_name, ts, stream, line"),
{"h": hid})).all()
return [tuple(r) for r in rows]
assert asyncio.run(_go()) == [
("db", "stderr", "ready"),
("db", "stdout", "ready"),
("web", "stdout", "started"),
("web", "stdout", "listening"),
("web", "stdout", "worker 2 up"),
("web", "stdout", "GET /"),
]
@_NEEDS_DB
def test_persist_logs_respects_toggle_and_exclude(app):
"""Server-side controls (Settings): an excluded container's lines are dropped