feat(docker): per-container size+age log ring rotation [M79 step 4]
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 41s
CI / integration (push) Successful in 2m26s
CI / publish (push) Successful in 59s

Bound docker_logs growth in the periodic cleanup task (same architecture as the
metrics/events retention). run_docker_retention gains logs_retention_days +
logs_max_bytes_per_container: it prunes lines past the age window, then keeps
only the newest ~cap bytes per (host, container) via a window-function ring
(exclusive-prefix sum, so the newest line always survives even if it alone
exceeds the cap). Containers rotate independently.

- settings DEFAULTS: docker.logs.enabled/exclude/retention_days(3)/
  max_bytes_per_container(5MB) — operator preference: ~3 days / ~5 MB
- cleanup.py reads the two windows fresh each run (rule 25, no restart)
- integration rotate test: age prune + per-container byte cap + isolation

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CAGR73DUowdVFVvYzLXC5C
This commit is contained in:
2026-07-19 18:51:12 -04:00
parent a8de3570fe
commit 07a841d91e
4 changed files with 107 additions and 2 deletions
+61
View File
@@ -511,6 +511,67 @@ def test_retention_rollup_and_prune(app):
assert counts["events_pruned"] == 1 and counts["rollup_pruned"] == 1
@_NEEDS_DB
def test_retention_logs_ring(app):
"""docker_logs ring (m79): lines past the age window are pruned; within it,
each container keeps only the newest ~cap bytes; containers are independent."""
from datetime import timedelta
from sqlalchemy import text
from steward.models.hosts import Host
run_retention = _retention_fn(app)
now = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc)
line40 = "x" * 40 # 40 chars/line; cap=100 keeps 3 lines (excl-prefix 0/40/80)
async def _go():
async with app.db_sessionmaker() as s:
async with s.begin():
await s.execute(text("DELETE FROM docker_logs"))
h = Host(id=str(uuid.uuid4()), name="loghost2", address="10.7.7.8")
s.add(h)
await s.flush()
hid = h.id
def ins(cn, ts, line):
return s.execute(text(
"INSERT INTO docker_logs "
"(id, host_id, container_name, ts, stream, line) "
"VALUES (:id,:h,:cn,:ts,'stdout',:line)"),
{"id": str(uuid.uuid4()), "h": hid, "cn": cn,
"ts": ts, "line": line})
# Past the 3-day age window → age-pruned.
await ins("old", now - timedelta(days=10), "z")
# 5 recent 'web' lines (40 bytes each = 200 > cap 100) → keep 3.
for i in range(1, 6):
await ins("web", now - timedelta(minutes=6 - i), line40)
# 2 recent 'db' lines (80 bytes ≤ cap) → both survive (isolation).
await ins("db", now - timedelta(minutes=2), line40)
await ins("db", now - timedelta(minutes=1), line40)
async with s.begin():
counts = await run_retention(
s, events_days=30, metrics_raw_days=7, metrics_rollup_days=90,
logs_retention_days=3, logs_max_bytes_per_container=100, now=now,
)
def n(cn):
return s.execute(text(
"SELECT COUNT(*) FROM docker_logs WHERE host_id=:h AND container_name=:cn"),
{"h": hid, "cn": cn})
web = (await n("web")).scalar()
db = (await n("db")).scalar()
old = (await n("old")).scalar()
return counts, web, db, old
counts, web, db, old = asyncio.run(_go())
assert old == 0 # age window
assert web == 3 # newest ~100 bytes kept, oldest 2 dropped
assert db == 2 # under cap → untouched (per-container)
assert counts["logs_age_pruned"] == 1
assert counts["logs_size_pruned"] == 2
def test_widget_dedup_collapses_cross_manager_duplicates():
"""The same swarm task is reported by every manager (identical container_id);
the dashboard widget must count it once. Older agents send no container_id,