feat(docker): per-container size+age log ring rotation [M79 step 4]
Bound docker_logs growth in the periodic cleanup task (same architecture as the metrics/events retention). run_docker_retention gains logs_retention_days + logs_max_bytes_per_container: it prunes lines past the age window, then keeps only the newest ~cap bytes per (host, container) via a window-function ring (exclusive-prefix sum, so the newest line always survives even if it alone exceeds the cap). Containers rotate independently. - settings DEFAULTS: docker.logs.enabled/exclude/retention_days(3)/ max_bytes_per_container(5MB) — operator preference: ~3 days / ~5 MB - cleanup.py reads the two windows fresh each run (rule 25, no restart) - integration rotate test: age prune + per-container byte cap + isolation Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CAGR73DUowdVFVvYzLXC5C
This commit is contained in:
@@ -511,6 +511,67 @@ def test_retention_rollup_and_prune(app):
|
||||
assert counts["events_pruned"] == 1 and counts["rollup_pruned"] == 1
|
||||
|
||||
|
||||
@_NEEDS_DB
|
||||
def test_retention_logs_ring(app):
|
||||
"""docker_logs ring (m79): lines past the age window are pruned; within it,
|
||||
each container keeps only the newest ~cap bytes; containers are independent."""
|
||||
from datetime import timedelta
|
||||
from sqlalchemy import text
|
||||
from steward.models.hosts import Host
|
||||
|
||||
run_retention = _retention_fn(app)
|
||||
now = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc)
|
||||
line40 = "x" * 40 # 40 chars/line; cap=100 keeps 3 lines (excl-prefix 0/40/80)
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await s.execute(text("DELETE FROM docker_logs"))
|
||||
h = Host(id=str(uuid.uuid4()), name="loghost2", address="10.7.7.8")
|
||||
s.add(h)
|
||||
await s.flush()
|
||||
hid = h.id
|
||||
|
||||
def ins(cn, ts, line):
|
||||
return s.execute(text(
|
||||
"INSERT INTO docker_logs "
|
||||
"(id, host_id, container_name, ts, stream, line) "
|
||||
"VALUES (:id,:h,:cn,:ts,'stdout',:line)"),
|
||||
{"id": str(uuid.uuid4()), "h": hid, "cn": cn,
|
||||
"ts": ts, "line": line})
|
||||
|
||||
# Past the 3-day age window → age-pruned.
|
||||
await ins("old", now - timedelta(days=10), "z")
|
||||
# 5 recent 'web' lines (40 bytes each = 200 > cap 100) → keep 3.
|
||||
for i in range(1, 6):
|
||||
await ins("web", now - timedelta(minutes=6 - i), line40)
|
||||
# 2 recent 'db' lines (80 bytes ≤ cap) → both survive (isolation).
|
||||
await ins("db", now - timedelta(minutes=2), line40)
|
||||
await ins("db", now - timedelta(minutes=1), line40)
|
||||
|
||||
async with s.begin():
|
||||
counts = await run_retention(
|
||||
s, events_days=30, metrics_raw_days=7, metrics_rollup_days=90,
|
||||
logs_retention_days=3, logs_max_bytes_per_container=100, now=now,
|
||||
)
|
||||
|
||||
def n(cn):
|
||||
return s.execute(text(
|
||||
"SELECT COUNT(*) FROM docker_logs WHERE host_id=:h AND container_name=:cn"),
|
||||
{"h": hid, "cn": cn})
|
||||
web = (await n("web")).scalar()
|
||||
db = (await n("db")).scalar()
|
||||
old = (await n("old")).scalar()
|
||||
return counts, web, db, old
|
||||
|
||||
counts, web, db, old = asyncio.run(_go())
|
||||
assert old == 0 # age window
|
||||
assert web == 3 # newest ~100 bytes kept, oldest 2 dropped
|
||||
assert db == 2 # under cap → untouched (per-container)
|
||||
assert counts["logs_age_pruned"] == 1
|
||||
assert counts["logs_size_pruned"] == 2
|
||||
|
||||
|
||||
def test_widget_dedup_collapses_cross_manager_duplicates():
|
||||
"""The same swarm task is reported by every manager (identical container_id);
|
||||
the dashboard widget must count it once. Older agents send no container_id,
|
||||
|
||||
Reference in New Issue
Block a user