perf(docker): the hourly log prune ranks only the containers over their cap (#5449)
CI / lint (push) Successful in 3s
CI / integration (push) Successful in 1m30s
CI / unit (push) Successful in 1m42s
CI / publish (push) Successful in 1m52s

#5449 "The hourly Docker log prune ranks every log line in the table to find
the few over their cap": the size ring put a running-total window over every
row of docker_logs each hour, sorting the whole table on NFS-backed storage
to find the handful of containers past docker.logs.max_bytes_per_container.

- prune_docker_logs (capability docker.prune_logs) does the age cutoff, then
  a per-container byte total picks the containers at or over the cap, and
  only their rows are ranked and trimmed. Same keep rule as before: the
  newest line always survives.
- run_docker_retention keeps the metrics rollup and the event prune.
- Cleanup runs the log ring in its own transaction after them.

Arose from #5431 "Ingest dropped for ~3 minutes after each deploy while
startup retention runs".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-08 22:31:00 -04:00
co-authored by Claude Opus 5.5
parent 4a33046aef
commit b3a5bbfeff
4 changed files with 111 additions and 57 deletions
+29 -14
View File
@@ -118,6 +118,15 @@ def _retention_fn(app):
return run_docker_retention
def _prune_logs_fn(app):
"""Resolve prune_docker_logs via capability (or direct import if unloaded)."""
from steward.core.capabilities import has_capability, get_capability
if has_capability("docker.prune_logs"):
return get_capability("docker.prune_logs").fn
from plugins.docker.retention import prune_docker_logs
return prune_docker_logs
@_NEEDS_DB
def test_persist_scopes_containers_by_host(app):
from sqlalchemy import text
@@ -739,12 +748,14 @@ def test_retention_rollup_and_prune(app):
@_NEEDS_DB
def test_retention_logs_ring(app):
"""docker_logs ring (m79): lines past the age window are pruned; within it,
each container keeps only the newest ~cap bytes; containers are independent."""
each container keeps only the newest ~cap bytes; containers are independent,
including a same-named container on another host (#5449 ranks only the
containers over the cap, joined on host and name)."""
from datetime import timedelta
from sqlalchemy import text
from steward.models.hosts import Host
run_retention = _retention_fn(app)
prune_logs = _prune_logs_fn(app)
now = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc)
line40 = "x" * 40 # 40 chars/line; cap=100 keeps 3 lines (excl-prefix 0/40/80)
@@ -753,16 +764,17 @@ def test_retention_logs_ring(app):
async with s.begin():
await s.execute(text("DELETE FROM docker_logs"))
h = Host(id=str(uuid.uuid4()), name="loghost2", address="10.7.7.8")
s.add(h)
h2 = Host(id=str(uuid.uuid4()), name="loghost3", address="10.7.7.9")
s.add_all([h, h2])
await s.flush()
hid = h.id
hid, hid2 = h.id, h2.id
def ins(cn, ts, line):
def ins(cn, ts, line, host=None):
return s.execute(text(
"INSERT INTO docker_logs "
"(id, host_id, container_name, ts, stream, line) "
"VALUES (:id,:h,:cn,:ts,'stdout',:line)"),
{"id": str(uuid.uuid4()), "h": hid, "cn": cn,
{"id": str(uuid.uuid4()), "h": host or hid, "cn": cn,
"ts": ts, "line": line})
# Past the 3-day age window → age-pruned.
@@ -773,26 +785,29 @@ def test_retention_logs_ring(app):
# 2 recent 'db' lines (80 bytes ≤ cap) → both survive (isolation).
await ins("db", now - timedelta(minutes=2), line40)
await ins("db", now - timedelta(minutes=1), line40)
# 'web' on the other host: 2 lines, under the cap → both survive.
await ins("web", now - timedelta(minutes=2), line40, host=hid2)
await ins("web", now - timedelta(minutes=1), line40, host=hid2)
async with s.begin():
counts = await run_retention(
s, events_days=30, metrics_raw_days=7, metrics_rollup_days=90,
logs_retention_days=3, logs_max_bytes_per_container=100, now=now,
)
counts = await prune_logs(
s, retention_days=3, max_bytes_per_container=100, now=now)
def n(cn):
def n(cn, host=None):
return s.execute(text(
"SELECT COUNT(*) FROM docker_logs WHERE host_id=:h AND container_name=:cn"),
{"h": hid, "cn": cn})
{"h": host or hid, "cn": cn})
web = (await n("web")).scalar()
db = (await n("db")).scalar()
old = (await n("old")).scalar()
return counts, web, db, old
web2 = (await n("web", hid2)).scalar()
return counts, web, db, old, web2
counts, web, db, old = asyncio.run(_go())
counts, web, db, old, web2 = asyncio.run(_go())
assert old == 0 # age window
assert web == 3 # newest ~100 bytes kept, oldest 2 dropped
assert db == 2 # under cap → untouched (per-container)
assert web2 == 2 # same name, other host → its own ring
assert counts["logs_age_pruned"] == 1
assert counts["logs_size_pruned"] == 2