perf(docker): the hourly log prune ranks only the containers over their cap (#5449)
#5449 "The hourly Docker log prune ranks every log line in the table to find the few over their cap": the size ring put a running-total window over every row of docker_logs each hour, sorting the whole table on NFS-backed storage to find the handful of containers past docker.logs.max_bytes_per_container. - prune_docker_logs (capability docker.prune_logs) does the age cutoff, then a per-container byte total picks the containers at or over the cap, and only their rows are ranked and trimmed. Same keep rule as before: the newest line always survives. - run_docker_retention keeps the metrics rollup and the event prune. - Cleanup runs the log ring in its own transaction after them. Arose from #5431 "Ingest dropped for ~3 minutes after each deploy while startup retention runs". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -118,6 +118,15 @@ def _retention_fn(app):
|
||||
return run_docker_retention
|
||||
|
||||
|
||||
def _prune_logs_fn(app):
|
||||
"""Resolve prune_docker_logs via capability (or direct import if unloaded)."""
|
||||
from steward.core.capabilities import has_capability, get_capability
|
||||
if has_capability("docker.prune_logs"):
|
||||
return get_capability("docker.prune_logs").fn
|
||||
from plugins.docker.retention import prune_docker_logs
|
||||
return prune_docker_logs
|
||||
|
||||
|
||||
@_NEEDS_DB
|
||||
def test_persist_scopes_containers_by_host(app):
|
||||
from sqlalchemy import text
|
||||
@@ -739,12 +748,14 @@ def test_retention_rollup_and_prune(app):
|
||||
@_NEEDS_DB
|
||||
def test_retention_logs_ring(app):
|
||||
"""docker_logs ring (m79): lines past the age window are pruned; within it,
|
||||
each container keeps only the newest ~cap bytes; containers are independent."""
|
||||
each container keeps only the newest ~cap bytes; containers are independent,
|
||||
including a same-named container on another host (#5449 ranks only the
|
||||
containers over the cap, joined on host and name)."""
|
||||
from datetime import timedelta
|
||||
from sqlalchemy import text
|
||||
from steward.models.hosts import Host
|
||||
|
||||
run_retention = _retention_fn(app)
|
||||
prune_logs = _prune_logs_fn(app)
|
||||
now = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc)
|
||||
line40 = "x" * 40 # 40 chars/line; cap=100 keeps 3 lines (excl-prefix 0/40/80)
|
||||
|
||||
@@ -753,16 +764,17 @@ def test_retention_logs_ring(app):
|
||||
async with s.begin():
|
||||
await s.execute(text("DELETE FROM docker_logs"))
|
||||
h = Host(id=str(uuid.uuid4()), name="loghost2", address="10.7.7.8")
|
||||
s.add(h)
|
||||
h2 = Host(id=str(uuid.uuid4()), name="loghost3", address="10.7.7.9")
|
||||
s.add_all([h, h2])
|
||||
await s.flush()
|
||||
hid = h.id
|
||||
hid, hid2 = h.id, h2.id
|
||||
|
||||
def ins(cn, ts, line):
|
||||
def ins(cn, ts, line, host=None):
|
||||
return s.execute(text(
|
||||
"INSERT INTO docker_logs "
|
||||
"(id, host_id, container_name, ts, stream, line) "
|
||||
"VALUES (:id,:h,:cn,:ts,'stdout',:line)"),
|
||||
{"id": str(uuid.uuid4()), "h": hid, "cn": cn,
|
||||
{"id": str(uuid.uuid4()), "h": host or hid, "cn": cn,
|
||||
"ts": ts, "line": line})
|
||||
|
||||
# Past the 3-day age window → age-pruned.
|
||||
@@ -773,26 +785,29 @@ def test_retention_logs_ring(app):
|
||||
# 2 recent 'db' lines (80 bytes ≤ cap) → both survive (isolation).
|
||||
await ins("db", now - timedelta(minutes=2), line40)
|
||||
await ins("db", now - timedelta(minutes=1), line40)
|
||||
# 'web' on the other host: 2 lines, under the cap → both survive.
|
||||
await ins("web", now - timedelta(minutes=2), line40, host=hid2)
|
||||
await ins("web", now - timedelta(minutes=1), line40, host=hid2)
|
||||
|
||||
async with s.begin():
|
||||
counts = await run_retention(
|
||||
s, events_days=30, metrics_raw_days=7, metrics_rollup_days=90,
|
||||
logs_retention_days=3, logs_max_bytes_per_container=100, now=now,
|
||||
)
|
||||
counts = await prune_logs(
|
||||
s, retention_days=3, max_bytes_per_container=100, now=now)
|
||||
|
||||
def n(cn):
|
||||
def n(cn, host=None):
|
||||
return s.execute(text(
|
||||
"SELECT COUNT(*) FROM docker_logs WHERE host_id=:h AND container_name=:cn"),
|
||||
{"h": hid, "cn": cn})
|
||||
{"h": host or hid, "cn": cn})
|
||||
web = (await n("web")).scalar()
|
||||
db = (await n("db")).scalar()
|
||||
old = (await n("old")).scalar()
|
||||
return counts, web, db, old
|
||||
web2 = (await n("web", hid2)).scalar()
|
||||
return counts, web, db, old, web2
|
||||
|
||||
counts, web, db, old = asyncio.run(_go())
|
||||
counts, web, db, old, web2 = asyncio.run(_go())
|
||||
assert old == 0 # age window
|
||||
assert web == 3 # newest ~100 bytes kept, oldest 2 dropped
|
||||
assert db == 2 # under cap → untouched (per-container)
|
||||
assert web2 == 2 # same name, other host → its own ring
|
||||
assert counts["logs_age_pruned"] == 1
|
||||
assert counts["logs_size_pruned"] == 2
|
||||
|
||||
|
||||
Reference in New Issue
Block a user