perf(docker): the hourly log prune ranks only the containers over their cap (#5449)
#5449 "The hourly Docker log prune ranks every log line in the table to find the few over their cap": the size ring put a running-total window over every row of docker_logs each hour, sorting the whole table on NFS-backed storage to find the handful of containers past docker.logs.max_bytes_per_container. - prune_docker_logs (capability docker.prune_logs) does the age cutoff, then a per-container byte total picks the containers at or over the cap, and only their rows are ranked and trimmed. Same keep rule as before: the newest line always survives. - run_docker_retention keeps the metrics rollup and the event prune. - Cleanup runs the log ring in its own transaction after them. Arose from #5431 "Ingest dropped for ~3 minutes after each deploy while startup retention runs". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+15
-9
@@ -103,12 +103,14 @@ async def _run_metrics_retention(session, now: datetime) -> None:
|
||||
|
||||
|
||||
async def _run_docker_retention(session, now: datetime) -> None:
|
||||
"""Drive the docker plugin's rollup + prune via its capability, if loaded.
|
||||
"""Drive the docker plugin's rollup + prune and its log ring via its
|
||||
capabilities, if loaded.
|
||||
|
||||
Windows are read fresh from settings each run (rule 25 — a change in the
|
||||
Settings UI takes effect on the next hourly cleanup, no restart). Kept in its
|
||||
own transaction so a docker-side failure can't roll back the generic prune
|
||||
above. No-op when the docker plugin is disabled (capability absent).
|
||||
Settings UI takes effect on the next hourly cleanup, no restart). Each runs
|
||||
in its own transaction, so a docker-side failure can't roll back the generic
|
||||
prune above, and the log ring's deletes don't wait on the metrics rollup
|
||||
(#5449). No-op when the docker plugin is disabled (capability absent).
|
||||
"""
|
||||
from steward.core.capabilities import has_capability, invoke_capability
|
||||
if not has_capability("docker.run_retention"):
|
||||
@@ -122,14 +124,18 @@ async def _run_docker_retention(session, now: datetime) -> None:
|
||||
raw_days = int(await get_setting(session, "docker.retention.metrics_raw_days") or 7)
|
||||
rollup_days = int(await get_setting(session, "docker.retention.metrics_rollup_days") or 90)
|
||||
events_days = int(await get_setting(session, "docker.retention.events_days") or 30)
|
||||
logs_days = int(await get_setting(session, "docker.logs.retention_days") or 3)
|
||||
logs_cap = int(
|
||||
await get_setting(session, "docker.logs.max_bytes_per_container") or 5_000_000)
|
||||
counts = await invoke_capability(
|
||||
"docker.run_retention", UserRole.viewer, session,
|
||||
events_days=events_days, metrics_raw_days=raw_days,
|
||||
metrics_rollup_days=rollup_days, logs_retention_days=logs_days,
|
||||
logs_max_bytes_per_container=logs_cap, now=now,
|
||||
metrics_rollup_days=rollup_days, now=now,
|
||||
)
|
||||
async with session.begin():
|
||||
logs_days = int(await get_setting(session, "docker.logs.retention_days") or 3)
|
||||
logs_cap = int(
|
||||
await get_setting(session, "docker.logs.max_bytes_per_container") or 5_000_000)
|
||||
counts |= await invoke_capability(
|
||||
"docker.prune_logs", UserRole.viewer, session,
|
||||
retention_days=logs_days, max_bytes_per_container=logs_cap, now=now,
|
||||
)
|
||||
if counts and any(counts.values()):
|
||||
logger.info("Docker retention: %s", counts)
|
||||
|
||||
Reference in New Issue
Block a user