refactor(docker): container CPU and memory live in the one metric store (#5752)
#5752 "Container CPU and memory history lives in the one metric store; docker_metrics and docker_metrics_hourly are retired". Container cpu_pct and mem_pct already went to plugin_metrics for alerting; now mem_usage_bytes does too, and every snapshot in an ingest (backlog included) is recorded at its own time. The container history chart, Busiest Containers, a service's history and the MCP container tools read plugin_metrics under resource "<host>/<container>", rolled up and pruned by the shared metric retention. docker_012_drop_metrics drops both tables. Their history is not copied (lesson #5221 "A data migration's cost is set by the live table"); the store already holds 7 days of container CPU and memory %. Docker retention now prunes only lifecycle events, and Settings -> Thresholds has one "Metric retention" card for every source. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
+15
-107
@@ -1,133 +1,41 @@
|
||||
# plugins/docker/retention.py
|
||||
"""Bound Docker time-series growth: roll up old metrics, prune old rows.
|
||||
"""Bound Docker history growth: prune old lifecycle events and log lines.
|
||||
|
||||
Published as the "docker.run_retention" capability (see __init__.setup) so the
|
||||
core cleanup task can drive it WITHOUT importing the docker models (same
|
||||
opportunistic-coupling pattern as docker.persist_host_samples). Runs inside the
|
||||
caller's open transaction; never opens or commits its own.
|
||||
Published as the "docker.run_retention" and "docker.prune_logs" capabilities
|
||||
(see __init__.setup) so the core cleanup task can drive them WITHOUT importing
|
||||
the docker models (same opportunistic-coupling pattern as
|
||||
docker.persist_host_samples). Each runs inside the caller's open transaction;
|
||||
never opens or commits its own.
|
||||
|
||||
The scaling concern is docker_metrics: ~2880 rows/container/day at a 30s sample.
|
||||
We keep raw samples for a short window, then aggregate everything older into
|
||||
hourly averages (docker_metrics_hourly) and delete the raw rows — so multi-day
|
||||
history stays cheap to store and query. docker_events is light but unbounded
|
||||
without a cutoff, so it gets a (longer) window too.
|
||||
Container CPU and memory are not here: they live in the one metric store
|
||||
(plugin_metrics) and its rollup and retention (#5752).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
|
||||
def _hour_floor(dt: datetime) -> datetime:
|
||||
"""Truncate a datetime down to the start of its hour (drops min/sec/µs)."""
|
||||
return dt.replace(minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
def _rollup_cutoff(now: datetime, raw_days: int) -> datetime:
|
||||
"""Hour-aligned boundary below which raw metrics get rolled up + deleted.
|
||||
|
||||
Aligning to the hour means we only ever roll up *whole* elapsed hours — a
|
||||
bucket is never split across the keep/roll boundary, so re-running can't
|
||||
produce a partial-then-complete duplicate for the same hour.
|
||||
"""
|
||||
return _hour_floor(now - timedelta(days=raw_days))
|
||||
|
||||
|
||||
async def run_docker_retention(
|
||||
session,
|
||||
*,
|
||||
events_days: int,
|
||||
metrics_raw_days: int,
|
||||
metrics_rollup_days: int,
|
||||
now: datetime | None = None,
|
||||
) -> dict:
|
||||
"""Roll up + prune Docker time-series. Returns a counts dict for logging.
|
||||
|
||||
1. Aggregate docker_metrics older than the (hour-aligned) raw window into
|
||||
docker_metrics_hourly (avg cpu/mem per container per hour), upserting so a
|
||||
re-run is idempotent, then delete those raw rows.
|
||||
2. Prune rolled-up rows older than the rollup window.
|
||||
3. Prune docker_events older than the events window.
|
||||
|
||||
The container-log ring is prune_docker_logs, run in its own transaction.
|
||||
"""
|
||||
"""Prune docker_events older than the events window. Returns a counts
|
||||
dict for logging. The container-log ring is prune_docker_logs, run in its
|
||||
own transaction."""
|
||||
from datetime import timezone
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy import delete
|
||||
|
||||
from .models import DockerEvent, DockerMetric, DockerMetricHourly
|
||||
from .models import DockerEvent
|
||||
|
||||
if now is None:
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
rolled = rolled_rows = events_pruned = rollup_pruned = 0
|
||||
|
||||
# ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
|
||||
raw_cutoff = _rollup_cutoff(now, metrics_raw_days)
|
||||
hour = func.date_trunc("hour", DockerMetric.scraped_at)
|
||||
agg = (
|
||||
select(
|
||||
DockerMetric.host_id,
|
||||
DockerMetric.container_name,
|
||||
hour.label("bucket"),
|
||||
func.avg(DockerMetric.cpu_pct).label("cpu_pct"),
|
||||
func.avg(DockerMetric.mem_pct).label("mem_pct"),
|
||||
func.avg(DockerMetric.mem_usage_bytes).label("mem_usage_bytes"),
|
||||
func.count().label("sample_count"),
|
||||
)
|
||||
.where(DockerMetric.scraped_at < raw_cutoff)
|
||||
.group_by(DockerMetric.host_id, DockerMetric.container_name, hour)
|
||||
)
|
||||
for r in (await session.execute(agg)).all():
|
||||
stmt = (
|
||||
pg_insert(DockerMetricHourly)
|
||||
.values(
|
||||
host_id=r.host_id,
|
||||
container_name=r.container_name,
|
||||
bucket=r.bucket,
|
||||
cpu_pct=float(r.cpu_pct or 0.0),
|
||||
mem_pct=float(r.mem_pct or 0.0),
|
||||
mem_usage_bytes=int(r.mem_usage_bytes or 0),
|
||||
sample_count=int(r.sample_count or 0),
|
||||
)
|
||||
.on_conflict_do_update(
|
||||
constraint="uq_docker_metrics_hourly_bucket",
|
||||
set_={
|
||||
"cpu_pct": float(r.cpu_pct or 0.0),
|
||||
"mem_pct": float(r.mem_pct or 0.0),
|
||||
"mem_usage_bytes": int(r.mem_usage_bytes or 0),
|
||||
"sample_count": int(r.sample_count or 0),
|
||||
},
|
||||
)
|
||||
)
|
||||
await session.execute(stmt)
|
||||
rolled += 1
|
||||
rolled_rows += int(r.sample_count or 0)
|
||||
if rolled:
|
||||
await session.execute(
|
||||
delete(DockerMetric).where(DockerMetric.scraped_at < raw_cutoff)
|
||||
)
|
||||
|
||||
# ── 2. Prune rolled-up rows beyond the rollup window ──
|
||||
rollup_cutoff = now - timedelta(days=metrics_rollup_days)
|
||||
res = await session.execute(
|
||||
delete(DockerMetricHourly).where(DockerMetricHourly.bucket < rollup_cutoff)
|
||||
delete(DockerEvent).where(DockerEvent.at < now - timedelta(days=events_days))
|
||||
)
|
||||
rollup_pruned = res.rowcount or 0
|
||||
|
||||
# ── 3. Prune lifecycle events beyond the events window ──
|
||||
events_cutoff = now - timedelta(days=events_days)
|
||||
res = await session.execute(
|
||||
delete(DockerEvent).where(DockerEvent.at < events_cutoff)
|
||||
)
|
||||
events_pruned = res.rowcount or 0
|
||||
|
||||
return {
|
||||
"buckets_rolled": rolled,
|
||||
"raw_rows_rolled": rolled_rows,
|
||||
"rollup_pruned": rollup_pruned,
|
||||
"events_pruned": events_pruned,
|
||||
}
|
||||
return {"events_pruned": res.rowcount or 0}
|
||||
|
||||
|
||||
async def prune_docker_logs(
|
||||
|
||||
Reference in New Issue
Block a user