refactor(docker): container CPU and memory live in the one metric store (#5752)
#5752 "Container CPU and memory history lives in the one metric store; docker_metrics and docker_metrics_hourly are retired". Container cpu_pct and mem_pct already went to plugin_metrics for alerting; now mem_usage_bytes does too, and every snapshot in an ingest (backlog included) is recorded at its own time. The container history chart, Busiest Containers, a service's history and the MCP container tools read plugin_metrics under resource "<host>/<container>", rolled up and pruned by the shared metric retention. docker_012_drop_metrics drops both tables. Their history is not copied (lesson #5221 "A data migration's cost is set by the live table"); the store already holds 7 days of container CPU and memory %. Docker retention now prunes only lifecycle events, and Settings -> Thresholds has one "Metric retention" card for every source. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -36,8 +36,8 @@ def test_docker_tables_are_host_scoped(app):
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
# Both tables carry a host_id column after docker_002.
|
||||
for tbl in ("docker_containers", "docker_metrics"):
|
||||
# docker_containers carries a host_id column after docker_002.
|
||||
for tbl in ("docker_containers",):
|
||||
cols = {r[0] for r in (await s.execute(text(
|
||||
"SELECT column_name FROM information_schema.columns "
|
||||
"WHERE table_name = :t"), {"t": tbl})).all()}
|
||||
@@ -148,7 +148,6 @@ def test_persist_scopes_containers_by_host(app):
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await s.execute(text("DELETE FROM docker_metrics"))
|
||||
await s.execute(text("DELETE FROM docker_containers"))
|
||||
await s.execute(text(
|
||||
"DELETE FROM plugin_metrics WHERE source_module = 'docker'"))
|
||||
@@ -162,10 +161,12 @@ def test_persist_scopes_containers_by_host(app):
|
||||
|
||||
web_rows = (await s.execute(text(
|
||||
"SELECT COUNT(*) FROM docker_containers WHERE name = 'web'"))).scalar()
|
||||
# The series went into the one metric store, host-scoped (#5752),
|
||||
# at the snapshot's time.
|
||||
metric_hosts = (await s.execute(text(
|
||||
"SELECT COUNT(DISTINCT host_id) FROM docker_metrics "
|
||||
"WHERE container_name = 'web'"))).scalar()
|
||||
# Alert pipeline received host-scoped resource names.
|
||||
"SELECT COUNT(DISTINCT resource_name) FROM plugin_metrics "
|
||||
"WHERE source_module = 'docker' AND metric_name = 'mem_usage_bytes' "
|
||||
"AND recorded_at = :at"), {"at": now})).scalar()
|
||||
resources = {r[0] for r in (await s.execute(text(
|
||||
"SELECT DISTINCT resource_name FROM plugin_metrics "
|
||||
"WHERE source_module = 'docker'"))).all()}
|
||||
@@ -295,7 +296,6 @@ def test_large_memory_values_persist_as_bigint(app):
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await s.execute(text("DELETE FROM docker_metrics"))
|
||||
await s.execute(text("DELETE FROM docker_containers"))
|
||||
h = Host(id=str(uuid.uuid4()), name="bigmemhost", address="10.0.0.9")
|
||||
s.add(h)
|
||||
@@ -305,8 +305,9 @@ def test_large_memory_values_persist_as_bigint(app):
|
||||
"SELECT mem_usage_bytes, mem_limit_bytes FROM docker_containers "
|
||||
"WHERE name = 'bigmem'"))).first()
|
||||
metric = (await s.execute(text(
|
||||
"SELECT mem_usage_bytes FROM docker_metrics "
|
||||
"WHERE container_name = 'bigmem'"))).scalar()
|
||||
"SELECT value FROM plugin_metrics WHERE source_module = 'docker' "
|
||||
"AND resource_name = 'bigmemhost/bigmem' "
|
||||
"AND metric_name = 'mem_usage_bytes'"))).scalar()
|
||||
return row, metric
|
||||
|
||||
row, metric = asyncio.run(_go())
|
||||
@@ -332,7 +333,6 @@ def test_lifecycle_events_derived_across_snapshots(app):
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await s.execute(text("DELETE FROM docker_events"))
|
||||
await s.execute(text("DELETE FROM docker_metrics"))
|
||||
await s.execute(text("DELETE FROM docker_containers"))
|
||||
h = Host(id=str(uuid.uuid4()), name="evt", address="10.9.9.9")
|
||||
s.add(h)
|
||||
@@ -659,90 +659,43 @@ def test_disk_usage_persisted(app):
|
||||
|
||||
|
||||
@_NEEDS_DB
|
||||
def test_retention_rollup_and_prune(app):
|
||||
"""Old raw metrics roll up to hourly averages then delete; stale rollup +
|
||||
events prune; recent rows survive."""
|
||||
def test_retention_prunes_events_past_their_window(app):
|
||||
"""Docker retention prunes lifecycle events past the events window; the
|
||||
container series roll up with the one metric store (#5752)."""
|
||||
from sqlalchemy import text
|
||||
from steward.models.hosts import Host
|
||||
|
||||
run_retention = _retention_fn(app)
|
||||
now = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc)
|
||||
# One old hour (10 days back) with three samples → one rolled-up bucket.
|
||||
old_hour = datetime(2026, 6, 9, 9, 0, 0, tzinfo=timezone.utc)
|
||||
recent = datetime(2026, 6, 19, 11, 0, 0, tzinfo=timezone.utc) # inside raw window
|
||||
ancient_bucket = datetime(2026, 3, 1, 0, 0, 0, tzinfo=timezone.utc) # > rollup window
|
||||
old_event = datetime(2026, 5, 1, 0, 0, 0, tzinfo=timezone.utc) # > events window
|
||||
new_event = datetime(2026, 6, 18, 0, 0, 0, tzinfo=timezone.utc) # inside events window
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await s.execute(text("DELETE FROM docker_metrics_hourly"))
|
||||
await s.execute(text("DELETE FROM docker_metrics"))
|
||||
await s.execute(text("DELETE FROM docker_events"))
|
||||
h = Host(id=str(uuid.uuid4()), name="ret", address="10.7.7.7")
|
||||
s.add(h)
|
||||
await s.flush()
|
||||
hid = h.id
|
||||
# Three raw samples in the old hour: cpu 10/20/30, mem 40/50/60.
|
||||
for i, (ts, cpu, mem, usage) in enumerate([
|
||||
(old_hour, 10.0, 40.0, 100),
|
||||
(old_hour.replace(second=30), 20.0, 50.0, 200),
|
||||
(old_hour.replace(minute=1), 30.0, 60.0, 300),
|
||||
]):
|
||||
await s.execute(text(
|
||||
"INSERT INTO docker_metrics "
|
||||
"(id, host_id, container_name, scraped_at, cpu_pct, mem_pct, mem_usage_bytes) "
|
||||
"VALUES (:id,:h,'web',:ts,:cpu,:mem,:usage)"),
|
||||
{"id": str(uuid.uuid4()), "h": hid, "ts": ts,
|
||||
"cpu": cpu, "mem": mem, "usage": usage})
|
||||
# A recent sample (within the 7-day raw window) — must survive raw.
|
||||
await s.execute(text(
|
||||
"INSERT INTO docker_metrics "
|
||||
"(id, host_id, container_name, scraped_at, cpu_pct, mem_pct, mem_usage_bytes) "
|
||||
"VALUES (:id,:h,'web',:ts,5.0,5.0,50)"),
|
||||
{"id": str(uuid.uuid4()), "h": hid, "ts": recent})
|
||||
# A pre-existing rollup row older than the 90-day rollup window.
|
||||
await s.execute(text(
|
||||
"INSERT INTO docker_metrics_hourly "
|
||||
"(id, host_id, container_name, bucket, cpu_pct, mem_pct, mem_usage_bytes, sample_count) "
|
||||
"VALUES (:id,:h,'ancient',:b,1,1,1,1)"),
|
||||
{"id": str(uuid.uuid4()), "h": hid, "b": ancient_bucket})
|
||||
# Events either side of the 30-day events window.
|
||||
for ev_at, ev in [(old_event, "stop"), (new_event, "start")]:
|
||||
await s.execute(text(
|
||||
"INSERT INTO docker_events (id, host_id, container_name, event, at) "
|
||||
"VALUES (:id,:h,'web',:ev,:at)"),
|
||||
{"id": str(uuid.uuid4()), "h": hid, "ev": ev, "at": ev_at})
|
||||
|
||||
async with s.begin():
|
||||
counts = await run_retention(
|
||||
s, events_days=30, metrics_raw_days=7,
|
||||
metrics_rollup_days=90, now=now,
|
||||
)
|
||||
|
||||
raw_left = (await s.execute(text(
|
||||
"SELECT COUNT(*) FROM docker_metrics WHERE host_id=:h"), {"h": hid})).scalar()
|
||||
bucket = (await s.execute(text(
|
||||
"SELECT cpu_pct, mem_pct, mem_usage_bytes, sample_count "
|
||||
"FROM docker_metrics_hourly WHERE host_id=:h AND container_name='web'"),
|
||||
{"h": hid})).first()
|
||||
ancient_left = (await s.execute(text(
|
||||
"SELECT COUNT(*) FROM docker_metrics_hourly "
|
||||
"WHERE host_id=:h AND container_name='ancient'"), {"h": hid})).scalar()
|
||||
counts = await run_retention(s, events_days=30, now=now)
|
||||
events_left = (await s.execute(text(
|
||||
"SELECT COUNT(*) FROM docker_events WHERE host_id=:h"), {"h": hid})).scalar()
|
||||
return counts, raw_left, bucket, ancient_left, events_left
|
||||
tables = {r[0] for r in (await s.execute(text(
|
||||
"SELECT table_name FROM information_schema.tables "
|
||||
"WHERE table_name LIKE 'docker_metrics%'"))).all()}
|
||||
return counts, events_left, tables
|
||||
|
||||
counts, raw_left, bucket, ancient_left, events_left = asyncio.run(_go())
|
||||
assert raw_left == 1 # only the recent sample survives raw
|
||||
assert bucket is not None
|
||||
assert bucket[0] == 20.0 and bucket[1] == 50.0 # avg cpu / mem over the 3 samples
|
||||
assert bucket[2] == 200 and bucket[3] == 3 # avg usage + sample_count
|
||||
assert ancient_left == 0 # stale rollup pruned
|
||||
counts, events_left, tables = asyncio.run(_go())
|
||||
assert events_left == 1 # only the in-window event survives
|
||||
assert counts["buckets_rolled"] == 1 and counts["raw_rows_rolled"] == 3
|
||||
assert counts["events_pruned"] == 1 and counts["rollup_pruned"] == 1
|
||||
assert counts == {"events_pruned": 1}
|
||||
assert tables == set() # docker_012 dropped both tables
|
||||
|
||||
|
||||
@_NEEDS_DB
|
||||
|
||||
Reference in New Issue
Block a user