refactor(docker): container CPU and memory live in the one metric store (#5752)
#5752 "Container CPU and memory history lives in the one metric store; docker_metrics and docker_metrics_hourly are retired". Container cpu_pct and mem_pct already went to plugin_metrics for alerting; now mem_usage_bytes does too, and every snapshot in an ingest (backlog included) is recorded at its own time. The container history chart, Busiest Containers, a service's history and the MCP container tools read plugin_metrics under resource "<host>/<container>", rolled up and pruned by the shared metric retention. docker_012_drop_metrics drops both tables. Their history is not copied (lesson #5221 "A data migration's cost is set by the live table"); the store already holds 7 days of container CPU and memory %. Docker retention now prunes only lifecycle events, and Settings -> Thresholds has one "Metric retention" card for every source. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+18
-28
@@ -306,9 +306,10 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
|
||||
`snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
|
||||
one entry per ingested sample carrying docker data (usually one; more when
|
||||
the agent flushes a backlog). Every snapshot contributes time-series
|
||||
DockerMetric rows; the newest snapshot drives current container state, the
|
||||
alert pipeline, and lifecycle-event derivation. `swarm` is the newest
|
||||
the agent flushes a backlog). Every snapshot's running containers record
|
||||
cpu_pct, mem_pct and mem_usage_bytes into the one metric store, at the
|
||||
snapshot's time; the newest snapshot drives current container state and
|
||||
lifecycle-event derivation. `swarm` is the newest
|
||||
sample's swarm object (or None off managers) — persisted when present.
|
||||
`disk` is the newest sample's /system/df summary (or None on Docker-less
|
||||
hosts) — persisted when present. `logs` is a list of (recorded_at, records)
|
||||
@@ -323,7 +324,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
flushed before its mark, so a slow ingest names the part (#5181).
|
||||
"""
|
||||
from steward.core.alerts import MetricBatch
|
||||
from .models import DockerContainer, DockerEvent, DockerMetric
|
||||
from .models import DockerContainer, DockerEvent
|
||||
|
||||
async def mark(name: str) -> None:
|
||||
if phases is not None:
|
||||
@@ -350,18 +351,20 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
ordered = sorted(snapshots, key=lambda s: s[0])
|
||||
latest_at, latest_containers = ordered[-1]
|
||||
|
||||
# Time-series points for every snapshot (running containers with a CPU read).
|
||||
# Time series for every snapshot (running containers with a CPU read), in
|
||||
# the one metric store (#5752): resource "<host>/<container>", so
|
||||
# containers of the same name on different hosts don't collide. Recorded
|
||||
# with the alert batch below, at each snapshot's own time.
|
||||
batch = MetricBatch()
|
||||
for recorded_at, containers in ordered:
|
||||
for c in containers:
|
||||
if c.get("status") == "running" and c.get("cpu_pct") is not None:
|
||||
session.add(DockerMetric(
|
||||
host_id=host.id,
|
||||
container_name=c["name"],
|
||||
scraped_at=recorded_at,
|
||||
cpu_pct=c["cpu_pct"],
|
||||
mem_pct=c.get("mem_pct") or 0.0,
|
||||
mem_usage_bytes=c.get("mem_usage_bytes") or 0,
|
||||
))
|
||||
if c.get("status") != "running" or c.get("cpu_pct") is None or not c.get("name"):
|
||||
continue
|
||||
resource = f"{host.name}/{c['name']}"
|
||||
for metric in ("cpu_pct", "mem_pct", "mem_usage_bytes"):
|
||||
if c.get(metric) is not None:
|
||||
batch.add(source_module="docker", resource_name=resource,
|
||||
metric_name=metric, value=c[metric], recorded_at=recorded_at)
|
||||
await mark("metrics")
|
||||
|
||||
# Snapshot of stored per-container state BEFORE the upsert overwrites it —
|
||||
@@ -389,8 +392,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
))
|
||||
await mark("events")
|
||||
|
||||
# Current state + alerts from the newest snapshot only.
|
||||
batch = MetricBatch()
|
||||
# Current state, and the restart and health alerts, from the newest snapshot only.
|
||||
for c in latest_containers:
|
||||
name = c.get("name")
|
||||
if not name:
|
||||
@@ -424,19 +426,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
existing.blk_read_bytes = c.get("blk_read_bytes")
|
||||
existing.blk_write_bytes = c.get("blk_write_bytes")
|
||||
|
||||
# Alert pipeline — resource is host-scoped so containers of the same name
|
||||
# on different hosts don't collide in the metric/alert namespace.
|
||||
resource = f"{host.name}/{name}"
|
||||
if c.get("status") == "running" and c.get("cpu_pct") is not None:
|
||||
batch.add(
|
||||
source_module="docker",
|
||||
resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"],
|
||||
)
|
||||
if c.get("mem_pct") is not None:
|
||||
batch.add(
|
||||
source_module="docker",
|
||||
resource_name=resource, metric_name="mem_pct", value=c["mem_pct"],
|
||||
)
|
||||
# Restart count is alertable regardless of state (crash-looping matters
|
||||
# most while the container is down/restarting).
|
||||
batch.add(
|
||||
|
||||
Reference in New Issue
Block a user