refactor(docker): container CPU and memory live in the one metric store (#5752)
CI / lint (push) Successful in 3s
CI / integration (push) Failing after 1m18s
CI / unit (push) Successful in 1m24s
CI / publish (push) Skipped

#5752 "Container CPU and memory history lives in the one metric store;
docker_metrics and docker_metrics_hourly are retired". Container cpu_pct and
mem_pct already went to plugin_metrics for alerting; now mem_usage_bytes does
too, and every snapshot in an ingest (backlog included) is recorded at its
own time. The container history chart, Busiest Containers, a service's
history and the MCP container tools read plugin_metrics under resource
"<host>/<container>", rolled up and pruned by the shared metric retention.

docker_012_drop_metrics drops both tables. Their history is not copied
(lesson #5221 "A data migration's cost is set by the live table"); the
store already holds 7 days of container CPU and memory %. Docker retention
now prunes only lifecycle events, and Settings -> Thresholds has one
"Metric retention" card for every source.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-14 14:39:56 -04:00
co-authored by Claude Opus 5.5
parent b889584906
commit 5a0e150789
19 changed files with 242 additions and 430 deletions
+5 -4
View File
@@ -11,7 +11,7 @@ _app: "Quart | None" = None
def setup(app: "Quart") -> None:
global _app
_app = app
from .models import DockerContainer, DockerMetric # noqa: F401 — register with Base
from .models import DockerContainer # noqa: F401 — register with Base
# Publish the per-host persist hook so the host agent can hand off the
# `docker` array from each sample without importing our models (opportunistic
@@ -28,12 +28,13 @@ def setup(app: "Quart") -> None:
description="Store per-host container state + metrics pushed by the host agent.",
required_role=UserRole.viewer,
)
# Roll up + prune Docker time-series, driven by the core cleanup task without
# it importing our models. Same trusted server-side data-plane role as above.
# Prune lifecycle events, driven by the core cleanup task without it
# importing our models. Same trusted server-side data-plane role as above.
# Container CPU and memory live in the one metric store and its retention.
register_capability(
"docker.run_retention", run_docker_retention,
label="Run Docker retention",
description="Roll up old docker_metrics to hourly + prune stale metrics/events.",
description="Prune Docker lifecycle events past their window.",
required_role=UserRole.viewer,
)
register_capability(
+18 -28
View File
@@ -306,9 +306,10 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
`snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
one entry per ingested sample carrying docker data (usually one; more when
the agent flushes a backlog). Every snapshot contributes time-series
DockerMetric rows; the newest snapshot drives current container state, the
alert pipeline, and lifecycle-event derivation. `swarm` is the newest
the agent flushes a backlog). Every snapshot's running containers record
cpu_pct, mem_pct and mem_usage_bytes into the one metric store, at the
snapshot's time; the newest snapshot drives current container state and
lifecycle-event derivation. `swarm` is the newest
sample's swarm object (or None off managers) — persisted when present.
`disk` is the newest sample's /system/df summary (or None on Docker-less
hosts) — persisted when present. `logs` is a list of (recorded_at, records)
@@ -323,7 +324,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
flushed before its mark, so a slow ingest names the part (#5181).
"""
from steward.core.alerts import MetricBatch
from .models import DockerContainer, DockerEvent, DockerMetric
from .models import DockerContainer, DockerEvent
async def mark(name: str) -> None:
if phases is not None:
@@ -350,18 +351,20 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
ordered = sorted(snapshots, key=lambda s: s[0])
latest_at, latest_containers = ordered[-1]
# Time-series points for every snapshot (running containers with a CPU read).
# Time series for every snapshot (running containers with a CPU read), in
# the one metric store (#5752): resource "<host>/<container>", so
# containers of the same name on different hosts don't collide. Recorded
# with the alert batch below, at each snapshot's own time.
batch = MetricBatch()
for recorded_at, containers in ordered:
for c in containers:
if c.get("status") == "running" and c.get("cpu_pct") is not None:
session.add(DockerMetric(
host_id=host.id,
container_name=c["name"],
scraped_at=recorded_at,
cpu_pct=c["cpu_pct"],
mem_pct=c.get("mem_pct") or 0.0,
mem_usage_bytes=c.get("mem_usage_bytes") or 0,
))
if c.get("status") != "running" or c.get("cpu_pct") is None or not c.get("name"):
continue
resource = f"{host.name}/{c['name']}"
for metric in ("cpu_pct", "mem_pct", "mem_usage_bytes"):
if c.get(metric) is not None:
batch.add(source_module="docker", resource_name=resource,
metric_name=metric, value=c[metric], recorded_at=recorded_at)
await mark("metrics")
# Snapshot of stored per-container state BEFORE the upsert overwrites it —
@@ -389,8 +392,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
))
await mark("events")
# Current state + alerts from the newest snapshot only.
batch = MetricBatch()
# Current state, and the restart and health alerts, from the newest snapshot only.
for c in latest_containers:
name = c.get("name")
if not name:
@@ -424,19 +426,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
existing.blk_read_bytes = c.get("blk_read_bytes")
existing.blk_write_bytes = c.get("blk_write_bytes")
# Alert pipeline — resource is host-scoped so containers of the same name
# on different hosts don't collide in the metric/alert namespace.
resource = f"{host.name}/{name}"
if c.get("status") == "running" and c.get("cpu_pct") is not None:
batch.add(
source_module="docker",
resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"],
)
if c.get("mem_pct") is not None:
batch.add(
source_module="docker",
resource_name=resource, metric_name="mem_pct", value=c["mem_pct"],
)
# Restart count is alertable regardless of state (crash-looping matters
# most while the container is down/restarting).
batch.add(
+1 -1
View File
@@ -15,7 +15,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
from steward.models.base import Base
import steward.models # noqa: F401
from plugins.docker.models import DockerContainer, DockerMetric # noqa: F401
from plugins.docker.models import DockerContainer # noqa: F401
config = context.config
if config.config_file_name is not None:
@@ -0,0 +1,56 @@
"""Container CPU and memory move to the one metric store
Drops docker_metrics and docker_metrics_hourly (#5752). Container cpu_pct,
mem_pct and mem_usage_bytes are recorded into plugin_metrics as source
"docker", resource "<host>/<container>", and roll up with every other metric.
CPU and memory % were already recorded there for alerting, so their history
stays; memory in bytes starts fresh.
Revision ID: docker_012_drop_metrics
Revises: docker_011_host_status
Create Date: 2026-10-14
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "docker_012_drop_metrics"
down_revision: Union[str, None] = "docker_011_host_status"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.execute("DROP TABLE IF EXISTS docker_metrics_hourly")
op.execute("DROP TABLE IF EXISTS docker_metrics")
def downgrade() -> None:
op.create_table(
"docker_metrics",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("host_id", sa.String(36), sa.ForeignKey("hosts.id", ondelete="CASCADE"),
nullable=False, index=True),
sa.Column("container_name", sa.String(255), nullable=False, index=True),
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False, index=True),
sa.Column("cpu_pct", sa.Float, nullable=False),
sa.Column("mem_pct", sa.Float, nullable=False),
sa.Column("mem_usage_bytes", sa.BigInteger, nullable=False),
)
op.create_index("ix_docker_metrics_host_container_time", "docker_metrics",
["host_id", "container_name", "scraped_at"])
op.create_table(
"docker_metrics_hourly",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("host_id", sa.String(36), sa.ForeignKey("hosts.id", ondelete="CASCADE"),
nullable=False),
sa.Column("container_name", sa.String(255), nullable=False),
sa.Column("bucket", sa.DateTime(timezone=True), nullable=False),
sa.Column("cpu_pct", sa.Float, nullable=False),
sa.Column("mem_pct", sa.Float, nullable=False),
sa.Column("mem_usage_bytes", sa.BigInteger, nullable=False),
sa.Column("sample_count", sa.Integer, nullable=False),
sa.UniqueConstraint("host_id", "container_name", "bucket",
name="uq_docker_metrics_hourly_bucket"),
)
op.create_index("ix_docker_metrics_hourly_bucket", "docker_metrics_hourly", ["bucket"])
+1 -65
View File
@@ -3,7 +3,6 @@ from __future__ import annotations
from datetime import datetime, timezone
from sqlalchemy import (
BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, String, Text,
UniqueConstraint,
)
from sqlalchemy.orm import Mapped, mapped_column
from steward.models.base import Base
@@ -61,69 +60,6 @@ class DockerContainer(Base):
blk_write_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
class DockerMetric(Base):
"""Time-series CPU/memory per container — one row per sample per running
container, scoped to the reporting host."""
__tablename__ = "docker_metrics"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=time_ordered_id
)
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
)
container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
scraped_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False,
default=lambda: datetime.now(timezone.utc),
index=True,
)
cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
# BigInteger: per-container memory usage exceeds 2^31 bytes (~2.1 GB) routinely.
mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
# Per-container history lookups filter on (host_id, container_name) then sort
# by time — a composite index keeps the rows() sparkline queries cheap.
__table_args__ = (
Index("ix_docker_metrics_host_container_time",
"host_id", "container_name", "scraped_at"),
)
class DockerMetricHourly(Base):
"""Hourly rollup of docker_metrics — avg cpu/mem per container per hour.
Raw per-sample rows (~2880/container/day at 30s) are pruned beyond a short
window; before deletion they're aggregated here so multi-day history stays
cheap to store and query. One row per (host, container, hour bucket); the
unique constraint lets retention upsert idempotently if it re-runs before the
raw rows are deleted. `bucket` is the hour-truncated sample time.
"""
__tablename__ = "docker_metrics_hourly"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=time_ordered_id
)
host_id: Mapped[str] = mapped_column(
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
)
container_name: Mapped[str] = mapped_column(String(255), nullable=False)
bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
__table_args__ = (
# One bucket per container per host — the conflict target for the
# idempotent rollup upsert; doubles as the history-query index.
UniqueConstraint("host_id", "container_name", "bucket",
name="uq_docker_metrics_hourly_bucket"),
Index("ix_docker_metrics_hourly_bucket", "bucket"),
)
class DockerEvent(Base):
"""Lifecycle events derived by diffing consecutive host snapshots.
@@ -167,7 +103,7 @@ class DockerLog(Base):
"""Container log lines pushed by the host agent — one row per line.
Time-series, scoped to the reporting host (container names are only unique
within a host, same identity as docker_metrics). The agent tails each running
within a host, same identity as docker_containers). The agent tails each running
container incrementally and folds new lines into its metrics push; `ts` is the
line's own Docker timestamp. Bounded by a per-container size+age ring
(retention), so a chatty container just keeps a shorter window rather than
+15 -107
View File
@@ -1,133 +1,41 @@
# plugins/docker/retention.py
"""Bound Docker time-series growth: roll up old metrics, prune old rows.
"""Bound Docker history growth: prune old lifecycle events and log lines.
Published as the "docker.run_retention" capability (see __init__.setup) so the
core cleanup task can drive it WITHOUT importing the docker models (same
opportunistic-coupling pattern as docker.persist_host_samples). Runs inside the
caller's open transaction; never opens or commits its own.
Published as the "docker.run_retention" and "docker.prune_logs" capabilities
(see __init__.setup) so the core cleanup task can drive them WITHOUT importing
the docker models (same opportunistic-coupling pattern as
docker.persist_host_samples). Each runs inside the caller's open transaction;
never opens or commits its own.
The scaling concern is docker_metrics: ~2880 rows/container/day at a 30s sample.
We keep raw samples for a short window, then aggregate everything older into
hourly averages (docker_metrics_hourly) and delete the raw rows — so multi-day
history stays cheap to store and query. docker_events is light but unbounded
without a cutoff, so it gets a (longer) window too.
Container CPU and memory are not here: they live in the one metric store
(plugin_metrics) and its rollup and retention (#5752).
"""
from __future__ import annotations
from datetime import datetime, timedelta
def _hour_floor(dt: datetime) -> datetime:
"""Truncate a datetime down to the start of its hour (drops min/sec/µs)."""
return dt.replace(minute=0, second=0, microsecond=0)
def _rollup_cutoff(now: datetime, raw_days: int) -> datetime:
"""Hour-aligned boundary below which raw metrics get rolled up + deleted.
Aligning to the hour means we only ever roll up *whole* elapsed hours — a
bucket is never split across the keep/roll boundary, so re-running can't
produce a partial-then-complete duplicate for the same hour.
"""
return _hour_floor(now - timedelta(days=raw_days))
async def run_docker_retention(
session,
*,
events_days: int,
metrics_raw_days: int,
metrics_rollup_days: int,
now: datetime | None = None,
) -> dict:
"""Roll up + prune Docker time-series. Returns a counts dict for logging.
1. Aggregate docker_metrics older than the (hour-aligned) raw window into
docker_metrics_hourly (avg cpu/mem per container per hour), upserting so a
re-run is idempotent, then delete those raw rows.
2. Prune rolled-up rows older than the rollup window.
3. Prune docker_events older than the events window.
The container-log ring is prune_docker_logs, run in its own transaction.
"""
"""Prune docker_events older than the events window. Returns a counts
dict for logging. The container-log ring is prune_docker_logs, run in its
own transaction."""
from datetime import timezone
from sqlalchemy import delete, func, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy import delete
from .models import DockerEvent, DockerMetric, DockerMetricHourly
from .models import DockerEvent
if now is None:
now = datetime.now(timezone.utc)
rolled = rolled_rows = events_pruned = rollup_pruned = 0
# ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
raw_cutoff = _rollup_cutoff(now, metrics_raw_days)
hour = func.date_trunc("hour", DockerMetric.scraped_at)
agg = (
select(
DockerMetric.host_id,
DockerMetric.container_name,
hour.label("bucket"),
func.avg(DockerMetric.cpu_pct).label("cpu_pct"),
func.avg(DockerMetric.mem_pct).label("mem_pct"),
func.avg(DockerMetric.mem_usage_bytes).label("mem_usage_bytes"),
func.count().label("sample_count"),
)
.where(DockerMetric.scraped_at < raw_cutoff)
.group_by(DockerMetric.host_id, DockerMetric.container_name, hour)
)
for r in (await session.execute(agg)).all():
stmt = (
pg_insert(DockerMetricHourly)
.values(
host_id=r.host_id,
container_name=r.container_name,
bucket=r.bucket,
cpu_pct=float(r.cpu_pct or 0.0),
mem_pct=float(r.mem_pct or 0.0),
mem_usage_bytes=int(r.mem_usage_bytes or 0),
sample_count=int(r.sample_count or 0),
)
.on_conflict_do_update(
constraint="uq_docker_metrics_hourly_bucket",
set_={
"cpu_pct": float(r.cpu_pct or 0.0),
"mem_pct": float(r.mem_pct or 0.0),
"mem_usage_bytes": int(r.mem_usage_bytes or 0),
"sample_count": int(r.sample_count or 0),
},
)
)
await session.execute(stmt)
rolled += 1
rolled_rows += int(r.sample_count or 0)
if rolled:
await session.execute(
delete(DockerMetric).where(DockerMetric.scraped_at < raw_cutoff)
)
# ── 2. Prune rolled-up rows beyond the rollup window ──
rollup_cutoff = now - timedelta(days=metrics_rollup_days)
res = await session.execute(
delete(DockerMetricHourly).where(DockerMetricHourly.bucket < rollup_cutoff)
delete(DockerEvent).where(DockerEvent.at < now - timedelta(days=events_days))
)
rollup_pruned = res.rowcount or 0
# ── 3. Prune lifecycle events beyond the events window ──
events_cutoff = now - timedelta(days=events_days)
res = await session.execute(
delete(DockerEvent).where(DockerEvent.at < events_cutoff)
)
events_pruned = res.rowcount or 0
return {
"buckets_rolled": rolled,
"raw_rows_rolled": rolled_rows,
"rollup_pruned": rollup_pruned,
"events_pruned": events_pruned,
}
return {"events_pruned": res.rowcount or 0}
async def prune_docker_logs(
+2 -6
View File
@@ -26,7 +26,7 @@ from .overview import (
stack_of,
)
from .models import (
DockerContainer, DockerDiskUsage, DockerHostStatus, DockerImage, DockerMetric,
DockerContainer, DockerDiskUsage, DockerHostStatus, DockerImage,
DockerSwarmNode, DockerSwarmService,
)
@@ -303,11 +303,7 @@ async def widget_resources():
# fill the list with duplicates of the same container.
running = dedup_by_container_id([
c for c in await docker_svc.list_containers(db) if c.status == "running"])
average = {(h, n): v for h, n, v in (await db.execute(
select(DockerMetric.host_id, DockerMetric.container_name, func.avg(DockerMetric.cpu_pct))
.where(DockerMetric.scraped_at >= since)
.group_by(DockerMetric.host_id, DockerMetric.container_name)
)).all()}
average = await docker_svc.average_cpu(db, {(c.host_id, c.name) for c in running}, since)
def cpu_of(c):
v = average.get((c.host_id, c.name))
+74 -56
View File
@@ -11,12 +11,13 @@ from datetime import datetime, timedelta, timezone
from sqlalchemy import func, select
from steward.core.time_range import bucket_seconds
from steward.core import metric_series
from steward.models.hosts import Host
from steward.models.metrics import PluginMetric
from .dedup import dedup_by_container_id
from .models import (
DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly, DockerSwarmService,
DockerContainer, DockerEvent, DockerLog, DockerSwarmService,
)
from .swarm_view import replaced_tasks
@@ -270,39 +271,65 @@ async def query_logs(db, host_id: str, name: str, stream: str, query: str,
CONTAINER_SERIES = ("cpu_pct", "mem_pct", "mem_usage_bytes")
# A container's series live in the one metric store (#5752): source "docker",
# resource "<host name>/<container name>", recorded at ingest.
def container_resource(host_name: str, name: str) -> str:
return f"{host_name}/{name}"
async def container_resources(db, keys) -> dict[str, tuple[str, str]]:
"""{resource: (host_id, container_name)} for (host_id, container_name)
keys whose host still exists."""
keys = set(keys)
hosts = await host_map(db, {h for h, _ in keys})
return {container_resource(hosts[h].name, n): (h, n) for h, n in keys if h in hosts}
async def container_points(db, host_id: str, name: str, metric: str, since,
target: int = 120) -> list[tuple]:
"""(time, average) points of one container metric since `since`, oldest first.
Raw samples are bucket-averaged in SQL; the part of the range older than
the oldest raw sample (already rolled up and pruned by retention) comes
from the hourly table.
"""
"""(time, average) points of one container metric since `since`, oldest
first: raw samples bucket-averaged, and the hourly rollup for the part of
the range past the raw window (metric_series.points)."""
if metric not in CONTAINER_SERIES:
raise ValueError(f"metric must be one of {', '.join(CONTAINER_SERIES)}")
raw_col = getattr(DockerMetric, metric)
width_s = bucket_seconds(since, target)
rbucket = func.date_bin(
func.make_interval(0, 0, 0, 0, 0, 0, width_s), DockerMetric.scraped_at, func.to_timestamp(0),
).label("bucket")
raw = (await db.execute(
select(rbucket, func.avg(raw_col))
.where(DockerMetric.host_id == host_id, DockerMetric.container_name == name,
DockerMetric.scraped_at >= since)
.group_by(rbucket).order_by(rbucket)
)).all()
oldest_raw = raw[0][0] if raw else None
hourly_stmt = (
select(DockerMetricHourly.bucket, getattr(DockerMetricHourly, metric))
.where(DockerMetricHourly.host_id == host_id,
DockerMetricHourly.container_name == name,
DockerMetricHourly.bucket >= since)
.order_by(DockerMetricHourly.bucket)
)
if oldest_raw is not None:
hourly_stmt = hourly_stmt.where(DockerMetricHourly.bucket < oldest_raw)
hourly = (await db.execute(hourly_stmt)).all()
return [(t, float(v)) for t, v in hourly] + [(t, float(v)) for t, v in raw]
host = await db.get(Host, host_id)
if host is None:
return []
return await metric_series.points(db, "docker", container_resource(host.name, name),
metric, since, target=target)
async def _samples(db, resources, metrics, since: datetime, width: int | None = None):
"""Rows of (resource, time, metric, value) for the given containers since
`since`: each sample, or with `width` (seconds) each bucket's average."""
at = PluginMetric.recorded_at
if width:
at = func.date_bin(func.make_interval(0, 0, 0, 0, 0, 0, int(width)),
PluginMetric.recorded_at, func.to_timestamp(0))
value = func.avg(PluginMetric.value) if width else PluginMetric.value
q = (select(PluginMetric.resource_name, at.label("at"), PluginMetric.metric_name, value)
.where(PluginMetric.source_module == "docker",
PluginMetric.resource_name.in_(list(resources)),
PluginMetric.metric_name.in_(list(metrics)),
PluginMetric.recorded_at >= since))
if width:
q = q.group_by(PluginMetric.resource_name, at, PluginMetric.metric_name)
return (await db.execute(q.order_by(at))).all()
async def average_cpu(db, keys, since: datetime) -> dict[tuple[str, str], float]:
"""{(host_id, container_name): its average cpu_pct since `since`}."""
resources = await container_resources(db, keys)
if not resources:
return {}
rows = (await db.execute(
select(PluginMetric.resource_name, func.avg(PluginMetric.value))
.where(PluginMetric.source_module == "docker",
PluginMetric.resource_name.in_(list(resources)),
PluginMetric.metric_name == "cpu_pct",
PluginMetric.recorded_at >= since)
.group_by(PluginMetric.resource_name))).all()
return {resources[r]: float(v) for r, v in rows if v is not None}
# Busiest Containers bars (#5214): the recent past in BARS equal slices.
@@ -322,25 +349,15 @@ def bar_slots(now: datetime, hours: int, bars: int = BARS) -> tuple[list[datetim
async def recent_usage(db, keys, since: datetime, width: int) -> dict:
"""{(host_id, container_name): {bucket start: (cpu_pct, mem_usage_bytes)}}
averaged per bucket since `since`, for the given containers, in one query."""
keys = set(keys)
if not keys:
resources = await container_resources(db, keys)
if not resources:
return {}
bucket = func.date_bin(
func.make_interval(0, 0, 0, 0, 0, 0, int(width)), DockerMetric.scraped_at, func.to_timestamp(0),
).label("bucket")
rows = (await db.execute(
select(DockerMetric.host_id, DockerMetric.container_name, bucket,
func.avg(DockerMetric.cpu_pct), func.avg(DockerMetric.mem_usage_bytes))
.where(DockerMetric.scraped_at >= since,
DockerMetric.host_id.in_({h for h, _ in keys}),
DockerMetric.container_name.in_({n for _, n in keys}))
.group_by(DockerMetric.host_id, DockerMetric.container_name, bucket)
)).all()
out: dict = {}
for host_id, name, at, cpu, mem in rows:
if (host_id, name) in keys:
out.setdefault((host_id, name), {})[at] = (
None if cpu is None else float(cpu), None if mem is None else float(mem))
for resource, at, metric, value in await _samples(
db, resources, ("cpu_pct", "mem_usage_bytes"), since, width):
cpu, mem = out.setdefault(resources[resource], {}).get(at, (None, None))
v = None if value is None else float(value)
out[resources[resource]][at] = (v, mem) if metric == "cpu_pct" else (cpu, v)
return out
@@ -523,15 +540,16 @@ def sum_series(samples, since: datetime, until: datetime, points: int = 60) -> l
async def service_history(db, containers, since: datetime) -> list:
"""sum_series over a service's task containers' samples since `since`."""
from sqlalchemy import tuple_
if not containers:
resources = await container_resources(db, containers)
if not resources:
return []
rows = (await db.execute(
select(DockerMetric.container_name, DockerMetric.scraped_at,
DockerMetric.cpu_pct, DockerMetric.mem_usage_bytes)
.where(tuple_(DockerMetric.host_id, DockerMetric.container_name).in_(list(containers)),
DockerMetric.scraped_at >= since)
.order_by(DockerMetric.scraped_at))).all()
joined: dict[tuple[str, datetime], list] = {}
for resource, at, metric, value in await _samples(
db, resources, ("cpu_pct", "mem_usage_bytes"), since):
pair = joined.setdefault((resource, at), [None, None])
pair[0 if metric == "cpu_pct" else 1] = value
rows = [(resource, at, cpu, mem) for (resource, at), (cpu, mem) in
sorted(joined.items(), key=lambda kv: kv[0][1])]
return sum_series(rows, since, datetime.now(timezone.utc))