refactor(docker): container CPU and memory live in the one metric store (#5752)
#5752 "Container CPU and memory history lives in the one metric store; docker_metrics and docker_metrics_hourly are retired". Container cpu_pct and mem_pct already went to plugin_metrics for alerting; now mem_usage_bytes does too, and every snapshot in an ingest (backlog included) is recorded at its own time. The container history chart, Busiest Containers, a service's history and the MCP container tools read plugin_metrics under resource "<host>/<container>", rolled up and pruned by the shared metric retention. docker_012_drop_metrics drops both tables. Their history is not copied (lesson #5221 "A data migration's cost is set by the live table"); the store already holds 7 days of container CPU and memory %. Docker retention now prunes only lifecycle events, and Settings -> Thresholds has one "Metric retention" card for every source. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -11,7 +11,7 @@ _app: "Quart | None" = None
|
||||
def setup(app: "Quart") -> None:
|
||||
global _app
|
||||
_app = app
|
||||
from .models import DockerContainer, DockerMetric # noqa: F401 — register with Base
|
||||
from .models import DockerContainer # noqa: F401 — register with Base
|
||||
|
||||
# Publish the per-host persist hook so the host agent can hand off the
|
||||
# `docker` array from each sample without importing our models (opportunistic
|
||||
@@ -28,12 +28,13 @@ def setup(app: "Quart") -> None:
|
||||
description="Store per-host container state + metrics pushed by the host agent.",
|
||||
required_role=UserRole.viewer,
|
||||
)
|
||||
# Roll up + prune Docker time-series, driven by the core cleanup task without
|
||||
# it importing our models. Same trusted server-side data-plane role as above.
|
||||
# Prune lifecycle events, driven by the core cleanup task without it
|
||||
# importing our models. Same trusted server-side data-plane role as above.
|
||||
# Container CPU and memory live in the one metric store and its retention.
|
||||
register_capability(
|
||||
"docker.run_retention", run_docker_retention,
|
||||
label="Run Docker retention",
|
||||
description="Roll up old docker_metrics to hourly + prune stale metrics/events.",
|
||||
description="Prune Docker lifecycle events past their window.",
|
||||
required_role=UserRole.viewer,
|
||||
)
|
||||
register_capability(
|
||||
|
||||
+18
-28
@@ -306,9 +306,10 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
|
||||
`snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
|
||||
one entry per ingested sample carrying docker data (usually one; more when
|
||||
the agent flushes a backlog). Every snapshot contributes time-series
|
||||
DockerMetric rows; the newest snapshot drives current container state, the
|
||||
alert pipeline, and lifecycle-event derivation. `swarm` is the newest
|
||||
the agent flushes a backlog). Every snapshot's running containers record
|
||||
cpu_pct, mem_pct and mem_usage_bytes into the one metric store, at the
|
||||
snapshot's time; the newest snapshot drives current container state and
|
||||
lifecycle-event derivation. `swarm` is the newest
|
||||
sample's swarm object (or None off managers) — persisted when present.
|
||||
`disk` is the newest sample's /system/df summary (or None on Docker-less
|
||||
hosts) — persisted when present. `logs` is a list of (recorded_at, records)
|
||||
@@ -323,7 +324,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
flushed before its mark, so a slow ingest names the part (#5181).
|
||||
"""
|
||||
from steward.core.alerts import MetricBatch
|
||||
from .models import DockerContainer, DockerEvent, DockerMetric
|
||||
from .models import DockerContainer, DockerEvent
|
||||
|
||||
async def mark(name: str) -> None:
|
||||
if phases is not None:
|
||||
@@ -350,18 +351,20 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
ordered = sorted(snapshots, key=lambda s: s[0])
|
||||
latest_at, latest_containers = ordered[-1]
|
||||
|
||||
# Time-series points for every snapshot (running containers with a CPU read).
|
||||
# Time series for every snapshot (running containers with a CPU read), in
|
||||
# the one metric store (#5752): resource "<host>/<container>", so
|
||||
# containers of the same name on different hosts don't collide. Recorded
|
||||
# with the alert batch below, at each snapshot's own time.
|
||||
batch = MetricBatch()
|
||||
for recorded_at, containers in ordered:
|
||||
for c in containers:
|
||||
if c.get("status") == "running" and c.get("cpu_pct") is not None:
|
||||
session.add(DockerMetric(
|
||||
host_id=host.id,
|
||||
container_name=c["name"],
|
||||
scraped_at=recorded_at,
|
||||
cpu_pct=c["cpu_pct"],
|
||||
mem_pct=c.get("mem_pct") or 0.0,
|
||||
mem_usage_bytes=c.get("mem_usage_bytes") or 0,
|
||||
))
|
||||
if c.get("status") != "running" or c.get("cpu_pct") is None or not c.get("name"):
|
||||
continue
|
||||
resource = f"{host.name}/{c['name']}"
|
||||
for metric in ("cpu_pct", "mem_pct", "mem_usage_bytes"):
|
||||
if c.get(metric) is not None:
|
||||
batch.add(source_module="docker", resource_name=resource,
|
||||
metric_name=metric, value=c[metric], recorded_at=recorded_at)
|
||||
await mark("metrics")
|
||||
|
||||
# Snapshot of stored per-container state BEFORE the upsert overwrites it —
|
||||
@@ -389,8 +392,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
))
|
||||
await mark("events")
|
||||
|
||||
# Current state + alerts from the newest snapshot only.
|
||||
batch = MetricBatch()
|
||||
# Current state, and the restart and health alerts, from the newest snapshot only.
|
||||
for c in latest_containers:
|
||||
name = c.get("name")
|
||||
if not name:
|
||||
@@ -424,19 +426,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
existing.blk_read_bytes = c.get("blk_read_bytes")
|
||||
existing.blk_write_bytes = c.get("blk_write_bytes")
|
||||
|
||||
# Alert pipeline — resource is host-scoped so containers of the same name
|
||||
# on different hosts don't collide in the metric/alert namespace.
|
||||
resource = f"{host.name}/{name}"
|
||||
if c.get("status") == "running" and c.get("cpu_pct") is not None:
|
||||
batch.add(
|
||||
source_module="docker",
|
||||
resource_name=resource, metric_name="cpu_pct", value=c["cpu_pct"],
|
||||
)
|
||||
if c.get("mem_pct") is not None:
|
||||
batch.add(
|
||||
source_module="docker",
|
||||
resource_name=resource, metric_name="mem_pct", value=c["mem_pct"],
|
||||
)
|
||||
# Restart count is alertable regardless of state (crash-looping matters
|
||||
# most while the container is down/restarting).
|
||||
batch.add(
|
||||
|
||||
@@ -15,7 +15,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
|
||||
|
||||
from steward.models.base import Base
|
||||
import steward.models # noqa: F401
|
||||
from plugins.docker.models import DockerContainer, DockerMetric # noqa: F401
|
||||
from plugins.docker.models import DockerContainer # noqa: F401
|
||||
|
||||
config = context.config
|
||||
if config.config_file_name is not None:
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Container CPU and memory move to the one metric store
|
||||
|
||||
Drops docker_metrics and docker_metrics_hourly (#5752). Container cpu_pct,
|
||||
mem_pct and mem_usage_bytes are recorded into plugin_metrics as source
|
||||
"docker", resource "<host>/<container>", and roll up with every other metric.
|
||||
CPU and memory % were already recorded there for alerting, so their history
|
||||
stays; memory in bytes starts fresh.
|
||||
|
||||
Revision ID: docker_012_drop_metrics
|
||||
Revises: docker_011_host_status
|
||||
Create Date: 2026-10-14
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "docker_012_drop_metrics"
|
||||
down_revision: Union[str, None] = "docker_011_host_status"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("DROP TABLE IF EXISTS docker_metrics_hourly")
|
||||
op.execute("DROP TABLE IF EXISTS docker_metrics")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.create_table(
|
||||
"docker_metrics",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("host_id", sa.String(36), sa.ForeignKey("hosts.id", ondelete="CASCADE"),
|
||||
nullable=False, index=True),
|
||||
sa.Column("container_name", sa.String(255), nullable=False, index=True),
|
||||
sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False, index=True),
|
||||
sa.Column("cpu_pct", sa.Float, nullable=False),
|
||||
sa.Column("mem_pct", sa.Float, nullable=False),
|
||||
sa.Column("mem_usage_bytes", sa.BigInteger, nullable=False),
|
||||
)
|
||||
op.create_index("ix_docker_metrics_host_container_time", "docker_metrics",
|
||||
["host_id", "container_name", "scraped_at"])
|
||||
op.create_table(
|
||||
"docker_metrics_hourly",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("host_id", sa.String(36), sa.ForeignKey("hosts.id", ondelete="CASCADE"),
|
||||
nullable=False),
|
||||
sa.Column("container_name", sa.String(255), nullable=False),
|
||||
sa.Column("bucket", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("cpu_pct", sa.Float, nullable=False),
|
||||
sa.Column("mem_pct", sa.Float, nullable=False),
|
||||
sa.Column("mem_usage_bytes", sa.BigInteger, nullable=False),
|
||||
sa.Column("sample_count", sa.Integer, nullable=False),
|
||||
sa.UniqueConstraint("host_id", "container_name", "bucket",
|
||||
name="uq_docker_metrics_hourly_bucket"),
|
||||
)
|
||||
op.create_index("ix_docker_metrics_hourly_bucket", "docker_metrics_hourly", ["bucket"])
|
||||
@@ -3,7 +3,6 @@ from __future__ import annotations
|
||||
from datetime import datetime, timezone
|
||||
from sqlalchemy import (
|
||||
BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, String, Text,
|
||||
UniqueConstraint,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from steward.models.base import Base
|
||||
@@ -61,69 +60,6 @@ class DockerContainer(Base):
|
||||
blk_write_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
|
||||
|
||||
class DockerMetric(Base):
|
||||
"""Time-series CPU/memory per container — one row per sample per running
|
||||
container, scoped to the reporting host."""
|
||||
__tablename__ = "docker_metrics"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36), primary_key=True, default=time_ordered_id
|
||||
)
|
||||
host_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
container_name: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
scraped_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False,
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
index=True,
|
||||
)
|
||||
cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||
mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||
# BigInteger: per-container memory usage exceeds 2^31 bytes (~2.1 GB) routinely.
|
||||
mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
|
||||
# Per-container history lookups filter on (host_id, container_name) then sort
|
||||
# by time — a composite index keeps the rows() sparkline queries cheap.
|
||||
__table_args__ = (
|
||||
Index("ix_docker_metrics_host_container_time",
|
||||
"host_id", "container_name", "scraped_at"),
|
||||
)
|
||||
|
||||
|
||||
class DockerMetricHourly(Base):
|
||||
"""Hourly rollup of docker_metrics — avg cpu/mem per container per hour.
|
||||
|
||||
Raw per-sample rows (~2880/container/day at 30s) are pruned beyond a short
|
||||
window; before deletion they're aggregated here so multi-day history stays
|
||||
cheap to store and query. One row per (host, container, hour bucket); the
|
||||
unique constraint lets retention upsert idempotently if it re-runs before the
|
||||
raw rows are deleted. `bucket` is the hour-truncated sample time.
|
||||
"""
|
||||
__tablename__ = "docker_metrics_hourly"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36), primary_key=True, default=time_ordered_id
|
||||
)
|
||||
host_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("hosts.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
container_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||
mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||
mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
|
||||
__table_args__ = (
|
||||
# One bucket per container per host — the conflict target for the
|
||||
# idempotent rollup upsert; doubles as the history-query index.
|
||||
UniqueConstraint("host_id", "container_name", "bucket",
|
||||
name="uq_docker_metrics_hourly_bucket"),
|
||||
Index("ix_docker_metrics_hourly_bucket", "bucket"),
|
||||
)
|
||||
|
||||
|
||||
class DockerEvent(Base):
|
||||
"""Lifecycle events derived by diffing consecutive host snapshots.
|
||||
|
||||
@@ -167,7 +103,7 @@ class DockerLog(Base):
|
||||
"""Container log lines pushed by the host agent — one row per line.
|
||||
|
||||
Time-series, scoped to the reporting host (container names are only unique
|
||||
within a host, same identity as docker_metrics). The agent tails each running
|
||||
within a host, same identity as docker_containers). The agent tails each running
|
||||
container incrementally and folds new lines into its metrics push; `ts` is the
|
||||
line's own Docker timestamp. Bounded by a per-container size+age ring
|
||||
(retention), so a chatty container just keeps a shorter window rather than
|
||||
|
||||
+15
-107
@@ -1,133 +1,41 @@
|
||||
# plugins/docker/retention.py
|
||||
"""Bound Docker time-series growth: roll up old metrics, prune old rows.
|
||||
"""Bound Docker history growth: prune old lifecycle events and log lines.
|
||||
|
||||
Published as the "docker.run_retention" capability (see __init__.setup) so the
|
||||
core cleanup task can drive it WITHOUT importing the docker models (same
|
||||
opportunistic-coupling pattern as docker.persist_host_samples). Runs inside the
|
||||
caller's open transaction; never opens or commits its own.
|
||||
Published as the "docker.run_retention" and "docker.prune_logs" capabilities
|
||||
(see __init__.setup) so the core cleanup task can drive them WITHOUT importing
|
||||
the docker models (same opportunistic-coupling pattern as
|
||||
docker.persist_host_samples). Each runs inside the caller's open transaction;
|
||||
never opens or commits its own.
|
||||
|
||||
The scaling concern is docker_metrics: ~2880 rows/container/day at a 30s sample.
|
||||
We keep raw samples for a short window, then aggregate everything older into
|
||||
hourly averages (docker_metrics_hourly) and delete the raw rows — so multi-day
|
||||
history stays cheap to store and query. docker_events is light but unbounded
|
||||
without a cutoff, so it gets a (longer) window too.
|
||||
Container CPU and memory are not here: they live in the one metric store
|
||||
(plugin_metrics) and its rollup and retention (#5752).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
|
||||
def _hour_floor(dt: datetime) -> datetime:
|
||||
"""Truncate a datetime down to the start of its hour (drops min/sec/µs)."""
|
||||
return dt.replace(minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
def _rollup_cutoff(now: datetime, raw_days: int) -> datetime:
|
||||
"""Hour-aligned boundary below which raw metrics get rolled up + deleted.
|
||||
|
||||
Aligning to the hour means we only ever roll up *whole* elapsed hours — a
|
||||
bucket is never split across the keep/roll boundary, so re-running can't
|
||||
produce a partial-then-complete duplicate for the same hour.
|
||||
"""
|
||||
return _hour_floor(now - timedelta(days=raw_days))
|
||||
|
||||
|
||||
async def run_docker_retention(
|
||||
session,
|
||||
*,
|
||||
events_days: int,
|
||||
metrics_raw_days: int,
|
||||
metrics_rollup_days: int,
|
||||
now: datetime | None = None,
|
||||
) -> dict:
|
||||
"""Roll up + prune Docker time-series. Returns a counts dict for logging.
|
||||
|
||||
1. Aggregate docker_metrics older than the (hour-aligned) raw window into
|
||||
docker_metrics_hourly (avg cpu/mem per container per hour), upserting so a
|
||||
re-run is idempotent, then delete those raw rows.
|
||||
2. Prune rolled-up rows older than the rollup window.
|
||||
3. Prune docker_events older than the events window.
|
||||
|
||||
The container-log ring is prune_docker_logs, run in its own transaction.
|
||||
"""
|
||||
"""Prune docker_events older than the events window. Returns a counts
|
||||
dict for logging. The container-log ring is prune_docker_logs, run in its
|
||||
own transaction."""
|
||||
from datetime import timezone
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy import delete
|
||||
|
||||
from .models import DockerEvent, DockerMetric, DockerMetricHourly
|
||||
from .models import DockerEvent
|
||||
|
||||
if now is None:
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
rolled = rolled_rows = events_pruned = rollup_pruned = 0
|
||||
|
||||
# ── 1. Roll up raw metrics older than the raw window into hourly buckets ──
|
||||
raw_cutoff = _rollup_cutoff(now, metrics_raw_days)
|
||||
hour = func.date_trunc("hour", DockerMetric.scraped_at)
|
||||
agg = (
|
||||
select(
|
||||
DockerMetric.host_id,
|
||||
DockerMetric.container_name,
|
||||
hour.label("bucket"),
|
||||
func.avg(DockerMetric.cpu_pct).label("cpu_pct"),
|
||||
func.avg(DockerMetric.mem_pct).label("mem_pct"),
|
||||
func.avg(DockerMetric.mem_usage_bytes).label("mem_usage_bytes"),
|
||||
func.count().label("sample_count"),
|
||||
)
|
||||
.where(DockerMetric.scraped_at < raw_cutoff)
|
||||
.group_by(DockerMetric.host_id, DockerMetric.container_name, hour)
|
||||
)
|
||||
for r in (await session.execute(agg)).all():
|
||||
stmt = (
|
||||
pg_insert(DockerMetricHourly)
|
||||
.values(
|
||||
host_id=r.host_id,
|
||||
container_name=r.container_name,
|
||||
bucket=r.bucket,
|
||||
cpu_pct=float(r.cpu_pct or 0.0),
|
||||
mem_pct=float(r.mem_pct or 0.0),
|
||||
mem_usage_bytes=int(r.mem_usage_bytes or 0),
|
||||
sample_count=int(r.sample_count or 0),
|
||||
)
|
||||
.on_conflict_do_update(
|
||||
constraint="uq_docker_metrics_hourly_bucket",
|
||||
set_={
|
||||
"cpu_pct": float(r.cpu_pct or 0.0),
|
||||
"mem_pct": float(r.mem_pct or 0.0),
|
||||
"mem_usage_bytes": int(r.mem_usage_bytes or 0),
|
||||
"sample_count": int(r.sample_count or 0),
|
||||
},
|
||||
)
|
||||
)
|
||||
await session.execute(stmt)
|
||||
rolled += 1
|
||||
rolled_rows += int(r.sample_count or 0)
|
||||
if rolled:
|
||||
await session.execute(
|
||||
delete(DockerMetric).where(DockerMetric.scraped_at < raw_cutoff)
|
||||
)
|
||||
|
||||
# ── 2. Prune rolled-up rows beyond the rollup window ──
|
||||
rollup_cutoff = now - timedelta(days=metrics_rollup_days)
|
||||
res = await session.execute(
|
||||
delete(DockerMetricHourly).where(DockerMetricHourly.bucket < rollup_cutoff)
|
||||
delete(DockerEvent).where(DockerEvent.at < now - timedelta(days=events_days))
|
||||
)
|
||||
rollup_pruned = res.rowcount or 0
|
||||
|
||||
# ── 3. Prune lifecycle events beyond the events window ──
|
||||
events_cutoff = now - timedelta(days=events_days)
|
||||
res = await session.execute(
|
||||
delete(DockerEvent).where(DockerEvent.at < events_cutoff)
|
||||
)
|
||||
events_pruned = res.rowcount or 0
|
||||
|
||||
return {
|
||||
"buckets_rolled": rolled,
|
||||
"raw_rows_rolled": rolled_rows,
|
||||
"rollup_pruned": rollup_pruned,
|
||||
"events_pruned": events_pruned,
|
||||
}
|
||||
return {"events_pruned": res.rowcount or 0}
|
||||
|
||||
|
||||
async def prune_docker_logs(
|
||||
|
||||
@@ -26,7 +26,7 @@ from .overview import (
|
||||
stack_of,
|
||||
)
|
||||
from .models import (
|
||||
DockerContainer, DockerDiskUsage, DockerHostStatus, DockerImage, DockerMetric,
|
||||
DockerContainer, DockerDiskUsage, DockerHostStatus, DockerImage,
|
||||
DockerSwarmNode, DockerSwarmService,
|
||||
)
|
||||
|
||||
@@ -303,11 +303,7 @@ async def widget_resources():
|
||||
# fill the list with duplicates of the same container.
|
||||
running = dedup_by_container_id([
|
||||
c for c in await docker_svc.list_containers(db) if c.status == "running"])
|
||||
average = {(h, n): v for h, n, v in (await db.execute(
|
||||
select(DockerMetric.host_id, DockerMetric.container_name, func.avg(DockerMetric.cpu_pct))
|
||||
.where(DockerMetric.scraped_at >= since)
|
||||
.group_by(DockerMetric.host_id, DockerMetric.container_name)
|
||||
)).all()}
|
||||
average = await docker_svc.average_cpu(db, {(c.host_id, c.name) for c in running}, since)
|
||||
|
||||
def cpu_of(c):
|
||||
v = average.get((c.host_id, c.name))
|
||||
|
||||
+74
-56
@@ -11,12 +11,13 @@ from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from steward.core.time_range import bucket_seconds
|
||||
from steward.core import metric_series
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.metrics import PluginMetric
|
||||
|
||||
from .dedup import dedup_by_container_id
|
||||
from .models import (
|
||||
DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly, DockerSwarmService,
|
||||
DockerContainer, DockerEvent, DockerLog, DockerSwarmService,
|
||||
)
|
||||
from .swarm_view import replaced_tasks
|
||||
|
||||
@@ -270,39 +271,65 @@ async def query_logs(db, host_id: str, name: str, stream: str, query: str,
|
||||
CONTAINER_SERIES = ("cpu_pct", "mem_pct", "mem_usage_bytes")
|
||||
|
||||
|
||||
# A container's series live in the one metric store (#5752): source "docker",
|
||||
# resource "<host name>/<container name>", recorded at ingest.
|
||||
def container_resource(host_name: str, name: str) -> str:
|
||||
return f"{host_name}/{name}"
|
||||
|
||||
|
||||
async def container_resources(db, keys) -> dict[str, tuple[str, str]]:
|
||||
"""{resource: (host_id, container_name)} for (host_id, container_name)
|
||||
keys whose host still exists."""
|
||||
keys = set(keys)
|
||||
hosts = await host_map(db, {h for h, _ in keys})
|
||||
return {container_resource(hosts[h].name, n): (h, n) for h, n in keys if h in hosts}
|
||||
|
||||
|
||||
async def container_points(db, host_id: str, name: str, metric: str, since,
|
||||
target: int = 120) -> list[tuple]:
|
||||
"""(time, average) points of one container metric since `since`, oldest first.
|
||||
|
||||
Raw samples are bucket-averaged in SQL; the part of the range older than
|
||||
the oldest raw sample (already rolled up and pruned by retention) comes
|
||||
from the hourly table.
|
||||
"""
|
||||
"""(time, average) points of one container metric since `since`, oldest
|
||||
first: raw samples bucket-averaged, and the hourly rollup for the part of
|
||||
the range past the raw window (metric_series.points)."""
|
||||
if metric not in CONTAINER_SERIES:
|
||||
raise ValueError(f"metric must be one of {', '.join(CONTAINER_SERIES)}")
|
||||
raw_col = getattr(DockerMetric, metric)
|
||||
width_s = bucket_seconds(since, target)
|
||||
rbucket = func.date_bin(
|
||||
func.make_interval(0, 0, 0, 0, 0, 0, width_s), DockerMetric.scraped_at, func.to_timestamp(0),
|
||||
).label("bucket")
|
||||
raw = (await db.execute(
|
||||
select(rbucket, func.avg(raw_col))
|
||||
.where(DockerMetric.host_id == host_id, DockerMetric.container_name == name,
|
||||
DockerMetric.scraped_at >= since)
|
||||
.group_by(rbucket).order_by(rbucket)
|
||||
)).all()
|
||||
oldest_raw = raw[0][0] if raw else None
|
||||
hourly_stmt = (
|
||||
select(DockerMetricHourly.bucket, getattr(DockerMetricHourly, metric))
|
||||
.where(DockerMetricHourly.host_id == host_id,
|
||||
DockerMetricHourly.container_name == name,
|
||||
DockerMetricHourly.bucket >= since)
|
||||
.order_by(DockerMetricHourly.bucket)
|
||||
)
|
||||
if oldest_raw is not None:
|
||||
hourly_stmt = hourly_stmt.where(DockerMetricHourly.bucket < oldest_raw)
|
||||
hourly = (await db.execute(hourly_stmt)).all()
|
||||
return [(t, float(v)) for t, v in hourly] + [(t, float(v)) for t, v in raw]
|
||||
host = await db.get(Host, host_id)
|
||||
if host is None:
|
||||
return []
|
||||
return await metric_series.points(db, "docker", container_resource(host.name, name),
|
||||
metric, since, target=target)
|
||||
|
||||
|
||||
async def _samples(db, resources, metrics, since: datetime, width: int | None = None):
|
||||
"""Rows of (resource, time, metric, value) for the given containers since
|
||||
`since`: each sample, or with `width` (seconds) each bucket's average."""
|
||||
at = PluginMetric.recorded_at
|
||||
if width:
|
||||
at = func.date_bin(func.make_interval(0, 0, 0, 0, 0, 0, int(width)),
|
||||
PluginMetric.recorded_at, func.to_timestamp(0))
|
||||
value = func.avg(PluginMetric.value) if width else PluginMetric.value
|
||||
q = (select(PluginMetric.resource_name, at.label("at"), PluginMetric.metric_name, value)
|
||||
.where(PluginMetric.source_module == "docker",
|
||||
PluginMetric.resource_name.in_(list(resources)),
|
||||
PluginMetric.metric_name.in_(list(metrics)),
|
||||
PluginMetric.recorded_at >= since))
|
||||
if width:
|
||||
q = q.group_by(PluginMetric.resource_name, at, PluginMetric.metric_name)
|
||||
return (await db.execute(q.order_by(at))).all()
|
||||
|
||||
|
||||
async def average_cpu(db, keys, since: datetime) -> dict[tuple[str, str], float]:
|
||||
"""{(host_id, container_name): its average cpu_pct since `since`}."""
|
||||
resources = await container_resources(db, keys)
|
||||
if not resources:
|
||||
return {}
|
||||
rows = (await db.execute(
|
||||
select(PluginMetric.resource_name, func.avg(PluginMetric.value))
|
||||
.where(PluginMetric.source_module == "docker",
|
||||
PluginMetric.resource_name.in_(list(resources)),
|
||||
PluginMetric.metric_name == "cpu_pct",
|
||||
PluginMetric.recorded_at >= since)
|
||||
.group_by(PluginMetric.resource_name))).all()
|
||||
return {resources[r]: float(v) for r, v in rows if v is not None}
|
||||
|
||||
|
||||
# Busiest Containers bars (#5214): the recent past in BARS equal slices.
|
||||
@@ -322,25 +349,15 @@ def bar_slots(now: datetime, hours: int, bars: int = BARS) -> tuple[list[datetim
|
||||
async def recent_usage(db, keys, since: datetime, width: int) -> dict:
|
||||
"""{(host_id, container_name): {bucket start: (cpu_pct, mem_usage_bytes)}}
|
||||
averaged per bucket since `since`, for the given containers, in one query."""
|
||||
keys = set(keys)
|
||||
if not keys:
|
||||
resources = await container_resources(db, keys)
|
||||
if not resources:
|
||||
return {}
|
||||
bucket = func.date_bin(
|
||||
func.make_interval(0, 0, 0, 0, 0, 0, int(width)), DockerMetric.scraped_at, func.to_timestamp(0),
|
||||
).label("bucket")
|
||||
rows = (await db.execute(
|
||||
select(DockerMetric.host_id, DockerMetric.container_name, bucket,
|
||||
func.avg(DockerMetric.cpu_pct), func.avg(DockerMetric.mem_usage_bytes))
|
||||
.where(DockerMetric.scraped_at >= since,
|
||||
DockerMetric.host_id.in_({h for h, _ in keys}),
|
||||
DockerMetric.container_name.in_({n for _, n in keys}))
|
||||
.group_by(DockerMetric.host_id, DockerMetric.container_name, bucket)
|
||||
)).all()
|
||||
out: dict = {}
|
||||
for host_id, name, at, cpu, mem in rows:
|
||||
if (host_id, name) in keys:
|
||||
out.setdefault((host_id, name), {})[at] = (
|
||||
None if cpu is None else float(cpu), None if mem is None else float(mem))
|
||||
for resource, at, metric, value in await _samples(
|
||||
db, resources, ("cpu_pct", "mem_usage_bytes"), since, width):
|
||||
cpu, mem = out.setdefault(resources[resource], {}).get(at, (None, None))
|
||||
v = None if value is None else float(value)
|
||||
out[resources[resource]][at] = (v, mem) if metric == "cpu_pct" else (cpu, v)
|
||||
return out
|
||||
|
||||
|
||||
@@ -523,15 +540,16 @@ def sum_series(samples, since: datetime, until: datetime, points: int = 60) -> l
|
||||
|
||||
async def service_history(db, containers, since: datetime) -> list:
|
||||
"""sum_series over a service's task containers' samples since `since`."""
|
||||
from sqlalchemy import tuple_
|
||||
if not containers:
|
||||
resources = await container_resources(db, containers)
|
||||
if not resources:
|
||||
return []
|
||||
rows = (await db.execute(
|
||||
select(DockerMetric.container_name, DockerMetric.scraped_at,
|
||||
DockerMetric.cpu_pct, DockerMetric.mem_usage_bytes)
|
||||
.where(tuple_(DockerMetric.host_id, DockerMetric.container_name).in_(list(containers)),
|
||||
DockerMetric.scraped_at >= since)
|
||||
.order_by(DockerMetric.scraped_at))).all()
|
||||
joined: dict[tuple[str, datetime], list] = {}
|
||||
for resource, at, metric, value in await _samples(
|
||||
db, resources, ("cpu_pct", "mem_usage_bytes"), since):
|
||||
pair = joined.setdefault((resource, at), [None, None])
|
||||
pair[0 if metric == "cpu_pct" else 1] = value
|
||||
rows = [(resource, at, cpu, mem) for (resource, at), (cpu, mem) in
|
||||
sorted(joined.items(), key=lambda kv: kv[0][1])]
|
||||
return sum_series(rows, since, datetime.now(timezone.utc))
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user