feat(docker): a swarm service's metrics stay under one key across its tasks (#6718)

#6718 "A swarm container's metrics, normal and anomalies are keyed by its task
name, so they start over whenever the task moves". The operator: "the metrics
should be keys to the service name not the task name... as that's what's
consistent between updates."

CPU, memory, restart count and health now go under "swarm/<service>", so a
normal, its anomalies and an alert rule carry across a redeploy or a move
between nodes. Network rates stay "<host>/<service>", the task id gone, as they
answer who on a host is sending. Tasks of a multi-task service share the key,
so its level reads as an average across them (of 51 live services, only
authentik-worker and portainer_agent run more than one task).

History under task names moves in a background task, 5000 raw samples every
15 s, not a migration: the raw table holds about 2M such rows and the app
would wait on it at boot. Hourly rows combine where two tasks share an hour,
and rows still to review merge to one per metric.

A service's anomalies no longer reach a host's page, as it has no one host;
get_anomaly reads what happened around one from the service's own events.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-30 19:45:16 -04:00
co-authored by Claude Opus 5.5
parent 18191aa858
commit db39fc2604
11 changed files with 427 additions and 33 deletions
+9 -6
View File
@@ -404,6 +404,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
"""
from steward.core.alerts import MetricBatch
from .models import DockerContainer, DockerEvent
from .service import container_resource, traffic_resource
async def mark(name: str) -> None:
if phases is not None:
@@ -432,9 +433,11 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
latest_at, latest_containers = ordered[-1]
# Time series for every snapshot (running containers with a CPU read), in
# the one metric store (#5752): resource "<host>/<container>", so
# containers of the same name on different hosts don't collide. Recorded
# with the alert batch below, at each snapshot's own time.
# the one metric store: resource "swarm/<service>" for a swarm task, so its
# history survives the task being replaced or moved, else
# "<host>/<container>", so containers of the same name on different hosts
# don't collide. Recorded with the alert batch below, at each snapshot's
# own time.
# Snapshot of stored per-container state BEFORE the upsert overwrites it —
# the network counters each rate counts from, and the lifecycle-event diff.
# Skip event derivation on the host's first snapshot (empty state) so we
@@ -448,7 +451,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
for c in containers:
if c.get("status") != "running" or c.get("cpu_pct") is None or not c.get("name"):
continue
resource = f"{host.name}/{c['name']}"
resource = container_resource(host.name, c["name"])
for metric in ("cpu_pct", "mem_used_pct", "mem_used_bytes", "mem_cache_bytes"):
if c.get(metric) is not None:
batch.add(source_module="docker", resource_name=resource,
@@ -457,7 +460,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
# can be put down to the container behind it (#6115).
before = {r.name: (r.scraped_at, r.container_id, r.net_rx_bytes, r.net_tx_bytes) for r in old_rows}
for recorded_at, name, rx_bps, tx_bps in net_rates(before, ordered):
resource = f"{host.name}/{name}"
resource = traffic_resource(host.name, name)
batch.add(source_module="docker", resource_name=resource,
metric_name="net_rx_bps", value=round(rx_bps, 1), recorded_at=recorded_at)
batch.add(source_module="docker", resource_name=resource,
@@ -524,7 +527,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
existing.blk_read_bytes = c.get("blk_read_bytes")
existing.blk_write_bytes = c.get("blk_write_bytes")
resource = f"{host.name}/{name}"
resource = container_resource(host.name, name)
# Restart count is alertable regardless of state (crash-looping matters
# most while the container is down/restarting).
batch.add(