feat(docker): a swarm service's metrics stay under one key across its tasks (#6718)
#6718 "A swarm container's metrics, normal and anomalies are keyed by its task name, so they start over whenever the task moves". The operator: "the metrics should be keys to the service name not the task name... as that's what's consistent between updates." CPU, memory, restart count and health now go under "swarm/<service>", so a normal, its anomalies and an alert rule carry across a redeploy or a move between nodes. Network rates stay "<host>/<service>", the task id gone, as they answer who on a host is sending. Tasks of a multi-task service share the key, so its level reads as an average across them (of 51 live services, only authentik-worker and portainer_agent run more than one task). History under task names moves in a background task, 5000 raw samples every 15 s, not a migration: the raw table holds about 2M such rows and the app would wait on it at boot. Hourly rows combine where two tasks share an hour, and rows still to review merge to one per metric. A service's anomalies no longer reach a host's page, as it has no one host; get_anomaly reads what happened around one from the service's own events. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -404,6 +404,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
"""
|
||||
from steward.core.alerts import MetricBatch
|
||||
from .models import DockerContainer, DockerEvent
|
||||
from .service import container_resource, traffic_resource
|
||||
|
||||
async def mark(name: str) -> None:
|
||||
if phases is not None:
|
||||
@@ -432,9 +433,11 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
latest_at, latest_containers = ordered[-1]
|
||||
|
||||
# Time series for every snapshot (running containers with a CPU read), in
|
||||
# the one metric store (#5752): resource "<host>/<container>", so
|
||||
# containers of the same name on different hosts don't collide. Recorded
|
||||
# with the alert batch below, at each snapshot's own time.
|
||||
# the one metric store: resource "swarm/<service>" for a swarm task, so its
|
||||
# history survives the task being replaced or moved, else
|
||||
# "<host>/<container>", so containers of the same name on different hosts
|
||||
# don't collide. Recorded with the alert batch below, at each snapshot's
|
||||
# own time.
|
||||
# Snapshot of stored per-container state BEFORE the upsert overwrites it —
|
||||
# the network counters each rate counts from, and the lifecycle-event diff.
|
||||
# Skip event derivation on the host's first snapshot (empty state) so we
|
||||
@@ -448,7 +451,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
for c in containers:
|
||||
if c.get("status") != "running" or c.get("cpu_pct") is None or not c.get("name"):
|
||||
continue
|
||||
resource = f"{host.name}/{c['name']}"
|
||||
resource = container_resource(host.name, c["name"])
|
||||
for metric in ("cpu_pct", "mem_used_pct", "mem_used_bytes", "mem_cache_bytes"):
|
||||
if c.get(metric) is not None:
|
||||
batch.add(source_module="docker", resource_name=resource,
|
||||
@@ -457,7 +460,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
# can be put down to the container behind it (#6115).
|
||||
before = {r.name: (r.scraped_at, r.container_id, r.net_rx_bytes, r.net_tx_bytes) for r in old_rows}
|
||||
for recorded_at, name, rx_bps, tx_bps in net_rates(before, ordered):
|
||||
resource = f"{host.name}/{name}"
|
||||
resource = traffic_resource(host.name, name)
|
||||
batch.add(source_module="docker", resource_name=resource,
|
||||
metric_name="net_rx_bps", value=round(rx_bps, 1), recorded_at=recorded_at)
|
||||
batch.add(source_module="docker", resource_name=resource,
|
||||
@@ -524,7 +527,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
|
||||
existing.blk_read_bytes = c.get("blk_read_bytes")
|
||||
existing.blk_write_bytes = c.get("blk_write_bytes")
|
||||
|
||||
resource = f"{host.name}/{name}"
|
||||
resource = container_resource(host.name, name)
|
||||
# Restart count is alertable regardless of state (crash-looping matters
|
||||
# most while the container is down/restarting).
|
||||
batch.add(
|
||||
|
||||
Reference in New Issue
Block a user