feat(docker): a swarm service's metrics stay under one key across its tasks (#6718)

#6718 "A swarm container's metrics, normal and anomalies are keyed by its task
name, so they start over whenever the task moves". The operator: "the metrics
should be keys to the service name not the task name... as that's what's
consistent between updates."

CPU, memory, restart count and health now go under "swarm/<service>", so a
normal, its anomalies and an alert rule carry across a redeploy or a move
between nodes. Network rates stay "<host>/<service>", the task id gone, as they
answer who on a host is sending. Tasks of a multi-task service share the key,
so its level reads as an average across them (of 51 live services, only
authentik-worker and portainer_agent run more than one task).

History under task names moves in a background task, 5000 raw samples every
15 s, not a migration: the raw table holds about 2M such rows and the app
would wait on it at boot. Hourly rows combine where two tasks share an hour,
and rows still to review merge to one per metric.

A service's anomalies no longer reach a host's page, as it has no one host;
get_anomaly reads what happened around one from the service's own events.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/service.py

@@ -227,7 +227,8 @@ async def container_norms(db, keys, now: datetime | None = None) -> dict:
                          now or datetime.now(timezone.utc))
     out: dict = {}
     for (_s, resource, metric), norm in got.items():
-        out.setdefault(resources[resource], {})[metric] = norm
+        for key in resources[resource]:
+            out.setdefault(key, {})[metric] = norm
     return out
 
 
@@ -318,18 +319,43 @@ async def query_logs(db, host_id: str, name: str, stream: str, query: str,
 CONTAINER_SERIES = ("cpu_pct", "mem_used_pct", "mem_used_bytes", "mem_cache_bytes")
 
 
-# A container's series live in the one metric store (#5752): source "docker",
-# resource "<host name>/<container name>", recorded at ingest.
+# A swarm task's container: "<service>.<slot or node id>.<task id>".
+_TASK = re.compile(r"^(?P<service>.+?)\.(?:\d+|[a-z0-9]{25})\.[a-z0-9]{25}$")
+SERVICE_RESOURCE = "swarm/"
+
+
+def task_service(name: str) -> str | None:
+    """The swarm service a container is a task of, from its name, or None."""
+    m = _TASK.match(name or "")
+    return m["service"] if m else None
+
+
+# A container's series live in the one metric store: source "docker", recorded
+# at ingest. A swarm service's are under "swarm/<service>", which stays the same
+# when a task is replaced or moves host, and which all its tasks share. Any
+# other container's are under "<host name>/<container name>".
 def container_resource(host_name: str, name: str) -> str:
-    return f"{host_name}/{name}"
+    service = task_service(name)
+    return f"{SERVICE_RESOURCE}{service}" if service else f"{host_name}/{name}"
 
 
-async def container_resources(db, keys) -> dict[str, tuple[str, str]]:
-    """{resource: (host_id, container_name)} for (host_id, container_name)
-    keys whose host still exists."""
+def traffic_resource(host_name: str, name: str) -> str:
+    """Where a container's network rates are kept: "<host name>/<service or
+    container>", as they say who on that host is carrying its traffic."""
+    return f"{host_name}/{task_service(name) or name}"
+
+
+async def container_resources(db, keys) -> dict[str, list[tuple[str, str]]]:
+    """{resource: [(host_id, container_name), ...]} for (host_id,
+    container_name) keys whose host still exists. A swarm service's tasks share
+    one resource."""
     keys = set(keys)
     hosts = await host_map(db, {h for h, _ in keys})
-    return {container_resource(hosts[h].name, n): (h, n) for h, n in keys if h in hosts}
+    out: dict[str, list[tuple[str, str]]] = {}
+    for h, n in keys:
+        if h in hosts:
+            out.setdefault(container_resource(hosts[h].name, n), []).append((h, n))
+    return out
 
 
 async def container_points(db, host_id: str, name: str, metric: str, since,
@@ -376,7 +402,7 @@ async def average_cpu(db, keys, since: datetime) -> dict[tuple[str, str], float]
                PluginMetric.metric_name == "cpu_pct",
                PluginMetric.recorded_at >= since)
         .group_by(PluginMetric.resource_name))).all()
-    return {resources[r]: float(v) for r, v in rows if v is not None}
+    return {key: float(v) for r, v in rows if v is not None for key in resources[r]}
 
 
 # Busiest Containers bars (#5214): the recent past in BARS equal slices.
@@ -402,9 +428,10 @@ async def recent_usage(db, keys, since: datetime, width: int) -> dict:
     out: dict = {}
     for resource, at, metric, value in await _samples(
             db, resources, ("cpu_pct", "mem_used_bytes"), since, width):
-        cpu, mem = out.setdefault(resources[resource], {}).get(at, (None, None))
         v = None if value is None else float(value)
-        out[resources[resource]][at] = (v, mem) if metric == "cpu_pct" else (cpu, v)
+        for key in resources[resource]:
+            cpu, mem = out.setdefault(key, {}).get(at, (None, None))
+            out[key][at] = (v, mem) if metric == "cpu_pct" else (cpu, v)
     return out