feat(docker): a swarm service's metrics stay under one key across its tasks (#6718)

#6718 "A swarm container's metrics, normal and anomalies are keyed by its task
name, so they start over whenever the task moves". The operator: "the metrics
should be keys to the service name not the task name... as that's what's
consistent between updates."

CPU, memory, restart count and health now go under "swarm/<service>", so a
normal, its anomalies and an alert rule carry across a redeploy or a move
between nodes. Network rates stay "<host>/<service>", the task id gone, as they
answer who on a host is sending. Tasks of a multi-task service share the key,
so its level reads as an average across them (of 51 live services, only
authentik-worker and portainer_agent run more than one task).

History under task names moves in a background task, 5000 raw samples every
15 s, not a migration: the raw table holds about 2M such rows and the app
would wait on it at boot. Hourly rows combine where two tasks share an hour,
and rows still to review merge to one per metric.

A service's anomalies no longer reach a host's page, as it has no one host;
get_anomaly reads what happened around one from the service's own events.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_docker_rekey.py

@@ -0,0 +1,153 @@
+"""Integration: a swarm service's metrics stay under one key across its tasks,
+and history kept under a task's own name moves onto that key in the background.
+Requires STEWARD_DATABASE_URL."""
+from __future__ import annotations
+
+import asyncio
+import importlib
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+NOW = datetime(2026, 10, 12, 10, 0, tzinfo=timezone.utc)
+TASK = "a" * 25
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def _docker(sub):
+    """The app's loaded docker plugin module (see test_docker_recent_usage)."""
+    from steward.core.plugin_manager import get_loaded_plugin_modules
+    loaded = get_loaded_plugin_modules().get("docker")
+    base = loaded.__name__ if loaded is not None else "plugins.docker"
+    return importlib.import_module(f"{base}.{sub}")
+
+
+def _hour(res, metric, bucket, *, avg, top, count=60):
+    from steward.models.metrics import PluginMetricHourly
+    return PluginMetricHourly(source_module="docker", resource_name=res, metric_name=metric, bucket=bucket,
+                              value_avg=avg, value_max=top, value_p95=top, sample_count=count)
+
+
+def _anomaly(res, metric, at, *, peak):
+    from steward.models.norms import MetricAnomaly
+    return MetricAnomaly(source_module="docker", resource_name=res, metric_name=metric, started_at=at,
+                         last_started_at=at, ended_at=at + timedelta(minutes=30), occurrences=1,
+                         peak_level=peak, peak_at=at, usual_hold=1.0, usual_peak=2.0)
+
+
+def test_a_replica_or_a_global_task_keys_by_its_service_and_a_plain_container_by_its_host(app):
+    service = _docker("service")
+    assert service.container_resource("nas", f"stack_web.1.{TASK}") == "swarm/stack_web"
+    assert service.container_resource("nas", f"stack_agent.{'b' * 25}.{TASK}") == "swarm/stack_agent"
+    assert service.container_resource("nas", "plex") == "nas/plex"
+    assert service.container_resource("nas", "my.app.1") == "nas/my.app.1"
+    assert service.traffic_resource("nas", f"stack_web.1.{TASK}") == "nas/stack_web"
+    assert service.traffic_resource("nas", "plex") == "nas/plex"
+
+
+def test_a_tasks_history_moves_onto_its_service_a_batch_at_a_time(app, monkeypatch):
+    from sqlalchemy import select
+    from steward.models.metrics import PluginMetric, PluginMetricHourly, PluginMetricLatest
+    from steward.models.norms import MetricAnomaly, MetricAnomalySpan
+    rekey = _docker("rekey")
+    monkeypatch.setattr(rekey, "BATCH", 2)
+
+    name = f"it{uuid.uuid4().hex[:8]}_web"
+    old, new, traffic = f"nas/{name}.1.{TASK}", f"swarm/{name}", "net_rx_bps"
+    shared, alone = NOW - timedelta(hours=3), NOW - timedelta(hours=2)
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add_all([PluginMetric(source_module="docker", resource_name=old, metric_name="cpu_pct",
+                                        value=float(m), recorded_at=NOW - timedelta(minutes=m)) for m in range(3)])
+                s.add(PluginMetric(source_module="docker", resource_name=old, metric_name=traffic,
+                                   value=5.0, recorded_at=NOW))
+                s.add_all([_hour(old, "cpu_pct", shared, avg=10.0, top=30.0),
+                           _hour(new, "cpu_pct", shared, avg=20.0, top=25.0),
+                           _hour(old, "cpu_pct", alone, avg=7.0, top=8.0)])
+                s.add(PluginMetricLatest(source_module="docker", resource_name=old, metric_name="cpu_pct",
+                                         value=1.0, recorded_at=NOW))
+                older = _anomaly(old, "cpu_pct", NOW - timedelta(days=2), peak=90.0)
+                newer = _anomaly(new, "cpu_pct", NOW - timedelta(days=1), peak=40.0)
+                s.add_all([older, newer])
+                await s.flush()
+                s.add(MetricAnomalySpan(anomaly_id=older.id, started_at=older.started_at,
+                                        ended_at=older.ended_at, peak_level=90.0))
+            async with s.begin():
+                found = await rekey.old_keys(s)
+            async with s.begin():
+                first = await rekey.move_key(s, old)
+            async with s.begin():
+                second = await rekey.move_key(s, old)
+
+            raw = (await s.execute(select(PluginMetric.resource_name, PluginMetric.metric_name).where(
+                PluginMetric.resource_name.in_([old, new])))).all()
+            hours = {(h.resource_name, h.bucket): h for h in (await s.execute(select(PluginMetricHourly).where(
+                PluginMetricHourly.resource_name.in_([old, new])))).scalars()}
+            latest = (await s.execute(select(PluginMetricLatest).where(
+                PluginMetricLatest.resource_name == old))).scalars().all()
+            rows = (await s.execute(select(MetricAnomaly).where(
+                MetricAnomaly.resource_name.in_([old, new])))).scalars().all()
+            spans = (await s.execute(select(MetricAnomalySpan).where(
+                MetricAnomalySpan.anomaly_id.in_([r.id for r in rows])))).scalars().all()
+            left = await rekey.old_keys(s)
+            return found, first, second, raw, hours, latest, rows, spans, left
+
+    found, first, second, raw, hours, latest, rows, spans, left = asyncio.run(_go())
+
+    assert old in found and old not in left
+    assert (first, second) == (False, True)
+    assert sorted(raw) == sorted([(new, "cpu_pct")] * 3 + [(old, traffic)])
+    assert set(hours) == {(new, shared), (new, alone)}
+    combined = hours[(new, shared)]
+    assert combined.value_avg == 15.0 and combined.value_max == 30.0 and combined.sample_count == 120
+    assert hours[(new, alone)].value_avg == 7.0
+    assert latest == []
+    (row,) = rows
+    assert row.resource_name == new and row.occurrences == 2 and row.peak_level == 90.0
+    assert row.started_at == NOW - timedelta(days=2)
+    assert [sp.anomaly_id for sp in spans] == [row.id]
+
+
+def test_an_alert_rule_on_a_task_follows_it_to_its_service(app):
+    from sqlalchemy import select
+    from steward.models.alerts import AlertOperator, AlertRule
+    from steward.models.users import User, UserRole
+    rekey = _docker("rekey")
+
+    tag = uuid.uuid4().hex[:8]
+    old, new = f"nas/it{tag}_db.1.{TASK}", f"swarm/it{tag}_db"
+    uid = str(uuid.uuid4())
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add(User(id=uid, username=f"u{tag}", email=f"{tag}@example.com",
+                           password_hash="x", role=UserRole.viewer, is_active=True))
+                await s.flush()
+                s.add(AlertRule(name=f"busy {tag}", source_module="docker", resource_name=old,
+                                metric_name="cpu_pct", operator=AlertOperator(">"), threshold=90,
+                                created_by=uid))
+            async with s.begin():
+                found = await rekey.old_keys(s)
+            async with s.begin():
+                done = await rekey.move_key(s, old)
+            names = (await s.execute(select(AlertRule.resource_name).where(
+                AlertRule.name == f"busy {tag}"))).scalars().all()
+            return found, done, names
+
+    found, done, names = asyncio.run(_go())
+    assert old in found and done
+    assert names == [new]