"""Integration: a swarm service's metrics stay under one key across its tasks, and history kept under a task's own name moves onto that key in the background. Requires STEWARD_DATABASE_URL.""" from __future__ import annotations import asyncio import importlib import os import uuid from datetime import datetime, timedelta, timezone import pytest pytestmark = pytest.mark.integration NOW = datetime(2026, 10, 12, 10, 0, tzinfo=timezone.utc) TASK = "a" * 25 @pytest.fixture def app(): if not os.environ.get("STEWARD_DATABASE_URL"): pytest.skip("needs Postgres") from steward.app import create_app return create_app(testing=False) def _docker(sub): """The app's loaded docker plugin module (see test_docker_recent_usage).""" from steward.core.plugin_manager import get_loaded_plugin_modules loaded = get_loaded_plugin_modules().get("docker") base = loaded.__name__ if loaded is not None else "plugins.docker" return importlib.import_module(f"{base}.{sub}") def _hour(res, metric, bucket, *, avg, top, count=60): from steward.models.metrics import PluginMetricHourly return PluginMetricHourly(source_module="docker", resource_name=res, metric_name=metric, bucket=bucket, value_avg=avg, value_max=top, value_p95=top, sample_count=count) def _anomaly(res, metric, at, *, peak): from steward.models.norms import MetricAnomaly return MetricAnomaly(source_module="docker", resource_name=res, metric_name=metric, started_at=at, last_started_at=at, ended_at=at + timedelta(minutes=30), occurrences=1, peak_level=peak, peak_at=at, usual_hold=1.0, usual_peak=2.0) def test_a_replica_or_a_global_task_keys_by_its_service_and_a_plain_container_by_its_host(app): service = _docker("service") assert service.container_resource("nas", f"stack_web.1.{TASK}") == "swarm/stack_web" assert service.container_resource("nas", f"stack_agent.{'b' * 25}.{TASK}") == "swarm/stack_agent" assert service.container_resource("nas", "plex") == "nas/plex" assert service.container_resource("nas", "my.app.1") == "nas/my.app.1" assert service.traffic_resource("nas", f"stack_web.1.{TASK}") == "nas/stack_web" assert service.traffic_resource("nas", "plex") == "nas/plex" def test_a_tasks_history_moves_onto_its_service_a_batch_at_a_time(app, monkeypatch): from sqlalchemy import select from steward.models.metrics import PluginMetric, PluginMetricHourly, PluginMetricLatest from steward.models.norms import MetricAnomaly, MetricAnomalySpan rekey = _docker("rekey") monkeypatch.setattr(rekey, "BATCH", 2) name = f"it{uuid.uuid4().hex[:8]}_web" old, new, traffic = f"nas/{name}.1.{TASK}", f"swarm/{name}", "net_rx_bps" shared, alone = NOW - timedelta(hours=3), NOW - timedelta(hours=2) async def _go(): async with app.db_sessionmaker() as s: async with s.begin(): s.add_all([PluginMetric(source_module="docker", resource_name=old, metric_name="cpu_pct", value=float(m), recorded_at=NOW - timedelta(minutes=m)) for m in range(3)]) s.add(PluginMetric(source_module="docker", resource_name=old, metric_name=traffic, value=5.0, recorded_at=NOW)) s.add_all([_hour(old, "cpu_pct", shared, avg=10.0, top=30.0), _hour(new, "cpu_pct", shared, avg=20.0, top=25.0), _hour(old, "cpu_pct", alone, avg=7.0, top=8.0)]) s.add(PluginMetricLatest(source_module="docker", resource_name=old, metric_name="cpu_pct", value=1.0, recorded_at=NOW)) older = _anomaly(old, "cpu_pct", NOW - timedelta(days=2), peak=90.0) newer = _anomaly(new, "cpu_pct", NOW - timedelta(days=1), peak=40.0) s.add_all([older, newer]) await s.flush() s.add(MetricAnomalySpan(anomaly_id=older.id, started_at=older.started_at, ended_at=older.ended_at, peak_level=90.0)) async with s.begin(): found = await rekey.old_keys(s) async with s.begin(): first = await rekey.move_key(s, old) async with s.begin(): second = await rekey.move_key(s, old) raw = (await s.execute(select(PluginMetric.resource_name, PluginMetric.metric_name).where( PluginMetric.resource_name.in_([old, new])))).all() hours = {(h.resource_name, h.bucket): h for h in (await s.execute(select(PluginMetricHourly).where( PluginMetricHourly.resource_name.in_([old, new])))).scalars()} latest = (await s.execute(select(PluginMetricLatest).where( PluginMetricLatest.resource_name == old))).scalars().all() rows = (await s.execute(select(MetricAnomaly).where( MetricAnomaly.resource_name.in_([old, new])))).scalars().all() spans = (await s.execute(select(MetricAnomalySpan).where( MetricAnomalySpan.anomaly_id.in_([r.id for r in rows])))).scalars().all() left = await rekey.old_keys(s) return found, first, second, raw, hours, latest, rows, spans, left found, first, second, raw, hours, latest, rows, spans, left = asyncio.run(_go()) assert old in found and old not in left assert (first, second) == (False, True) assert sorted(raw) == sorted([(new, "cpu_pct")] * 3 + [(old, traffic)]) assert set(hours) == {(new, shared), (new, alone)} combined = hours[(new, shared)] assert combined.value_avg == 15.0 and combined.value_max == 30.0 and combined.sample_count == 120 assert hours[(new, alone)].value_avg == 7.0 assert latest == [] (row,) = rows assert row.resource_name == new and row.occurrences == 2 and row.peak_level == 90.0 assert row.started_at == NOW - timedelta(days=2) assert [sp.anomaly_id for sp in spans] == [row.id] def test_an_alert_rule_on_a_task_follows_it_to_its_service(app): from sqlalchemy import select from steward.models.alerts import AlertOperator, AlertRule from steward.models.users import User, UserRole rekey = _docker("rekey") tag = uuid.uuid4().hex[:8] old, new = f"nas/it{tag}_db.1.{TASK}", f"swarm/it{tag}_db" uid = str(uuid.uuid4()) async def _go(): async with app.db_sessionmaker() as s: async with s.begin(): s.add(User(id=uid, username=f"u{tag}", email=f"{tag}@example.com", password_hash="x", role=UserRole.viewer, is_active=True)) await s.flush() s.add(AlertRule(name=f"busy {tag}", source_module="docker", resource_name=old, metric_name="cpu_pct", operator=AlertOperator(">"), threshold=90, created_by=uid)) async with s.begin(): found = await rekey.old_keys(s) async with s.begin(): done = await rekey.move_key(s, old) names = (await s.execute(select(AlertRule.resource_name).where( AlertRule.name == f"busy {tag}"))).scalars().all() return found, done, names found, done, names = asyncio.run(_go()) assert old in found and done assert names == [new]