"""Integration: a swarm service's metrics stay under one key across its tasks, and history kept under a task's own name moves onto that key in the background. Requires STEWARD_DATABASE_URL.""" from __future__ import annotations import asyncio import importlib import os import uuid from datetime import datetime, timedelta, timezone import pytest pytestmark = pytest.mark.integration NOW = datetime(2026, 10, 12, 10, 0, tzinfo=timezone.utc) TASK = "a" * 25 @pytest.fixture def app(): if not os.environ.get("STEWARD_DATABASE_URL"): pytest.skip("needs Postgres") from steward.app import create_app return create_app(testing=False) def _docker(sub): """The app's loaded docker plugin module (see test_docker_recent_usage).""" from steward.core.plugin_manager import get_loaded_plugin_modules loaded = get_loaded_plugin_modules().get("docker") base = loaded.__name__ if loaded is not None else "plugins.docker" return importlib.import_module(f"{base}.{sub}") def _hour(res, metric, bucket, *, avg, top, count=60): from steward.models.metrics import PluginMetricHourly return PluginMetricHourly(source_module="docker", resource_name=res, metric_name=metric, bucket=bucket, value_avg=avg, value_max=top, value_p95=top, sample_count=count) def _anomaly(res, metric, at, *, peak): from steward.models.norms import MetricAnomaly return MetricAnomaly(source_module="docker", resource_name=res, metric_name=metric, started_at=at, last_started_at=at, ended_at=at + timedelta(minutes=30), occurrences=1, peak_level=peak, peak_at=at, usual_hold=1.0, usual_peak=2.0) def test_a_replica_or_a_global_task_keys_by_its_service_and_a_plain_container_by_its_host(app): service = _docker("service") assert service.container_resource("nas", f"stack_web.1.{TASK}") == "swarm/stack_web" assert service.container_resource("nas", f"stack_agent.{'b' * 25}.{TASK}") == "swarm/stack_agent" assert service.container_resource("nas", "plex") == "nas/plex" assert service.container_resource("nas", "my.app.1") == "nas/my.app.1" assert service.traffic_resource("nas", f"stack_web.1.{TASK}") == "nas/stack_web" assert service.traffic_resource("nas", "plex") == "nas/plex" def test_a_tasks_history_moves_onto_its_service_a_batch_at_a_time(app, monkeypatch): from sqlalchemy import select from steward.models.metrics import PluginMetric, PluginMetricHourly, PluginMetricLatest from steward.models.norms import MetricAnomaly, MetricAnomalySpan rekey = _docker("rekey") monkeypatch.setattr(rekey, "BATCH", 2) name = f"it{uuid.uuid4().hex[:8]}_web" old, new, traffic = f"nas/{name}.1.{TASK}", f"swarm/{name}", "net_rx_bps" shared, alone = NOW - timedelta(hours=3), NOW - timedelta(hours=2) async def _go(): async with app.db_sessionmaker() as s: async with s.begin(): s.add_all([PluginMetric(source_module="docker", resource_name=old, metric_name="cpu_pct", value=float(m), recorded_at=NOW - timedelta(minutes=m)) for m in range(3)]) s.add(PluginMetric(source_module="docker", resource_name=old, metric_name=traffic, value=5.0, recorded_at=NOW)) s.add_all([_hour(old, "cpu_pct", shared, avg=10.0, top=30.0), _hour(new, "cpu_pct", shared, avg=20.0, top=25.0), _hour(old, "cpu_pct", alone, avg=7.0, top=8.0)]) s.add(PluginMetricLatest(source_module="docker", resource_name=old, metric_name="cpu_pct", value=1.0, recorded_at=NOW)) older = _anomaly(old, "cpu_pct", NOW - timedelta(days=2), peak=90.0) newer = _anomaly(new, "cpu_pct", NOW - timedelta(days=1), peak=40.0) s.add_all([older, newer]) await s.flush() s.add(MetricAnomalySpan(anomaly_id=older.id, started_at=older.started_at, ended_at=older.ended_at, peak_level=90.0)) async with s.begin(): found = await rekey.old_keys(s) async with s.begin(): first = await rekey.move_key(s, old) async with s.begin(): second = await rekey.move_key(s, old) raw = (await s.execute(select(PluginMetric.resource_name, PluginMetric.metric_name).where( PluginMetric.resource_name.in_([old, new])))).all() hours = {(h.resource_name, h.bucket): h for h in (await s.execute(select(PluginMetricHourly).where( PluginMetricHourly.resource_name.in_([old, new])))).scalars()} latest = (await s.execute(select(PluginMetricLatest).where( PluginMetricLatest.resource_name == old))).scalars().all() rows = (await s.execute(select(MetricAnomaly).where( MetricAnomaly.resource_name.in_([old, new])))).scalars().all() spans = (await s.execute(select(MetricAnomalySpan).where( MetricAnomalySpan.anomaly_id.in_([r.id for r in rows])))).scalars().all() left = await rekey.old_keys(s) return found, first, second, raw, hours, latest, rows, spans, left found, first, second, raw, hours, latest, rows, spans, left = asyncio.run(_go()) assert old in found and old not in left assert (first, second) == (False, True) assert sorted(raw) == sorted([(new, "cpu_pct")] * 3 + [(old, traffic)]) assert set(hours) == {(new, shared), (new, alone)} combined = hours[(new, shared)] assert combined.value_avg == 15.0 and combined.value_max == 30.0 and combined.sample_count == 120 assert hours[(new, alone)].value_avg == 7.0 assert latest == [] (row,) = rows assert row.resource_name == new and row.occurrences == 2 and row.peak_level == 90.0 assert row.started_at == NOW - timedelta(days=2) assert [sp.anomaly_id for sp in spans] == [row.id] def test_keys_with_raw_samples_are_taken_first_and_a_pass_finishes_many_small_keys(app): """Taken from the end of old_keys: a chart and a normal read recent history, so a key with raw samples doesn't wait behind thousands that only have old hours. One pass moves every small key it has time for.""" from sqlalchemy import select from steward.models.metrics import PluginMetricHourly, PluginMetricLatest rekey = _docker("rekey") tag = uuid.uuid4().hex[:8] aged = [f"nas/it{tag}_old.1.{c * 25}" for c in "bcd"] recent = f"nas/it{tag}_new.1.{TASK}" async def _go(): async with app.db_sessionmaker() as s: async with s.begin(): s.add_all([_hour(k, "cpu_pct", NOW, avg=1.0, top=2.0) for k in aged]) s.add(PluginMetricLatest(source_module="docker", resource_name=recent, metric_name="cpu_pct", value=1.0, recorded_at=NOW)) async with s.begin(): found = await rekey.old_keys(s) await rekey.make_rekey_task(app).coro_factory() async with app.db_sessionmaker() as s: left = (await s.execute(select(PluginMetricHourly.resource_name).where( PluginMetricHourly.resource_name.in_(aged)))).scalars().all() moved = (await s.execute(select(PluginMetricHourly.value_avg).where( PluginMetricHourly.resource_name == f"swarm/it{tag}_old"))).scalars().all() return found, left, moved found, left, moved = asyncio.run(_go()) assert all(found.index(recent) > found.index(k) for k in aged) assert left == [] assert moved == [1.0] def test_an_alert_rule_on_a_task_follows_it_to_its_service(app): from sqlalchemy import select from steward.models.alerts import AlertOperator, AlertRule from steward.models.users import User, UserRole rekey = _docker("rekey") tag = uuid.uuid4().hex[:8] old, new = f"nas/it{tag}_db.1.{TASK}", f"swarm/it{tag}_db" uid = str(uuid.uuid4()) async def _go(): async with app.db_sessionmaker() as s: async with s.begin(): s.add(User(id=uid, username=f"u{tag}", email=f"{tag}@example.com", password_hash="x", role=UserRole.viewer, is_active=True)) await s.flush() s.add(AlertRule(name=f"busy {tag}", source_module="docker", resource_name=old, metric_name="cpu_pct", operator=AlertOperator(">"), threshold=90, created_by=uid)) async with s.begin(): found = await rekey.old_keys(s) async with s.begin(): done = await rekey.move_key(s, old) names = (await s.execute(select(AlertRule.resource_name).where( AlertRule.name == f"busy {tag}"))).scalars().all() return found, done, names found, done, names = asyncio.run(_go()) assert old in found and done assert names == [new]