FabledSteward/tests/integration/test_docker_rekey.py

"""Integration: a swarm service's metrics stay under one key across its tasks,
and history kept under a task's own name moves onto that key in the background.
Requires STEWARD_DATABASE_URL."""
from __future__ import annotations

import asyncio
import importlib
import os
import uuid
from datetime import datetime, timedelta, timezone

import pytest

pytestmark = pytest.mark.integration

NOW = datetime(2026, 10, 12, 10, 0, tzinfo=timezone.utc)
TASK = "a" * 25


@pytest.fixture
def app():
    if not os.environ.get("STEWARD_DATABASE_URL"):
        pytest.skip("needs Postgres")
    from steward.app import create_app
    return create_app(testing=False)


def _docker(sub):
    """The app's loaded docker plugin module (see test_docker_recent_usage)."""
    from steward.core.plugin_manager import get_loaded_plugin_modules
    loaded = get_loaded_plugin_modules().get("docker")
    base = loaded.__name__ if loaded is not None else "plugins.docker"
    return importlib.import_module(f"{base}.{sub}")


def _hour(res, metric, bucket, *, avg, top, count=60):
    from steward.models.metrics import PluginMetricHourly
    return PluginMetricHourly(source_module="docker", resource_name=res, metric_name=metric, bucket=bucket,
                              value_avg=avg, value_max=top, value_p95=top, sample_count=count)


def _anomaly(res, metric, at, *, peak):
    from steward.models.norms import MetricAnomaly
    return MetricAnomaly(source_module="docker", resource_name=res, metric_name=metric, started_at=at,
                         last_started_at=at, ended_at=at + timedelta(minutes=30), occurrences=1,
                         peak_level=peak, peak_at=at, usual_hold=1.0, usual_peak=2.0)


def test_a_replica_or_a_global_task_keys_by_its_service_and_a_plain_container_by_its_host(app):
    service = _docker("service")
    assert service.container_resource("nas", f"stack_web.1.{TASK}") == "swarm/stack_web"
    assert service.container_resource("nas", f"stack_agent.{'b' * 25}.{TASK}") == "swarm/stack_agent"
    assert service.container_resource("nas", "plex") == "nas/plex"
    assert service.container_resource("nas", "my.app.1") == "nas/my.app.1"
    assert service.traffic_resource("nas", f"stack_web.1.{TASK}") == "nas/stack_web"
    assert service.traffic_resource("nas", "plex") == "nas/plex"


def test_a_tasks_history_moves_onto_its_service_a_batch_at_a_time(app, monkeypatch):
    from sqlalchemy import select
    from steward.models.metrics import PluginMetric, PluginMetricHourly, PluginMetricLatest
    from steward.models.norms import MetricAnomaly, MetricAnomalySpan
    rekey = _docker("rekey")
    monkeypatch.setattr(rekey, "BATCH", 2)

    name = f"it{uuid.uuid4().hex[:8]}_web"
    old, new, traffic = f"nas/{name}.1.{TASK}", f"swarm/{name}", "net_rx_bps"
    shared, alone = NOW - timedelta(hours=3), NOW - timedelta(hours=2)

    async def _go():
        async with app.db_sessionmaker() as s:
            async with s.begin():
                s.add_all([PluginMetric(source_module="docker", resource_name=old, metric_name="cpu_pct",
                                        value=float(m), recorded_at=NOW - timedelta(minutes=m)) for m in range(3)])
                s.add(PluginMetric(source_module="docker", resource_name=old, metric_name=traffic,
                                   value=5.0, recorded_at=NOW))
                s.add_all([_hour(old, "cpu_pct", shared, avg=10.0, top=30.0),
                           _hour(new, "cpu_pct", shared, avg=20.0, top=25.0),
                           _hour(old, "cpu_pct", alone, avg=7.0, top=8.0)])
                s.add(PluginMetricLatest(source_module="docker", resource_name=old, metric_name="cpu_pct",
                                         value=1.0, recorded_at=NOW))
                older = _anomaly(old, "cpu_pct", NOW - timedelta(days=2), peak=90.0)
                newer = _anomaly(new, "cpu_pct", NOW - timedelta(days=1), peak=40.0)
                s.add_all([older, newer])
                await s.flush()
                s.add(MetricAnomalySpan(anomaly_id=older.id, started_at=older.started_at,
                                        ended_at=older.ended_at, peak_level=90.0))
            async with s.begin():
                found = await rekey.old_keys(s)
            async with s.begin():
                first = await rekey.move_key(s, old)
            async with s.begin():
                second = await rekey.move_key(s, old)

            raw = (await s.execute(select(PluginMetric.resource_name, PluginMetric.metric_name).where(
                PluginMetric.resource_name.in_([old, new])))).all()
            hours = {(h.resource_name, h.bucket): h for h in (await s.execute(select(PluginMetricHourly).where(
                PluginMetricHourly.resource_name.in_([old, new])))).scalars()}
            latest = (await s.execute(select(PluginMetricLatest).where(
                PluginMetricLatest.resource_name == old))).scalars().all()
            rows = (await s.execute(select(MetricAnomaly).where(
                MetricAnomaly.resource_name.in_([old, new])))).scalars().all()
            spans = (await s.execute(select(MetricAnomalySpan).where(
                MetricAnomalySpan.anomaly_id.in_([r.id for r in rows])))).scalars().all()
            left = await rekey.old_keys(s)
            return found, first, second, raw, hours, latest, rows, spans, left

    found, first, second, raw, hours, latest, rows, spans, left = asyncio.run(_go())

    assert old in found and old not in left
    assert (first, second) == (False, True)
    assert sorted(raw) == sorted([(new, "cpu_pct")] * 3 + [(old, traffic)])
    assert set(hours) == {(new, shared), (new, alone)}
    combined = hours[(new, shared)]
    assert combined.value_avg == 15.0 and combined.value_max == 30.0 and combined.sample_count == 120
    assert hours[(new, alone)].value_avg == 7.0
    assert latest == []
    (row,) = rows
    assert row.resource_name == new and row.occurrences == 2 and row.peak_level == 90.0
    assert row.started_at == NOW - timedelta(days=2)
    assert [sp.anomaly_id for sp in spans] == [row.id]


def test_an_alert_rule_on_a_task_follows_it_to_its_service(app):
    from sqlalchemy import select
    from steward.models.alerts import AlertOperator, AlertRule
    from steward.models.users import User, UserRole
    rekey = _docker("rekey")

    tag = uuid.uuid4().hex[:8]
    old, new = f"nas/it{tag}_db.1.{TASK}", f"swarm/it{tag}_db"
    uid = str(uuid.uuid4())

    async def _go():
        async with app.db_sessionmaker() as s:
            async with s.begin():
                s.add(User(id=uid, username=f"u{tag}", email=f"{tag}@example.com",
                           password_hash="x", role=UserRole.viewer, is_active=True))
                await s.flush()
                s.add(AlertRule(name=f"busy {tag}", source_module="docker", resource_name=old,
                                metric_name="cpu_pct", operator=AlertOperator(">"), threshold=90,
                                created_by=uid))
            async with s.begin():
                found = await rekey.old_keys(s)
            async with s.begin():
                done = await rekey.move_key(s, old)
            names = (await s.execute(select(AlertRule.resource_name).where(
                AlertRule.name == f"busy {tag}"))).scalars().all()
            return found, done, names

    found, done, names = asyncio.run(_go())
    assert old in found and done
    assert names == [new]