bvandeusenandClaude Opus 5.5
4689386b69
#6718 "A swarm container's metrics, normal and anomalies are keyed by its task name, so they start over whenever the task moves". Live, the move found 391,440 keys, about 2,500 of them with raw samples, and took one key per 15 s pass: about 68 days, in sorted order, so recent history could wait months while charts and normals read thin. A pass now runs for up to 2 s, and keys with raw samples go first. Old hourly keys take a few statements each. A key with no anomalies no longer rewrites the service's timeline entries. The start logs how many keys each table holds, as that count is far higher than expected and not explained yet. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
188 lines
8.9 KiB
Python
188 lines
8.9 KiB
Python
"""Integration: a swarm service's metrics stay under one key across its tasks,
|
|
and history kept under a task's own name moves onto that key in the background.
|
|
Requires STEWARD_DATABASE_URL."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import importlib
|
|
import os
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
NOW = datetime(2026, 10, 12, 10, 0, tzinfo=timezone.utc)
|
|
TASK = "a" * 25
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
def _docker(sub):
|
|
"""The app's loaded docker plugin module (see test_docker_recent_usage)."""
|
|
from steward.core.plugin_manager import get_loaded_plugin_modules
|
|
loaded = get_loaded_plugin_modules().get("docker")
|
|
base = loaded.__name__ if loaded is not None else "plugins.docker"
|
|
return importlib.import_module(f"{base}.{sub}")
|
|
|
|
|
|
def _hour(res, metric, bucket, *, avg, top, count=60):
|
|
from steward.models.metrics import PluginMetricHourly
|
|
return PluginMetricHourly(source_module="docker", resource_name=res, metric_name=metric, bucket=bucket,
|
|
value_avg=avg, value_max=top, value_p95=top, sample_count=count)
|
|
|
|
|
|
def _anomaly(res, metric, at, *, peak):
|
|
from steward.models.norms import MetricAnomaly
|
|
return MetricAnomaly(source_module="docker", resource_name=res, metric_name=metric, started_at=at,
|
|
last_started_at=at, ended_at=at + timedelta(minutes=30), occurrences=1,
|
|
peak_level=peak, peak_at=at, usual_hold=1.0, usual_peak=2.0)
|
|
|
|
|
|
def test_a_replica_or_a_global_task_keys_by_its_service_and_a_plain_container_by_its_host(app):
|
|
service = _docker("service")
|
|
assert service.container_resource("nas", f"stack_web.1.{TASK}") == "swarm/stack_web"
|
|
assert service.container_resource("nas", f"stack_agent.{'b' * 25}.{TASK}") == "swarm/stack_agent"
|
|
assert service.container_resource("nas", "plex") == "nas/plex"
|
|
assert service.container_resource("nas", "my.app.1") == "nas/my.app.1"
|
|
assert service.traffic_resource("nas", f"stack_web.1.{TASK}") == "nas/stack_web"
|
|
assert service.traffic_resource("nas", "plex") == "nas/plex"
|
|
|
|
|
|
def test_a_tasks_history_moves_onto_its_service_a_batch_at_a_time(app, monkeypatch):
|
|
from sqlalchemy import select
|
|
from steward.models.metrics import PluginMetric, PluginMetricHourly, PluginMetricLatest
|
|
from steward.models.norms import MetricAnomaly, MetricAnomalySpan
|
|
rekey = _docker("rekey")
|
|
monkeypatch.setattr(rekey, "BATCH", 2)
|
|
|
|
name = f"it{uuid.uuid4().hex[:8]}_web"
|
|
old, new, traffic = f"nas/{name}.1.{TASK}", f"swarm/{name}", "net_rx_bps"
|
|
shared, alone = NOW - timedelta(hours=3), NOW - timedelta(hours=2)
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
s.add_all([PluginMetric(source_module="docker", resource_name=old, metric_name="cpu_pct",
|
|
value=float(m), recorded_at=NOW - timedelta(minutes=m)) for m in range(3)])
|
|
s.add(PluginMetric(source_module="docker", resource_name=old, metric_name=traffic,
|
|
value=5.0, recorded_at=NOW))
|
|
s.add_all([_hour(old, "cpu_pct", shared, avg=10.0, top=30.0),
|
|
_hour(new, "cpu_pct", shared, avg=20.0, top=25.0),
|
|
_hour(old, "cpu_pct", alone, avg=7.0, top=8.0)])
|
|
s.add(PluginMetricLatest(source_module="docker", resource_name=old, metric_name="cpu_pct",
|
|
value=1.0, recorded_at=NOW))
|
|
older = _anomaly(old, "cpu_pct", NOW - timedelta(days=2), peak=90.0)
|
|
newer = _anomaly(new, "cpu_pct", NOW - timedelta(days=1), peak=40.0)
|
|
s.add_all([older, newer])
|
|
await s.flush()
|
|
s.add(MetricAnomalySpan(anomaly_id=older.id, started_at=older.started_at,
|
|
ended_at=older.ended_at, peak_level=90.0))
|
|
async with s.begin():
|
|
found = await rekey.old_keys(s)
|
|
async with s.begin():
|
|
first = await rekey.move_key(s, old)
|
|
async with s.begin():
|
|
second = await rekey.move_key(s, old)
|
|
|
|
raw = (await s.execute(select(PluginMetric.resource_name, PluginMetric.metric_name).where(
|
|
PluginMetric.resource_name.in_([old, new])))).all()
|
|
hours = {(h.resource_name, h.bucket): h for h in (await s.execute(select(PluginMetricHourly).where(
|
|
PluginMetricHourly.resource_name.in_([old, new])))).scalars()}
|
|
latest = (await s.execute(select(PluginMetricLatest).where(
|
|
PluginMetricLatest.resource_name == old))).scalars().all()
|
|
rows = (await s.execute(select(MetricAnomaly).where(
|
|
MetricAnomaly.resource_name.in_([old, new])))).scalars().all()
|
|
spans = (await s.execute(select(MetricAnomalySpan).where(
|
|
MetricAnomalySpan.anomaly_id.in_([r.id for r in rows])))).scalars().all()
|
|
left = await rekey.old_keys(s)
|
|
return found, first, second, raw, hours, latest, rows, spans, left
|
|
|
|
found, first, second, raw, hours, latest, rows, spans, left = asyncio.run(_go())
|
|
|
|
assert old in found and old not in left
|
|
assert (first, second) == (False, True)
|
|
assert sorted(raw) == sorted([(new, "cpu_pct")] * 3 + [(old, traffic)])
|
|
assert set(hours) == {(new, shared), (new, alone)}
|
|
combined = hours[(new, shared)]
|
|
assert combined.value_avg == 15.0 and combined.value_max == 30.0 and combined.sample_count == 120
|
|
assert hours[(new, alone)].value_avg == 7.0
|
|
assert latest == []
|
|
(row,) = rows
|
|
assert row.resource_name == new and row.occurrences == 2 and row.peak_level == 90.0
|
|
assert row.started_at == NOW - timedelta(days=2)
|
|
assert [sp.anomaly_id for sp in spans] == [row.id]
|
|
|
|
|
|
def test_keys_with_raw_samples_are_taken_first_and_a_pass_finishes_many_small_keys(app):
|
|
"""Taken from the end of old_keys: a chart and a normal read recent history,
|
|
so a key with raw samples doesn't wait behind thousands that only have old
|
|
hours. One pass moves every small key it has time for."""
|
|
from sqlalchemy import select
|
|
from steward.models.metrics import PluginMetricHourly, PluginMetricLatest
|
|
rekey = _docker("rekey")
|
|
|
|
tag = uuid.uuid4().hex[:8]
|
|
aged = [f"nas/it{tag}_old.1.{c * 25}" for c in "bcd"]
|
|
recent = f"nas/it{tag}_new.1.{TASK}"
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
s.add_all([_hour(k, "cpu_pct", NOW, avg=1.0, top=2.0) for k in aged])
|
|
s.add(PluginMetricLatest(source_module="docker", resource_name=recent, metric_name="cpu_pct",
|
|
value=1.0, recorded_at=NOW))
|
|
async with s.begin():
|
|
found = await rekey.old_keys(s)
|
|
await rekey.make_rekey_task(app).coro_factory()
|
|
async with app.db_sessionmaker() as s:
|
|
left = (await s.execute(select(PluginMetricHourly.resource_name).where(
|
|
PluginMetricHourly.resource_name.in_(aged)))).scalars().all()
|
|
moved = (await s.execute(select(PluginMetricHourly.value_avg).where(
|
|
PluginMetricHourly.resource_name == f"swarm/it{tag}_old"))).scalars().all()
|
|
return found, left, moved
|
|
|
|
found, left, moved = asyncio.run(_go())
|
|
assert all(found.index(recent) > found.index(k) for k in aged)
|
|
assert left == []
|
|
assert moved == [1.0]
|
|
|
|
|
|
def test_an_alert_rule_on_a_task_follows_it_to_its_service(app):
|
|
from sqlalchemy import select
|
|
from steward.models.alerts import AlertOperator, AlertRule
|
|
from steward.models.users import User, UserRole
|
|
rekey = _docker("rekey")
|
|
|
|
tag = uuid.uuid4().hex[:8]
|
|
old, new = f"nas/it{tag}_db.1.{TASK}", f"swarm/it{tag}_db"
|
|
uid = str(uuid.uuid4())
|
|
|
|
async def _go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
s.add(User(id=uid, username=f"u{tag}", email=f"{tag}@example.com",
|
|
password_hash="x", role=UserRole.viewer, is_active=True))
|
|
await s.flush()
|
|
s.add(AlertRule(name=f"busy {tag}", source_module="docker", resource_name=old,
|
|
metric_name="cpu_pct", operator=AlertOperator(">"), threshold=90,
|
|
created_by=uid))
|
|
async with s.begin():
|
|
found = await rekey.old_keys(s)
|
|
async with s.begin():
|
|
done = await rekey.move_key(s, old)
|
|
names = (await s.execute(select(AlertRule.resource_name).where(
|
|
AlertRule.name == f"busy {tag}"))).scalars().all()
|
|
return found, done, names
|
|
|
|
found, done, names = asyncio.run(_go())
|
|
assert old in found and done
|
|
assert names == [new]
|