"""Integration: the plugin sources and anomalies write the timeline as things happen. A host away is one entry kept up to date until it is back; a container on a new image is a deploy; deaths past the crash-loop count are one crash-loop entry; an anomaly's entry follows its review. The database is shared with other tests, so each test looks for its own uniquely named host. Requires STEWARD_DATABASE_URL. """ from __future__ import annotations import asyncio import os import uuid from datetime import datetime, timedelta, timezone import pytest pytestmark = pytest.mark.integration @pytest.fixture def app(): if not os.environ.get("STEWARD_DATABASE_URL"): pytest.skip("needs Postgres") from steward.app import create_app return create_app(testing=False) async def _entries(app, host: str): from sqlalchemy import select from steward.models.timeline import TimelineEntry async with app.db_sessionmaker() as s: return (await s.execute(select(TimelineEntry).where(TimelineEntry.hosts.contains([host])) .order_by(TimelineEntry.at, TimelineEntry.id))).scalars().all() async def _host(app, name: str): from steward.models.hosts import Host async with app.db_sessionmaker() as s: async with s.begin(): host = Host(name=name, address="10.7.7.7") s.add(host) return host def test_a_reboot_is_one_entry_that_gets_its_length_when_the_host_is_back(app): from steward.core.plugin_manager import loaded_plugin_submodule downtime = loaded_plugin_submodule("host_agent", "downtime") async def _go(): name = f"tl-dt-{uuid.uuid4().hex[:8]}" host = await _host(app, name) down = datetime.now(timezone.utc).replace(microsecond=0) - timedelta(minutes=10) async def post(events): async with app.db_sessionmaker() as s: async with s.begin(): await downtime.record_events(s, host.id, events, down - timedelta(minutes=1)) await post([{"kind": "stopping", "at": down.isoformat(), "host_shutdown": True}]) [entry] = await _entries(app, name) assert entry.type == "host.rebooted" and entry.until is None assert entry.summary == f"{name}: rebooted, not back yet" back = down + timedelta(minutes=3) await post([{"kind": "started", "at": back.isoformat(), "boot_at": (down + timedelta(minutes=2)).isoformat()}]) [entry] = await _entries(app, name) # the same entry, now with its end assert entry.until == back and entry.summary.startswith(f"{name}: rebooted for ") assert entry.ref_type == "host_downtime" and entry.area == "host_os" # Lost contact sorts under Network. await post([{"kind": "lost_contact", "from": (back + timedelta(minutes=1)).isoformat(), "to": (back + timedelta(minutes=4)).isoformat()}]) lost = [e for e in await _entries(app, name) if e.type == "host.lost_contact"] assert len(lost) == 1 and lost[0].area == "network" asyncio.run(_go()) def test_a_finished_maintenance_job_goes_on_the_timeline(app): from sqlalchemy import text from steward.core.plugin_manager import loaded_plugin_submodule maintenance = loaded_plugin_submodule("host_agent", "maintenance") async def _go(): name = f"tl-mj-{uuid.uuid4().hex[:8]}" host = await _host(app, name) job_id = str(uuid.uuid4()) now = datetime.now(timezone.utc) async with app.db_sessionmaker() as s: async with s.begin(): await s.execute(text( "INSERT INTO host_maintenance_jobs (id, host_id, state, state_at, reboot_policy, " "step_seen, step_id, packages, reboot_needed, created_at) " "VALUES (:id, :h, 'updating', :at, 'if_required', true, :step, NULL, NULL, :at)"), {"id": job_id, "h": host.id, "at": now - timedelta(minutes=5), "step": f"{job_id}-update"}) async with app.db_sessionmaker() as s: async with s.begin(): assert await maintenance.apply_events(s, host.id, [ {"kind": "maintenance", "id": f"{job_id}-update", "action": "update", "state": "done", "ok": True, "packages": 4, "reboot_required": False}], now) == 1 [entry] = await _entries(app, name) assert entry.type == "maintenance.done" assert (entry.ref_type, entry.ref_id) == ("maintenance_job", job_id) assert entry.summary == f"Maintenance on {name}: Done: 4 packages updated, no reboot needed" assert entry.until == now and entry.area == "host_os" asyncio.run(_go()) def test_docker_deploys_and_a_crash_loop_as_one_episode(app): from steward.core.capabilities import get_capability persist = get_capability("docker.persist_host_samples").fn def c(name, image, cid, restarts=0, **kw): return {"name": name, "container_id": cid, "image": image, "status": "running", "cpu_pct": 1.0, "mem_used_pct": 1.0, "mem_used_bytes": 1, "mem_limit_bytes": 2, "ports": [], "started_at": None, "health": None, "restart_count": restarts, "exit_code": 0, "oom_killed": False, **kw} async def _go(): name = f"tl-dk-{uuid.uuid4().hex[:8]}" host = await _host(app, name) t = datetime.now(timezone.utc) - timedelta(minutes=30) async def snap(minutes, containers): async with app.db_sessionmaker() as s: async with s.begin(): await persist(s, host, [(t + timedelta(minutes=minutes), containers)]) await snap(0, [c("web", "nginx:1.27", "a1"), c("job", "worker:3", "j1")]) # the baseline: nothing assert await _entries(app, name) == [] await snap(1, [c("web", "nginx:1.28", "a2"), c("job", "worker:3", "j1")]) [deploy] = await _entries(app, name) assert deploy.type == "docker.deployed" and deploy.subject == f"{name}/web" assert deploy.summary == f"web on {name}: nginx:1.27 → nginx:1.28" for i in (1, 2): await snap(1 + i, [c("web", "nginx:1.28", "a2"), c("job", "worker:3", "j1", restarts=i)]) singles = [e for e in await _entries(app, name) if e.type == "docker.restarted"] assert len(singles) == 2 # recorded, hidden for i in (3, 4): await snap(1 + i, [c("web", "nginx:1.28", "a2"), c("job", "worker:3", "j1", restarts=i)]) [loop] = [e for e in await _entries(app, name) if e.type == "docker.crash_loop"] assert loop.detail["count"] == 4 and loop.until == t + timedelta(minutes=5) assert loop.at == singles[0].at assert loop.summary == f"job on {name} crash-looping: 4 restarts or deaths" assert len([e for e in await _entries(app, name) if e.type == "docker.restarted"]) == 2 asyncio.run(_go()) def test_an_anomaly_s_entry_follows_its_review(app): from steward.core.anomalies import REAL, lapse, mark from steward.models.norms import MetricAnomaly from steward.timeline.record import anomaly_changed async def _go(): name = f"tl-an-{uuid.uuid4().hex[:8]}" await _host(app, name) start = datetime.now(timezone.utc) - timedelta(days=20) def anomaly(metric): return MetricAnomaly(source_module="host_agent", resource_name=name, metric_name=metric, started_at=start, last_started_at=start, ended_at=start + timedelta(hours=1), occurrences=1, peak_level=93.0, peak_at=start, usual_hold=20.0, usual_peak=35.0, review="to_review") async with app.db_sessionmaker() as s: async with s.begin(): cpu, mem = anomaly("cpu_pct"), anomaly("mem_used_pct") s.add_all([cpu, mem]) await anomaly_changed(s, cpu) await anomaly_changed(s, mem) assert {e.type for e in await _entries(app, name)} == {"anomaly.to_review"} async with app.db_sessionmaker() as s: async with s.begin(): await mark(s, cpu.id, REAL, "tester", "the backup hung") assert await lapse(s, datetime.now(timezone.utc), 14) >= 1 got = {e.ref_id: e for e in await _entries(app, name)} assert len(got) == 2 assert got[cpu.id].type == "anomaly.real" and got[cpu.id].actor == "tester" assert got[cpu.id].summary.endswith(": a real problem") assert got[cpu.id].detail["note"] == "the backup hung" assert got[mem.id].type == "anomaly.lapsed" asyncio.run(_go())