bvandeusenandClaude Opus 5.5
4ea735d87c
A stored log, timeline_entries, with one row per thing that happened. Each source writes its row in the same transaction as its own record, so the history outlives the 30/90/365-day retention of the tables it points to: - alert events (core/alerts._add_event): fired, resolved, acknowledged, went silent, readings back; - finished Ansible runs (executor, and the startup sweep of interrupted runs): changed something, changed nothing, failed, dry run; - audit rows (core/audit.log_audit): Steward's settings, users and access, sign-ins, backups. Actions another writer records are left to it. Entries carry a kind (fixed) and an area (Service, Host OS, Hardware, Storage, Network, Security, Monitoring). Settings → Timeline edits the areas and each type's area and visibility; both are applied when the timeline is read, so a change re-sorts past entries and a hidden type comes back when shown. Kept forever unless a retention is set. The first start fills the timeline from the last 90 days of records once, skipping any already covered. A host rename carries its entries. UI: Timeline in the sidebar (filter by host, area, kind; scrolls on), a History card on each host page. MCP: get_timeline. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
266 lines
13 KiB
Python
266 lines
13 KiB
Python
"""Integration: the timeline against real Postgres. Each source writes its
|
|
entry as the thing happens, the entry outlives its record, Settings re-sorts
|
|
and hides entries when they are read, the first fill skips what is covered,
|
|
and get_timeline reads it back. The database is shared with other tests, so
|
|
each test looks for its own uniquely named host.
|
|
Requires STEWARD_DATABASE_URL.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
async def _entries(app, host: str):
|
|
from sqlalchemy import select
|
|
from steward.models.timeline import TimelineEntry
|
|
async with app.db_sessionmaker() as s:
|
|
return (await s.execute(select(TimelineEntry).where(TimelineEntry.hosts.contains([host]))
|
|
.order_by(TimelineEntry.at))).scalars().all()
|
|
|
|
|
|
def test_each_source_writes_an_entry_that_outlives_its_record(app):
|
|
from sqlalchemy import delete
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.alerts import record_metric, record_metrics
|
|
from steward.core.audit import log_audit
|
|
from steward.models.alerts import AlertEvent
|
|
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
|
from steward.models.hosts import Host
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.users import User, UserRole
|
|
from steward.timeline import config as tl_config, service as tl
|
|
from steward.timeline.record import run_finished
|
|
|
|
async def _go():
|
|
tag = uuid.uuid4().hex[:8]
|
|
name, renamed, run_id = f"tl-{tag}", f"tl-{tag}-new", str(uuid.uuid4())
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
user = User(username=f"tl-{tag}", email=f"tl-{tag}@test", password_hash="x",
|
|
role=UserRole.admin)
|
|
host = Host(name=name, address="10.1.2.3")
|
|
s.add_all([user, host])
|
|
await record_metrics(s, [PluginMetric(source_module="host_agent", resource_name=name,
|
|
metric_name="cpu_pct", value=1.0)])
|
|
await s.flush()
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=f"hot {tag}", source_module="host_agent", resource_name=name,
|
|
metric_name="cpu_pct", operator=">", threshold=90,
|
|
consecutive_failures_required=1, consecutive_ok_required=1)
|
|
rule = await alert_svc.add_rule(s, fields, created_by=user.id)
|
|
user_id, host_id, rule_id = user.id, host.id, rule.id
|
|
|
|
# An alert firing.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await record_metric(s, "host_agent", name, "cpu_pct", 97.0)
|
|
# A change to Steward's settings about the host, and one the timeline leaves to the run.
|
|
await log_audit(app, user_id, f"tl-{tag}", "host.created", "host", host_id, {"name": name})
|
|
await log_audit(app, user_id, f"tl-{tag}", "ansible.run.started", "ansible_run", run_id, {})
|
|
# A playbook run that changed the host.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
s.add(AnsibleRun(id=run_id, playbook_path="update.yml", source_name="local",
|
|
inventory_scope=f"steward:target:{uuid.uuid4()}",
|
|
status=AnsibleRunStatus.success, triggered_by=user_id,
|
|
finished_at=datetime.now(timezone.utc),
|
|
results={"hosts": {name: {"ok": 3, "changed": 1}}, "failures": []}))
|
|
async with s.begin():
|
|
await run_finished(s, run_id)
|
|
|
|
got = {e.type: e for e in await _entries(app, name)}
|
|
assert set(got) == {"alert.fired", "steward.config", "ansible.changed"}
|
|
fired = got["alert.fired"]
|
|
assert fired.summary == f"hot {tag} fired: CPU on {name} is 97% (> 90%)"
|
|
assert fired.area == "host_os" and fired.ref_type == "alert_event"
|
|
assert got["steward.config"].summary == f"Added host {name}"
|
|
assert got["steward.config"].actor == f"tl-{tag}"
|
|
assert got["ansible.changed"].summary.endswith(": changed 1 host")
|
|
assert got["ansible.changed"].actor == f"tl-{tag}"
|
|
|
|
# The records age out; the entries stay, saying their details are gone.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await s.execute(delete(AlertEvent).where(AlertEvent.rule_id == rule_id))
|
|
await s.execute(delete(AnsibleRun).where(AnsibleRun.id == run_id))
|
|
async with app.db_sessionmaker() as s:
|
|
cfg = await tl_config.load(s)
|
|
card = {v["entry"].type: v for v in await tl.recent_for_host(s, cfg, name)}
|
|
assert set(card) == {"alert.fired", "steward.config", "ansible.changed"}
|
|
assert card["alert.fired"]["gone"] and card["alert.fired"]["url"] is None
|
|
assert card["ansible.changed"]["gone"]
|
|
assert not card["steward.config"]["gone"]
|
|
|
|
# Renaming the host carries its history to the new name.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
assert await tl.carry_host_rename(s, name, renamed) == 3
|
|
assert await _entries(app, name) == []
|
|
assert len(await _entries(app, renamed)) == 3
|
|
|
|
asyncio.run(_go())
|
|
|
|
|
|
def test_settings_re_sort_and_hide_entries_already_written(app):
|
|
from sqlalchemy import select
|
|
from steward.models.timeline import TimelineEntry
|
|
from steward.timeline import catalog
|
|
from steward.timeline.config import Config
|
|
from steward.timeline.record import record
|
|
from steward.timeline.service import query
|
|
|
|
async def _go():
|
|
host = f"tl-{uuid.uuid4().hex[:8]}"
|
|
now = datetime.now(timezone.utc)
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
record(s, type="ansible.changed", at=now, summary="ran", hosts=[host])
|
|
record(s, type="ansible.check", at=now, summary="dry run", hosts=[host])
|
|
record(s, type="alert.fired", at=now, summary="hot", hosts=[host], area="hardware")
|
|
|
|
async def types(cfg, **kw):
|
|
async with app.db_sessionmaker() as s:
|
|
rows = (await s.execute(query(cfg, host=host, **kw))).scalars().all()
|
|
return sorted(e.type for e in rows)
|
|
|
|
areas = [list(a) for a in catalog.DEFAULT_AREAS]
|
|
cfg = Config(areas=areas)
|
|
assert await types(cfg) == ["alert.fired", "ansible.changed"] # the dry run is hidden
|
|
assert await types(cfg, include_hidden=True) == ["alert.fired", "ansible.changed", "ansible.check"]
|
|
assert await types(Config(areas=areas, types={"ansible.check": {"shown": True}})) == \
|
|
["alert.fired", "ansible.changed", "ansible.check"]
|
|
assert await types(cfg, area="host_os") == ["ansible.changed"]
|
|
assert await types(cfg, area="hardware") == ["alert.fired"]
|
|
assert await types(cfg, kind="automation", include_hidden=True) == ["ansible.changed", "ansible.check"]
|
|
|
|
# A type given an area moves every entry of it, past ones included.
|
|
moved = Config(areas=areas, types={"ansible.changed": {"area": "service"}})
|
|
assert await types(moved, area="service") == ["ansible.changed"]
|
|
assert await types(moved, area="host_os") == []
|
|
# A removed area's entries sort under the one they were moved to.
|
|
merged = Config(areas=[a for a in areas if a[0] != "hardware"], moved={"hardware": "storage"})
|
|
assert await types(merged, area="storage") == ["alert.fired"]
|
|
# An area nobody lists any more sorts under Other.
|
|
lost = Config(areas=[a for a in areas if a[0] != "hardware"])
|
|
assert await types(lost, area="other") == ["alert.fired"]
|
|
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
for e in (await s.execute(select(TimelineEntry).where(
|
|
TimelineEntry.hosts.contains([host])))).scalars():
|
|
await s.delete(e)
|
|
|
|
asyncio.run(_go())
|
|
|
|
|
|
def test_the_first_fill_writes_what_predates_the_timeline_once(app):
|
|
from steward.core.settings import get_setting, set_setting
|
|
from steward.models.alerts import AlertEvent, AlertOperator, AlertRule
|
|
from steward.models.users import User, UserRole
|
|
from steward.timeline.service import backfill
|
|
|
|
async def _go():
|
|
tag = uuid.uuid4().hex[:8]
|
|
name = f"tl-{tag}"
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
before = await get_setting(s, "timeline.backfilled_at")
|
|
user = User(username=f"tl-{tag}", email=f"tl-{tag}@test", password_hash="x",
|
|
role=UserRole.admin)
|
|
s.add(user)
|
|
await s.flush()
|
|
rule = AlertRule(name=f"disk {tag}", source_module="host_agent", resource_name=f"{name}:/",
|
|
metric_name="disk_used_pct", operator=AlertOperator.gt, threshold=90,
|
|
created_by=user.id)
|
|
s.add(rule)
|
|
await s.flush()
|
|
# Written straight to the table, as before the timeline existed.
|
|
s.add(AlertEvent(rule_id=rule.id, from_state="pending", to_state="firing", metric_value=95.0,
|
|
transitioned_at=datetime.now(timezone.utc) - timedelta(days=3)))
|
|
await set_setting(s, "timeline.backfilled_at", "")
|
|
try:
|
|
assert await backfill(app) >= 1
|
|
[entry] = await _entries(app, name)
|
|
assert entry.type == "alert.fired" and entry.area == "storage"
|
|
assert entry.subject == f"{name}:/"
|
|
assert await backfill(app) == 0 # once
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await set_setting(s, "timeline.backfilled_at", "")
|
|
await backfill(app) # again: what is covered is skipped
|
|
assert len(await _entries(app, name)) == 1
|
|
finally:
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await set_setting(s, "timeline.backfilled_at", before or "done")
|
|
|
|
asyncio.run(_go())
|
|
|
|
|
|
def test_retention_prunes_only_once_it_is_set(app):
|
|
from steward.core.settings import set_setting
|
|
from steward.timeline.record import record
|
|
from steward.timeline.service import prune
|
|
|
|
async def _go():
|
|
old = datetime.now(timezone.utc) - timedelta(days=400)
|
|
async with app.db_sessionmaker() as s:
|
|
record(s, type="steward.config", at=old, summary="old", hosts=[f"tl-{uuid.uuid4().hex[:8]}"])
|
|
await s.flush()
|
|
assert await prune(s, datetime.now(timezone.utc)) == (0, 0) # 0 days keeps everything
|
|
await set_setting(s, "timeline.retention_days", 365)
|
|
removed, days = await prune(s, datetime.now(timezone.utc))
|
|
assert days == 365 and removed >= 1
|
|
await s.rollback() # leave the shared database as it was
|
|
|
|
asyncio.run(_go())
|
|
|
|
|
|
def test_get_timeline_reads_a_host_s_history(app):
|
|
from steward.core.api_tokens import Caller
|
|
from steward.mcp import _context
|
|
from steward.mcp.tools.timeline import get_timeline
|
|
from steward.models.users import UserRole
|
|
from steward.timeline.record import record
|
|
|
|
async def _go():
|
|
host = f"tl-{uuid.uuid4().hex[:8]}"
|
|
now = datetime.now(timezone.utc)
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
record(s, type="ansible.changed", at=now - timedelta(hours=2), summary="update.yml: changed 1 host",
|
|
hosts=[host], detail={"url": "/ansible/runs/x"}, ref=("ansible_run", str(uuid.uuid4())))
|
|
record(s, type="ansible.check", at=now - timedelta(hours=1), summary="dry run", hosts=[host])
|
|
_context._app_ctx.set(app)
|
|
_context._caller_ctx.set(Caller(user_id="u", username="viewer", role=UserRole.viewer, token_id=0))
|
|
out = await get_timeline(host=host)
|
|
assert [e["summary"] for e in out["entries"]] == ["update.yml: changed 1 host"]
|
|
[e] = out["entries"]
|
|
assert e["kind"] == "automation" and e["area"] == "host_os" and e["hosts"] == [host]
|
|
assert e["details_kept"] is False and e["url"] is None # the run is not in the table
|
|
assert out["areas"]["host_os"] == "Host OS" and "alert.fired" in out["types"]
|
|
both = await get_timeline(host=host, include_hidden=True)
|
|
assert [x["type"] for x in both["entries"]] == ["ansible.check", "ansible.changed"]
|
|
assert (await get_timeline(host=host, kind="alert"))["entries"] == []
|
|
with pytest.raises(ValueError, match="kind"):
|
|
await get_timeline(kind="nope")
|
|
with pytest.raises(ValueError, match="area"):
|
|
await get_timeline(area="nope")
|
|
|
|
asyncio.run(_go())
|