feat(timeline): one record of what happened to the infrastructure, newest first (#6348)

A stored log, timeline_entries, with one row per thing that happened. Each
source writes its row in the same transaction as its own record, so the
history outlives the 30/90/365-day retention of the tables it points to:

- alert events (core/alerts._add_event): fired, resolved, acknowledged,
  went silent, readings back;
- finished Ansible runs (executor, and the startup sweep of interrupted
  runs): changed something, changed nothing, failed, dry run;
- audit rows (core/audit.log_audit): Steward's settings, users and access,
  sign-ins, backups. Actions another writer records are left to it.

Entries carry a kind (fixed) and an area (Service, Host OS, Hardware,
Storage, Network, Security, Monitoring). Settings → Timeline edits the
areas and each type's area and visibility; both are applied when the
timeline is read, so a change re-sorts past entries and a hidden type
comes back when shown. Kept forever unless a retention is set.

The first start fills the timeline from the last 90 days of records once,
skipping any already covered. A host rename carries its entries.

UI: Timeline in the sidebar (filter by host, area, kind; scrolls on), a
History card on each host page. MCP: get_timeline.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_timeline.py

@@ -0,0 +1,265 @@
+"""Integration: the timeline against real Postgres. Each source writes its
+entry as the thing happens, the entry outlives its record, Settings re-sorts
+and hides entries when they are read, the first fill skips what is covered,
+and get_timeline reads it back. The database is shared with other tests, so
+each test looks for its own uniquely named host.
+Requires STEWARD_DATABASE_URL.
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+async def _entries(app, host: str):
+    from sqlalchemy import select
+    from steward.models.timeline import TimelineEntry
+    async with app.db_sessionmaker() as s:
+        return (await s.execute(select(TimelineEntry).where(TimelineEntry.hosts.contains([host]))
+                                .order_by(TimelineEntry.at))).scalars().all()
+
+
+def test_each_source_writes_an_entry_that_outlives_its_record(app):
+    from sqlalchemy import delete
+    from steward.alerts import service as alert_svc
+    from steward.core.alerts import record_metric, record_metrics
+    from steward.core.audit import log_audit
+    from steward.models.alerts import AlertEvent
+    from steward.models.ansible import AnsibleRun, AnsibleRunStatus
+    from steward.models.hosts import Host
+    from steward.models.metrics import PluginMetric
+    from steward.models.users import User, UserRole
+    from steward.timeline import config as tl_config, service as tl
+    from steward.timeline.record import run_finished
+
+    async def _go():
+        tag = uuid.uuid4().hex[:8]
+        name, renamed, run_id = f"tl-{tag}", f"tl-{tag}-new", str(uuid.uuid4())
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                user = User(username=f"tl-{tag}", email=f"tl-{tag}@test", password_hash="x",
+                            role=UserRole.admin)
+                host = Host(name=name, address="10.1.2.3")
+                s.add_all([user, host])
+                await record_metrics(s, [PluginMetric(source_module="host_agent", resource_name=name,
+                                                      metric_name="cpu_pct", value=1.0)])
+                await s.flush()
+                fields = await alert_svc.rule_fields(
+                    s, name=f"hot {tag}", source_module="host_agent", resource_name=name,
+                    metric_name="cpu_pct", operator=">", threshold=90,
+                    consecutive_failures_required=1, consecutive_ok_required=1)
+                rule = await alert_svc.add_rule(s, fields, created_by=user.id)
+            user_id, host_id, rule_id = user.id, host.id, rule.id
+
+        # An alert firing.
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await record_metric(s, "host_agent", name, "cpu_pct", 97.0)
+        # A change to Steward's settings about the host, and one the timeline leaves to the run.
+        await log_audit(app, user_id, f"tl-{tag}", "host.created", "host", host_id, {"name": name})
+        await log_audit(app, user_id, f"tl-{tag}", "ansible.run.started", "ansible_run", run_id, {})
+        # A playbook run that changed the host.
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add(AnsibleRun(id=run_id, playbook_path="update.yml", source_name="local",
+                                 inventory_scope=f"steward:target:{uuid.uuid4()}",
+                                 status=AnsibleRunStatus.success, triggered_by=user_id,
+                                 finished_at=datetime.now(timezone.utc),
+                                 results={"hosts": {name: {"ok": 3, "changed": 1}}, "failures": []}))
+            async with s.begin():
+                await run_finished(s, run_id)
+
+        got = {e.type: e for e in await _entries(app, name)}
+        assert set(got) == {"alert.fired", "steward.config", "ansible.changed"}
+        fired = got["alert.fired"]
+        assert fired.summary == f"hot {tag} fired: CPU on {name} is 97% (> 90%)"
+        assert fired.area == "host_os" and fired.ref_type == "alert_event"
+        assert got["steward.config"].summary == f"Added host {name}"
+        assert got["steward.config"].actor == f"tl-{tag}"
+        assert got["ansible.changed"].summary.endswith(": changed 1 host")
+        assert got["ansible.changed"].actor == f"tl-{tag}"
+
+        # The records age out; the entries stay, saying their details are gone.
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await s.execute(delete(AlertEvent).where(AlertEvent.rule_id == rule_id))
+                await s.execute(delete(AnsibleRun).where(AnsibleRun.id == run_id))
+        async with app.db_sessionmaker() as s:
+            cfg = await tl_config.load(s)
+            card = {v["entry"].type: v for v in await tl.recent_for_host(s, cfg, name)}
+        assert set(card) == {"alert.fired", "steward.config", "ansible.changed"}
+        assert card["alert.fired"]["gone"] and card["alert.fired"]["url"] is None
+        assert card["ansible.changed"]["gone"]
+        assert not card["steward.config"]["gone"]
+
+        # Renaming the host carries its history to the new name.
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                assert await tl.carry_host_rename(s, name, renamed) == 3
+        assert await _entries(app, name) == []
+        assert len(await _entries(app, renamed)) == 3
+
+    asyncio.run(_go())
+
+
+def test_settings_re_sort_and_hide_entries_already_written(app):
+    from sqlalchemy import select
+    from steward.models.timeline import TimelineEntry
+    from steward.timeline import catalog
+    from steward.timeline.config import Config
+    from steward.timeline.record import record
+    from steward.timeline.service import query
+
+    async def _go():
+        host = f"tl-{uuid.uuid4().hex[:8]}"
+        now = datetime.now(timezone.utc)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                record(s, type="ansible.changed", at=now, summary="ran", hosts=[host])
+                record(s, type="ansible.check", at=now, summary="dry run", hosts=[host])
+                record(s, type="alert.fired", at=now, summary="hot", hosts=[host], area="hardware")
+
+        async def types(cfg, **kw):
+            async with app.db_sessionmaker() as s:
+                rows = (await s.execute(query(cfg, host=host, **kw))).scalars().all()
+            return sorted(e.type for e in rows)
+
+        areas = [list(a) for a in catalog.DEFAULT_AREAS]
+        cfg = Config(areas=areas)
+        assert await types(cfg) == ["alert.fired", "ansible.changed"]          # the dry run is hidden
+        assert await types(cfg, include_hidden=True) == ["alert.fired", "ansible.changed", "ansible.check"]
+        assert await types(Config(areas=areas, types={"ansible.check": {"shown": True}})) == \
+            ["alert.fired", "ansible.changed", "ansible.check"]
+        assert await types(cfg, area="host_os") == ["ansible.changed"]
+        assert await types(cfg, area="hardware") == ["alert.fired"]
+        assert await types(cfg, kind="automation", include_hidden=True) == ["ansible.changed", "ansible.check"]
+
+        # A type given an area moves every entry of it, past ones included.
+        moved = Config(areas=areas, types={"ansible.changed": {"area": "service"}})
+        assert await types(moved, area="service") == ["ansible.changed"]
+        assert await types(moved, area="host_os") == []
+        # A removed area's entries sort under the one they were moved to.
+        merged = Config(areas=[a for a in areas if a[0] != "hardware"], moved={"hardware": "storage"})
+        assert await types(merged, area="storage") == ["alert.fired"]
+        # An area nobody lists any more sorts under Other.
+        lost = Config(areas=[a for a in areas if a[0] != "hardware"])
+        assert await types(lost, area="other") == ["alert.fired"]
+
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                for e in (await s.execute(select(TimelineEntry).where(
+                        TimelineEntry.hosts.contains([host])))).scalars():
+                    await s.delete(e)
+
+    asyncio.run(_go())
+
+
+def test_the_first_fill_writes_what_predates_the_timeline_once(app):
+    from steward.core.settings import get_setting, set_setting
+    from steward.models.alerts import AlertEvent, AlertOperator, AlertRule
+    from steward.models.users import User, UserRole
+    from steward.timeline.service import backfill
+
+    async def _go():
+        tag = uuid.uuid4().hex[:8]
+        name = f"tl-{tag}"
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                before = await get_setting(s, "timeline.backfilled_at")
+                user = User(username=f"tl-{tag}", email=f"tl-{tag}@test", password_hash="x",
+                            role=UserRole.admin)
+                s.add(user)
+                await s.flush()
+                rule = AlertRule(name=f"disk {tag}", source_module="host_agent", resource_name=f"{name}:/",
+                                 metric_name="disk_used_pct", operator=AlertOperator.gt, threshold=90,
+                                 created_by=user.id)
+                s.add(rule)
+                await s.flush()
+                # Written straight to the table, as before the timeline existed.
+                s.add(AlertEvent(rule_id=rule.id, from_state="pending", to_state="firing", metric_value=95.0,
+                                 transitioned_at=datetime.now(timezone.utc) - timedelta(days=3)))
+                await set_setting(s, "timeline.backfilled_at", "")
+        try:
+            assert await backfill(app) >= 1
+            [entry] = await _entries(app, name)
+            assert entry.type == "alert.fired" and entry.area == "storage"
+            assert entry.subject == f"{name}:/"
+            assert await backfill(app) == 0                         # once
+            async with app.db_sessionmaker() as s:
+                async with s.begin():
+                    await set_setting(s, "timeline.backfilled_at", "")
+            await backfill(app)                                      # again: what is covered is skipped
+            assert len(await _entries(app, name)) == 1
+        finally:
+            async with app.db_sessionmaker() as s:
+                async with s.begin():
+                    await set_setting(s, "timeline.backfilled_at", before or "done")
+
+    asyncio.run(_go())
+
+
+def test_retention_prunes_only_once_it_is_set(app):
+    from steward.core.settings import set_setting
+    from steward.timeline.record import record
+    from steward.timeline.service import prune
+
+    async def _go():
+        old = datetime.now(timezone.utc) - timedelta(days=400)
+        async with app.db_sessionmaker() as s:
+            record(s, type="steward.config", at=old, summary="old", hosts=[f"tl-{uuid.uuid4().hex[:8]}"])
+            await s.flush()
+            assert await prune(s, datetime.now(timezone.utc)) == (0, 0)    # 0 days keeps everything
+            await set_setting(s, "timeline.retention_days", 365)
+            removed, days = await prune(s, datetime.now(timezone.utc))
+            assert days == 365 and removed >= 1
+            await s.rollback()                                        # leave the shared database as it was
+
+    asyncio.run(_go())
+
+
+def test_get_timeline_reads_a_host_s_history(app):
+    from steward.core.api_tokens import Caller
+    from steward.mcp import _context
+    from steward.mcp.tools.timeline import get_timeline
+    from steward.models.users import UserRole
+    from steward.timeline.record import record
+
+    async def _go():
+        host = f"tl-{uuid.uuid4().hex[:8]}"
+        now = datetime.now(timezone.utc)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                record(s, type="ansible.changed", at=now - timedelta(hours=2), summary="update.yml: changed 1 host",
+                       hosts=[host], detail={"url": "/ansible/runs/x"}, ref=("ansible_run", str(uuid.uuid4())))
+                record(s, type="ansible.check", at=now - timedelta(hours=1), summary="dry run", hosts=[host])
+        _context._app_ctx.set(app)
+        _context._caller_ctx.set(Caller(user_id="u", username="viewer", role=UserRole.viewer, token_id=0))
+        out = await get_timeline(host=host)
+        assert [e["summary"] for e in out["entries"]] == ["update.yml: changed 1 host"]
+        [e] = out["entries"]
+        assert e["kind"] == "automation" and e["area"] == "host_os" and e["hosts"] == [host]
+        assert e["details_kept"] is False and e["url"] is None              # the run is not in the table
+        assert out["areas"]["host_os"] == "Host OS" and "alert.fired" in out["types"]
+        both = await get_timeline(host=host, include_hidden=True)
+        assert [x["type"] for x in both["entries"]] == ["ansible.check", "ansible.changed"]
+        assert (await get_timeline(host=host, kind="alert"))["entries"] == []
+        with pytest.raises(ValueError, match="kind"):
+            await get_timeline(kind="nope")
+        with pytest.raises(ValueError, match="area"):
+            await get_timeline(area="nope")
+
+    asyncio.run(_go())