feat(timeline): one record of what happened to the infrastructure, newest first (#6348)

A stored log, timeline_entries, with one row per thing that happened. Each
source writes its row in the same transaction as its own record, so the
history outlives the 30/90/365-day retention of the tables it points to:

- alert events (core/alerts._add_event): fired, resolved, acknowledged,
  went silent, readings back;
- finished Ansible runs (executor, and the startup sweep of interrupted
  runs): changed something, changed nothing, failed, dry run;
- audit rows (core/audit.log_audit): Steward's settings, users and access,
  sign-ins, backups. Actions another writer records are left to it.

Entries carry a kind (fixed) and an area (Service, Host OS, Hardware,
Storage, Network, Security, Monitoring). Settings → Timeline edits the
areas and each type's area and visibility; both are applied when the
timeline is read, so a change re-sorts past entries and a hidden type
comes back when shown. Kept forever unless a retention is set.

The first start fills the timeline from the last 90 days of records once,
skipping any already covered. A host rename carries its entries.

UI: Timeline in the sidebar (filter by host, area, kind; scrolls on), a
History card on each host page. MCP: get_timeline.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/timeline/__init__.py

@@ -0,0 +1,2 @@
+"""The timeline: one record of what happened to the infrastructure, from
+every source, newest first (catalog, config, record, service)."""

steward/timeline/catalog.py

@@ -0,0 +1,137 @@
+"""What can appear on the timeline: each entry type, the kind of event it is,
+and the area it sorts under until Settings → Timeline says otherwise.
+
+An area is what part of the infrastructure was affected (Service, Network,
+Hardware …); a kind is how the event came about (a change someone made, an
+automation run, an alert …). Kinds are fixed here. Areas, and each type's
+area and visibility, are settings (steward.timeline.config), so they can be
+changed without a release and the change re-sorts entries already written.
+
+A plugin adds its own types with register_type() from its setup().
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from steward.core.settings import DEFAULTS
+
+# How an event came about. Fixed: the page's "Kind" filter and the MCP
+# tool's `kind` argument take these keys.
+KINDS: dict[str, str] = {
+    "change": "Change",          # a person or an agent did it
+    "automation": "Automation",  # a playbook or maintenance run
+    "alert": "Alert",
+    "anomaly": "Anomaly",
+    "ticket": "Ticket",
+    "steward": "Steward",        # Steward's own settings and access
+}
+
+# The starting areas, [key, label]: the timeline.areas setting's default.
+# Settings → Timeline renames, adds and removes them.
+DEFAULT_AREAS: list[list[str]] = DEFAULTS["timeline.areas"]
+# Where an entry goes whose area is unknown: removed without a destination,
+# or named by a plugin that is no longer loaded.
+OTHER = ("other", "Other")
+
+
+@dataclass(frozen=True)
+class EntryType:
+    key: str
+    label: str        # how Settings → Timeline names it
+    kind: str         # a KINDS key
+    area: str         # the area its entries are written with, unless the writer names one
+    shown: bool       # whether the timeline shows it until Settings says otherwise
+    about: str        # one line for Settings → Timeline
+
+
+TYPES: dict[str, EntryType] = {}
+
+
+def register_type(key: str, label: str, kind: str, area: str, shown: bool, about: str) -> None:
+    if kind not in KINDS:
+        raise ValueError(f"timeline type {key!r}: unknown kind {kind!r}")
+    TYPES[key] = EntryType(key, label, kind, area, shown, about)
+
+
+for _t in (
+    ("alert.fired", "Alert fired", "alert", "monitoring", True,
+     "An alert rule's condition held long enough to fire."),
+    ("alert.resolved", "Alert resolved", "alert", "monitoring", True,
+     "A firing alert cleared, or its rule was changed."),
+    ("alert.acknowledged", "Alert acknowledged", "alert", "monitoring", True,
+     "Someone took a firing alert."),
+    ("alert.silent", "Readings stopped", "alert", "monitoring", True,
+     "What an alert rule watches stopped reporting."),
+    ("alert.resumed", "Readings resumed", "alert", "monitoring", True,
+     "What an alert rule watches reported again after a silence."),
+    ("ansible.changed", "Playbook changed something", "automation", "host_os", True,
+     "A playbook run finished and changed at least one host."),
+    ("ansible.unchanged", "Playbook changed nothing", "automation", "host_os", False,
+     "A playbook run finished with nothing to change."),
+    ("ansible.failed", "Playbook failed", "automation", "host_os", True,
+     "A playbook run failed, was cancelled, or was stopped by a restart."),
+    ("ansible.check", "Playbook dry run", "automation", "host_os", False,
+     "A playbook run in check mode: it showed what would change."),
+    ("steward.config", "Steward settings changed", "steward", "monitoring", True,
+     "Monitors, alert rules, maintenance windows, inventory, plugins, settings."),
+    ("steward.access", "Users and access", "steward", "security", True,
+     "Users, roles, passwords and AI access tokens."),
+    ("steward.signin", "Sign-ins", "steward", "security", False,
+     "Someone signed in, or failed to."),
+    ("steward.backup", "Steward backups", "steward", "storage", True,
+     "Steward backed itself up, or was restored from a backup."),
+):
+    register_type(*_t)
+
+
+# Which area an alert's entry is written with, from what its rule watches.
+# A source not listed here is Monitoring.
+_SOURCE_AREAS = {
+    "icmp": "network", "dns": "network", "tcp": "service", "http": "service",
+    "docker": "service", "prometheus": "service",
+    "traefik": "network", "unifi": "network",
+    "snmp": "hardware",
+}
+# host_agent metrics by prefix, first match wins; the rest are Host OS.
+_HOST_METRIC_AREAS = (
+    ("updates_security", "security"),
+    ("temp_", "hardware"), ("drive", "hardware"),
+    ("disk_", "storage"), ("inodes_", "storage"), ("mounts_", "storage"), ("nfs_", "storage"),
+    ("net_", "network"),
+)
+
+
+def area_of_metric(source: str, metric: str) -> str:
+    """The area a reading of `metric` from `source` is about."""
+    if source == "host_agent":
+        for prefix, area in _HOST_METRIC_AREAS:
+            if metric.startswith(prefix):
+                return area
+        return "host_os"
+    if source == "snmp" and any(w in metric for w in ("volume", "disk_used", "storage")):
+        return "storage"
+    return _SOURCE_AREAS.get(source, "monitoring")
+
+
+# Audit actions and the type each is entered as. The first prefix that
+# matches wins; None leaves the action off the timeline, because another
+# writer records the event itself (a run, an acknowledgement) or it is about
+# one person's own preferences.
+_AUDIT_TYPES = (
+    ("ansible.run.", None), ("ansible.host_run", None),
+    ("alert_rule.acknowledged", None), ("alert_rule.tested", None),
+    ("settings.notification.tested", None),
+    ("account.time_zone", None), ("account.email", None),
+    ("host_agent.maintenance_plan", "steward.config"), ("host_agent.maintenance", None),
+    ("docker.prune.", None),
+    ("auth.login", "steward.signin"), ("auth.setup", "steward.config"),
+    ("account.", "steward.access"), ("user.", "steward.access"), ("api_token.", "steward.access"),
+    ("backup.", "steward.backup"),
+)
+
+
+def audit_type(action: str) -> str | None:
+    for prefix, type_ in _AUDIT_TYPES:
+        if action.startswith(prefix):
+            return type_
+    return "steward.config"

steward/timeline/config.py

@@ -0,0 +1,150 @@
+"""Settings → Timeline, as the reader applies it.
+
+Three settings, read together:
+- timeline.areas: [[key, label], …], the areas in the order the page lists them.
+- timeline.types: {type: {"area": key, "shown": bool}}, each only where it
+  departs from the catalog. An "area" there sorts every entry of the type
+  under it; without one, an entry sorts under the area it was written with.
+- timeline.area_moved: {removed key: key its entries now sort under}.
+
+Nothing here is written onto an entry, so a change re-sorts the whole
+history, and a hidden type's entries come back when it is shown again.
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from sqlalchemy import and_, false, or_, true
+
+from . import catalog
+from steward.models.timeline import TimelineEntry
+
+
+@dataclass
+class Config:
+    areas: list[list[str]]
+    types: dict[str, dict] = field(default_factory=dict)
+    moved: dict[str, str] = field(default_factory=dict)
+
+    @property
+    def labels(self) -> dict[str, str]:
+        return {key: label for key, label in self.areas}
+
+    def follow(self, area: str) -> str:
+        """Where an area's entries sort now: through any removals, to a live
+        area, or Other."""
+        seen = set()
+        while area in self.moved and area not in seen:
+            seen.add(area)
+            area = self.moved[area]
+        return area if area in self.labels else catalog.OTHER[0]
+
+    def type_area(self, type_: str) -> str | None:
+        """The area Settings sorts every entry of this type under, or None."""
+        return (self.types.get(type_) or {}).get("area") or None
+
+    def shown(self, type_: str) -> bool:
+        override = (self.types.get(type_) or {}).get("shown")
+        if override is not None:
+            return bool(override)
+        known = catalog.TYPES.get(type_)
+        return known.shown if known else True
+
+    def area_of(self, entry) -> str:
+        return self.follow(self.type_area(entry.type) or entry.area)
+
+    def area_label(self, key: str) -> str:
+        return self.labels.get(key) or catalog.OTHER[1]
+
+    def hidden_types(self) -> set[str]:
+        keys = set(catalog.TYPES) | set(self.types)
+        return {k for k in keys if not self.shown(k)}
+
+    def area_filter(self, area: str):
+        """A SQL condition for "entries sorting under `area`"."""
+        sources = [k for k in [*self.labels, *self.moved, catalog.OTHER[0]]
+                   if self.follow(k) == area]
+        fixed = {t for t, o in self.types.items() if o.get("area")}
+        to_here = [t for t in fixed if self.follow(self.types[t]["area"]) == area]
+        written = TimelineEntry.area.in_(sources) if sources else false()
+        if area == catalog.OTHER[0]:
+            # An area nobody listed (a plugin's, gone) sorts under Other too.
+            written = or_(written, TimelineEntry.area.notin_([*self.labels, *self.moved]))
+        unfixed = TimelineEntry.type.notin_(fixed) if fixed else true()
+        return or_(TimelineEntry.type.in_(to_here) if to_here else false(),
+                   and_(unfixed, written))
+
+
+def kind_types(kind: str) -> list[str]:
+    return [t.key for t in catalog.TYPES.values() if t.kind == kind]
+
+
+async def load(db) -> Config:
+    from steward.core.settings import get_setting
+    areas = await get_setting(db, "timeline.areas") or catalog.DEFAULT_AREAS
+    return Config(areas=[list(a) for a in areas],
+                  types=dict(await get_setting(db, "timeline.types") or {}),
+                  moved=dict(await get_setting(db, "timeline.area_moved") or {}))
+
+
+def slug(label: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "_", label.strip().lower()).strip("_")[:32]
+
+
+def edited(cfg: Config, form) -> tuple[Config, str | None]:
+    """The settings Settings → Timeline's form asks for, or the reason it
+    can't have them. The form carries, per area, `area_label_<key>` and
+    `area_move_<key>` ("" to keep it, else the key its entries go to);
+    `new_area` for one more; and per type `type_area_<type>` ("" for as
+    written) and `type_shown_<type>` (present when ticked)."""
+    areas, moved = [], dict(cfg.moved)
+    for key, label in cfg.areas:
+        dest = (form.get(f"area_move_{key}") or "").strip()
+        if dest:
+            if dest == key or dest not in cfg.labels:
+                return cfg, f"Pick another area for {label}'s entries to go to."
+            moved[key] = dest
+            continue
+        areas.append([key, (form.get(f"area_label_{key}") or "").strip() or label])
+    new = (form.get("new_area") or "").strip()
+    if new:
+        key = slug(new)
+        if not key or key == catalog.OTHER[0] or key in {k for k, _ in cfg.areas}:
+            return cfg, f"There is already an area like “{new}”; pick another name."
+        areas.append([key, new])
+        moved.pop(key, None)
+    if not areas:
+        return cfg, "Keep at least one area."
+    live = {k for k, _ in areas}
+    # An area listed again is no longer moved; one moved into an area that
+    # was itself moved goes on to wherever that went (Config.follow).
+    moved = {k: d for k, d in moved.items() if k not in live}
+
+    after = Config(areas=areas, moved=moved)
+    types = {}
+    for key, t in catalog.TYPES.items():
+        override = {}
+        area = (form.get(f"type_area_{key}") or "").strip()
+        if area:
+            # A type sorted under an area removed in this same save follows its entries.
+            area = after.follow(area)
+            if area not in live:
+                return cfg, f"“{t.label}” names an area that isn't listed."
+            override["area"] = area
+        shown = f"type_shown_{key}" in form
+        if shown != t.shown:
+            override["shown"] = shown
+        if override:
+            types[key] = override
+    # Types of a plugin that isn't loaded keep what they had.
+    types.update({k: v for k, v in cfg.types.items() if k not in catalog.TYPES})
+    after.types = types
+    return after, None
+
+
+async def save(db, cfg: Config) -> None:
+    from steward.core.settings import set_setting
+    await set_setting(db, "timeline.areas", cfg.areas)
+    await set_setting(db, "timeline.types", cfg.types)
+    await set_setting(db, "timeline.area_moved", cfg.moved)

steward/timeline/record.py

@@ -0,0 +1,233 @@
+"""Writing the timeline. Every source writes through record(), in the same
+transaction as its own record, as the thing happens: the entry stands on its
+own, so it is still there after the source's own row ages out.
+
+from_alert, from_run and from_audit build the entry for one alert event,
+finished Ansible run and audit row. The live writers and the backfill use the
+same three, so an entry reads the same however it got there.
+"""
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+
+from sqlalchemy import select
+
+from . import catalog
+from steward.models.timeline import TimelineEntry
+
+logger = logging.getLogger(__name__)
+
+
+def record(session, *, type: str, at: datetime, summary: str, area: str | None = None,
+           hosts=(), subject: str | None = None, detail: dict | None = None,
+           actor: str = "system", until: datetime | None = None,
+           ref: tuple[str, str] | None = None) -> TimelineEntry:
+    """Add one entry to `session`. `area` defaults to its type's; `ref` is
+    (kind, id) of the full record it points to, which is at most one entry."""
+    known = catalog.TYPES.get(type)
+    entry = TimelineEntry(
+        at=at, until=until, type=type,
+        area=area or (known.area if known else catalog.OTHER[0]),
+        hosts=sorted({h for h in hosts if h}), subject=subject[:255] if subject else None, summary=summary,
+        detail=detail or None, actor=(actor or "system")[:64],
+        ref_type=ref[0] if ref else None, ref_id=str(ref[1]) if ref else None)
+    session.add(entry)
+    return entry
+
+
+async def username(session, user_id: str | None) -> str | None:
+    if not user_id:
+        return None
+    from steward.models.users import User
+    user = await session.get(User, user_id)
+    return user.username if user else None
+
+
+# ── Alerts ──────────────────────────────────────────────────────────────────
+
+_MONITOR_SOURCES = ("icmp", "tcp", "dns", "http")
+
+
+async def hosts_of_rule(session, source: str, resource: str) -> list[str]:
+    """The host an alert rule's resource is on, by name, when it has one: an
+    agent series ("<host>" or "<host>:<part>"), a container ("<host>/<name>")
+    or a monitor tied to a host."""
+    from steward.core.alerts import host_of_resource
+    if source == "host_agent":
+        return [host_of_resource(source, resource) or resource.split(":", 1)[0]]
+    if source == "docker":
+        return [resource.split("/", 1)[0]]
+    if source in _MONITOR_SOURCES:
+        from steward.models.hosts import Host
+        from steward.models.monitors import Monitor
+        name = (await session.execute(
+            select(Host.name).join(Monitor, Monitor.host_id == Host.id)
+            .where(Monitor.type == source, Monitor.name == resource).limit(1))).scalar()
+        return [name] if name else []
+    return []
+
+
+def _alert_type(from_state: str, to_state: str) -> str | None:
+    if to_state == "firing":
+        return "alert.fired"
+    if to_state == "resolved":
+        return "alert.resolved"
+    if to_state == "acknowledged":
+        return "alert.acknowledged"
+    if to_state == "no_data":
+        return "alert.silent"
+    if from_state == "no_data":
+        return "alert.resumed"
+    return None
+
+
+def alert_summary(type_: str, rule, value: float | None) -> str:
+    from steward.alerts.service import _BINARY
+    from steward.core.formatting import metric_label, metric_value
+    metric, res = rule.metric_name, rule.resource_name
+    said = metric_label(metric)
+    # The reading, and on a firing the condition it broke.
+    if metric in _BINARY:
+        reading = breach = f"{res} is {metric_value(metric, value)}"
+    else:
+        op = getattr(rule.operator, "value", rule.operator)
+        reading = f"{said} on {res} is {metric_value(metric, value)}"
+        breach = f"{reading} ({op} {metric_value(metric, rule.threshold)})"
+    return {
+        "alert.fired": f"{rule.name} fired: {breach}" if value is not None else f"{rule.name} fired",
+        "alert.resolved": f"{rule.name} resolved" + (f": {reading}" if value is not None else ""),
+        "alert.acknowledged": f"{rule.name} acknowledged",
+        "alert.silent": f"{rule.name}: no {said} reading from {res}",
+        "alert.resumed": f"{rule.name}: {said} readings from {res} again",
+    }[type_]
+
+
+async def from_alert(session, rule, event, actor: str | None = None,
+                     resumed: bool = False) -> TimelineEntry | None:
+    """The entry for one alert event (an AlertEvent row), or None for a
+    transition the timeline doesn't show (pending and back). `resumed` is a
+    series back from silence with the alert's state unchanged, which can be
+    an already-firing alert still firing."""
+    type_ = "alert.resumed" if resumed else _alert_type(event.from_state, event.to_state)
+    if type_ is None:
+        return None
+    return record(
+        session, type=type_, at=event.transitioned_at,
+        summary=alert_summary(type_, rule, event.metric_value),
+        area=catalog.area_of_metric(rule.source_module, rule.metric_name),
+        hosts=await hosts_of_rule(session, rule.source_module, rule.resource_name),
+        subject=rule.resource_name,
+        detail={"rule_id": rule.id, "rule": rule.name, "source": rule.source_module,
+                "metric": rule.metric_name, "value": event.metric_value, "note": event.note,
+                "url": f"/alerts/history?rule={rule.id}"},
+        actor=actor or await username(session, getattr(event, "actor_id", None)) or "system",
+        ref=("alert_event", event.id))
+
+
+# ── Ansible runs ────────────────────────────────────────────────────────────
+
+def _run_type(run) -> str:
+    status = getattr(run.status, "value", run.status)
+    if status != "success":
+        return "ansible.failed"
+    if (run.params or {}).get("check"):
+        return "ansible.check"
+    hosts = ((run.results or {}).get("hosts") or {}).values()
+    return "ansible.changed" if any((h or {}).get("changed") for h in hosts) else "ansible.unchanged"
+
+
+async def hosts_of_run(session, names) -> list[str]:
+    """The Steward hosts behind a run's inventory names: a target linked to a
+    host is that host; a name that is a host's own is that host."""
+    from steward.models.ansible_inventory import AnsibleTarget
+    from steward.models.hosts import Host
+    names = set(names)
+    if not names:
+        return []
+    linked = dict((await session.execute(
+        select(AnsibleTarget.name, Host.name).join(Host, AnsibleTarget.host_id == Host.id)
+        .where(AnsibleTarget.name.in_(names)))).all())
+    direct = set((await session.execute(
+        select(Host.name).where(Host.name.in_(names - set(linked))))).scalars())
+    return sorted(set(linked.values()) | direct)
+
+
+async def from_run(session, run) -> TimelineEntry:
+    """The entry for one finished Ansible run."""
+    from steward.ansible.inventory_gen import run_title, scope_labels
+    from steward.ansible.origins import run_origins
+    type_ = _run_type(run)
+    per_host = (run.results or {}).get("hosts") or {}
+    scope = (await scope_labels(session, [run.inventory_scope])).get(run.inventory_scope)
+    title = run_title(run.playbook_path, scope)
+    status = getattr(run.status, "value", run.status)
+    changed = sum(1 for h in per_host.values() if (h or {}).get("changed"))
+    failed = sum(1 for h in per_host.values() if (h or {}).get("failed") or (h or {}).get("unreachable"))
+    if type_ == "ansible.failed":
+        said = {"cancelled": "cancelled", "interrupted": "stopped by a restart"}.get(status, "failed")
+        tail = f"{said} on {failed} host{'s' if failed != 1 else ''}" if failed else said
+    elif type_ == "ansible.check":
+        tail = f"dry run, would change {changed} host{'s' if changed != 1 else ''}" if changed \
+            else "dry run, nothing to change"
+    elif type_ == "ansible.changed":
+        tail = f"changed {changed} host{'s' if changed != 1 else ''}"
+    else:
+        tail = "nothing to change"
+    origin = (await run_origins(session, [run])).get(run.id) or {}
+    return record(
+        session, type=type_, at=run.started_at, until=run.finished_at,
+        summary=f"{title}: {tail}", hosts=await hosts_of_run(session, per_host),
+        subject=run.playbook_path,
+        detail={"status": status, "scope": scope, "origin": origin.get("text"),
+                "failures": ((run.results or {}).get("failures") or [])[:3],
+                "url": f"/ansible/runs/{run.id}"},
+        actor=origin.get("text") or "Steward", ref=("ansible_run", run.id))
+
+
+async def run_finished(session, run_id: str) -> None:
+    """Write the entry for a run that has just recorded its final status, in
+    a savepoint of the caller's transaction. Never raises: the run's own
+    record matters more than its entry."""
+    from steward.models.ansible import AnsibleRun
+    try:
+        async with session.begin_nested():
+            run = await session.get(AnsibleRun, run_id)
+            if run is not None:
+                await from_run(session, run)
+    except Exception:
+        logger.exception("Ansible run %s: its timeline entry could not be written", run_id)
+
+
+# ── The audit log ───────────────────────────────────────────────────────────
+
+async def from_audit(session, event, detail: dict | None) -> TimelineEntry | None:
+    """The entry for one audit row (an AuditEvent), or None for an action the
+    timeline leaves to another writer (catalog.audit_type)."""
+    from steward.audit.service import action_text, entity_labels
+    type_ = catalog.audit_type(event.action)
+    if type_ is None:
+        return None
+    detail = detail or {}
+    entity = None
+    if event.entity_type and event.entity_id:
+        labels = await entity_labels(session, [{"event": event, "detail": detail}])
+        entity = labels.get((event.entity_type, str(event.entity_id)))
+    what = action_text(event.action)
+    if entity:
+        what = f"{what} {entity['label']}"
+    hosts = []
+    if event.entity_type == "host" and entity:
+        hosts = [entity["label"]]
+    elif event.entity_type == "monitor":
+        from steward.models.hosts import Host
+        from steward.models.monitors import Monitor
+        name = (await session.execute(
+            select(Host.name).join(Monitor, Monitor.host_id == Host.id)
+            .where(Monitor.id == str(event.entity_id)))).scalar()
+        hosts = [name] if name else []
+    return record(
+        session, type=type_, at=event.timestamp, summary=what[:1].upper() + what[1:],
+        hosts=hosts, subject=entity["label"] if entity else None,
+        detail={"action": event.action, "url": entity.get("url") if entity else None},
+        actor=event.username, ref=("audit_event", event.id))

steward/timeline/routes.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+from quart import Blueprint, current_app, redirect, render_template, request, session, url_for
+from sqlalchemy import select
+
+from steward.audit.service import mark_day_groups
+from steward.auth.middleware import require_role
+from steward.core import formatting as fmt
+from steward.core.audit import audit_request
+from steward.core.paging import keyset, next_page
+from steward.models.hosts import Host
+from steward.models.timeline import TimelineEntry
+from steward.models.users import UserRole
+from steward.timeline import catalog, config as tl_config, service as tl
+
+timeline_bp = Blueprint("timeline", __name__)
+
+
+def _filters() -> dict:
+    a = request.args
+    return {"host": a.get("host", "").strip(), "area": a.get("area", "").strip(),
+            "kind": a.get("kind", "").strip(), "hidden": a.get("hidden") == "1"}
+
+
+@timeline_bp.get("/timeline/")
+@require_role(UserRole.viewer)
+async def timeline():
+    """Everything that happened, newest first, a page at a time as the reader
+    scrolls; ?before=<cursor> is the next page's rows alone (core.paging)."""
+    f = _filters()
+    async with current_app.db_sessionmaker() as db:
+        cfg = await tl_config.load(db)
+        stmt = tl.query(cfg, host=f["host"], area=f["area"], kind=f["kind"],
+                        include_hidden=f["hidden"])
+        stmt = keyset(stmt, TimelineEntry.at, TimelineEntry.id, request.args.get("before"))
+        rows, after = next_page((await db.execute(stmt)).scalars().all(), lambda e: (e.at, e.id))
+        items = await tl.views(db, cfg, rows)
+        hosts = [] if request.args.get("before") else \
+            list((await db.execute(select(Host.name).order_by(Host.name))).scalars())
+    last_group = mark_day_groups(items, fmt.zone(session.get("tz")), request.args.get("group", ""),
+                                 at=lambda item: item["entry"].at)
+    keep = {k: v for k, v in (("host", f["host"]), ("area", f["area"]), ("kind", f["kind"]),
+                              ("hidden", "1" if f["hidden"] else "")) if v}
+    ctx = {"items": items, "after": after, "last_group": last_group, "filters": f, "keep": keep,
+           "areas": cfg.areas, "kinds": catalog.KINDS,
+           "hosts": hosts if not f["host"] or f["host"] in hosts else [f["host"], *hosts]}
+    if request.args.get("before"):
+        return await render_template("timeline/_rows.html", **ctx)
+    return await render_template("timeline/list.html", **ctx)
+
+
+@timeline_bp.get("/timeline/host/<host_id>")
+@require_role(UserRole.viewer)
+async def host_card(host_id: str):
+    """A host's newest entries, for its page (loaded into it)."""
+    async with current_app.db_sessionmaker() as db:
+        host = await db.get(Host, host_id)
+        if host is None:
+            return "", 404
+        cfg = await tl_config.load(db)
+        items = await tl.recent_for_host(db, cfg, host.name)
+    return await render_template("timeline/_host_card.html", host=host, items=items)
+
+
+@timeline_bp.get("/settings/timeline/")
+@require_role(UserRole.admin)
+async def settings_page():
+    async with current_app.db_sessionmaker() as db:
+        cfg = await tl_config.load(db)
+        from steward.core.settings import get_setting
+        retention = int(await get_setting(db, "timeline.retention_days") or 0)
+    return await render_template("settings/timeline.html", **_settings_ctx(cfg, retention))
+
+
+def _settings_ctx(cfg, retention: int, error: str | None = None) -> dict:
+    by_kind = {k: [t for t in catalog.TYPES.values() if t.kind == k] for k in catalog.KINDS}
+    return {"cfg": cfg, "retention": retention, "error": error, "kinds": catalog.KINDS,
+            "by_kind": {k: v for k, v in by_kind.items() if v}}
+
+
+@timeline_bp.post("/settings/timeline/")
+@require_role(UserRole.admin)
+async def save_settings():
+    from steward.core.settings import set_setting
+    form = await request.form
+    try:
+        retention = max(0, int(form.get("retention_days") or 0))
+    except ValueError:
+        retention = None
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            cfg = await tl_config.load(db)
+            new, error = tl_config.edited(cfg, form)
+            if error is None and retention is None:
+                error = "Keep for: a whole number of days, or 0 to keep everything."
+            if error is not None:
+                kept = retention if retention is not None else 0
+                return await render_template("settings/timeline.html",
+                                             **_settings_ctx(cfg, kept, error)), 400
+            await tl_config.save(db, new)
+            await set_setting(db, "timeline.retention_days", retention)
+    await audit_request("settings.saved", detail={"section": "timeline"})
+    return redirect(url_for("timeline.settings_page"))

steward/timeline/service.py

@@ -0,0 +1,196 @@
+"""Reading the timeline, and keeping it: the filtered list the page, the host
+card and get_timeline share, the first fill from records that predate it,
+its retention, and a host rename carried onto its entries."""
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import delete, func, select, text, update
+from sqlalchemy.dialects.postgresql import insert
+
+from . import catalog
+from .config import Config, kind_types
+from steward.models.timeline import TimelineEntry
+
+logger = logging.getLogger(__name__)
+
+# How far back the first fill reaches: the default retention of the records it reads.
+BACKFILL_DAYS = 90
+
+
+def query(cfg: Config, *, host: str = "", area: str = "", kind: str = "", type_: str = "",
+          since: datetime | None = None, include_hidden: bool = False):
+    """The entries matching the filters, unordered (core.paging orders them).
+    Hidden types are left out unless `include_hidden`, or a `type_` asks for one."""
+    stmt = select(TimelineEntry)
+    if host:
+        stmt = stmt.where(TimelineEntry.hosts.contains([host]))
+    if area:
+        stmt = stmt.where(cfg.area_filter(area))
+    if kind:
+        stmt = stmt.where(TimelineEntry.type.in_(kind_types(kind)))
+    if type_:
+        stmt = stmt.where(TimelineEntry.type == type_)
+    elif not include_hidden:
+        hidden = cfg.hidden_types()
+        if hidden:
+            stmt = stmt.where(TimelineEntry.type.notin_(hidden))
+    if since is not None:
+        stmt = stmt.where(TimelineEntry.at >= since)
+    return stmt
+
+
+# The full records an entry can point to, and whether one is still kept.
+def _ref_models() -> dict:
+    from steward.models.alerts import AlertEvent
+    from steward.models.ansible import AnsibleRun
+    from steward.models.audit import AuditEvent
+    return {"alert_event": AlertEvent, "ansible_run": AnsibleRun, "audit_event": AuditEvent}
+
+
+async def kept_refs(db, entries) -> set[tuple[str, str]]:
+    """The (ref_type, ref_id) of `entries` whose full record still exists."""
+    wanted: dict[str, set[str]] = {}
+    for e in entries:
+        if e.ref_type and e.ref_id:
+            wanted.setdefault(e.ref_type, set()).add(e.ref_id)
+    kept = set()
+    models = _ref_models()
+    for ref_type, ids in wanted.items():
+        model = models.get(ref_type)
+        if model is None:           # a kind only a plugin knows: assume it is kept
+            kept |= {(ref_type, i) for i in ids}
+            continue
+        found = (await db.execute(select(model.id).where(model.id.in_(ids)))).scalars()
+        kept |= {(ref_type, str(i)) for i in found}
+    return kept
+
+
+def view(entry: TimelineEntry, cfg: Config, kept: set) -> dict:
+    """One entry as the page and the MCP tool read it."""
+    known = catalog.TYPES.get(entry.type)
+    area = cfg.area_of(entry)
+    detail = entry.detail or {}
+    url = detail.get("url")
+    gone = bool(entry.ref_type) and (entry.ref_type, entry.ref_id) not in kept
+    return {
+        "entry": entry,
+        "type_label": known.label if known else entry.type,
+        "kind": known.kind if known else None,
+        "kind_label": catalog.KINDS.get(known.kind) if known else None,
+        "area": area,
+        "area_label": cfg.area_label(area),
+        "note": detail.get("note"),
+        "url": None if gone else url,
+        "gone": gone,
+    }
+
+
+async def views(db, cfg: Config, entries) -> list[dict]:
+    kept = await kept_refs(db, entries)
+    return [view(e, cfg, kept) for e in entries]
+
+
+async def recent_for_host(db, cfg: Config, host: str, limit: int = 8) -> list[dict]:
+    """A host's newest shown entries, for its page."""
+    rows = (await db.execute(
+        query(cfg, host=host).order_by(TimelineEntry.at.desc(), TimelineEntry.id.desc())
+        .limit(limit))).scalars().all()
+    return await views(db, cfg, rows)
+
+
+# ── Keeping it ──────────────────────────────────────────────────────────────
+
+async def prune(session, now: datetime) -> tuple[int, int]:
+    """Delete entries past timeline.retention_days; (rows, days), with 0 days
+    keeping everything. Call inside a transaction."""
+    from steward.core.settings import get_setting
+    days = int(await get_setting(session, "timeline.retention_days") or 0)
+    if days <= 0:
+        return 0, 0
+    result = await session.execute(
+        delete(TimelineEntry).where(TimelineEntry.at < now - timedelta(days=days)))
+    return result.rowcount or 0, days
+
+
+async def carry_host_rename(session, old: str, new: str) -> int:
+    """Point a renamed host's entries at its new name, so its page still
+    shows its history."""
+    if not old or old == new:
+        return 0
+    result = await session.execute(
+        update(TimelineEntry).where(TimelineEntry.hosts.contains([old]))
+        .values(hosts=func.array_replace(TimelineEntry.hosts, old, new))
+        .execution_options(synchronize_session=False))
+    return result.rowcount or 0
+
+
+async def backfill(app, now: datetime | None = None) -> int:
+    """Fill the timeline once from the alert events, Ansible runs and audit
+    rows that predate it, BACKFILL_DAYS back, through the same builders the
+    live writers use. An entry for a record already covered is skipped, so
+    it is safe to run again. Records the run in timeline.backfilled_at."""
+    from steward.core.settings import get_setting, set_setting
+    from steward.models.alerts import AlertEvent, AlertRule
+    from steward.models.ansible import AnsibleRun
+    from steward.models.audit import AuditEvent
+    from .record import from_alert, from_audit, from_run
+
+    now = now or datetime.now(timezone.utc)
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            if await get_setting(session, "timeline.backfilled_at"):
+                return 0
+        since = now - timedelta(days=BACKFILL_DAYS)
+        built: list[TimelineEntry] = []
+        async with session.begin():
+            # The builders add each entry to the session; no flush may write
+            # them before the insert below, which skips any already covered.
+            session.sync_session.autoflush = False
+
+            async def build(what: str, coro) -> None:
+                # One record that can't be read is skipped, not the whole fill.
+                try:
+                    built.append(await coro)
+                except Exception:
+                    logger.exception("Timeline fill: %s skipped", what)
+
+            for event, rule in (await session.execute(
+                    select(AlertEvent, AlertRule).join(AlertRule, AlertEvent.rule_id == AlertRule.id)
+                    .where(AlertEvent.transitioned_at >= since))).all():
+                await build(f"alert event {event.id}", from_alert(session, rule, event))
+            for run in (await session.execute(
+                    select(AnsibleRun).where(AnsibleRun.started_at >= since,
+                                             AnsibleRun.finished_at.is_not(None)))).scalars().all():
+                await build(f"Ansible run {run.id}", from_run(session, run))
+            for event in (await session.execute(
+                    select(AuditEvent).where(AuditEvent.timestamp >= since))).scalars().all():
+                await build(f"audit row {event.id}", from_audit(session, event, _audit_detail(event)))
+            rows = [e for e in built if e is not None]
+            for entry in rows:
+                session.expunge(entry)
+            for chunk in range(0, len(rows), 500):
+                await session.execute(
+                    insert(TimelineEntry).values([_row(e) for e in rows[chunk:chunk + 500]])
+                    .on_conflict_do_nothing(index_elements=["ref_type", "ref_id"],
+                                            index_where=text("ref_id IS NOT NULL")))
+            await set_setting(session, "timeline.backfilled_at", now.isoformat())
+    logger.info("Timeline: filled %d entries from the last %d days of records", len(rows), BACKFILL_DAYS)
+    return len(rows)
+
+
+def _audit_detail(event) -> dict:
+    import json
+    try:
+        return json.loads(event.detail_json) if event.detail_json else {}
+    except (TypeError, ValueError):
+        return {}
+
+
+def _row(entry: TimelineEntry) -> dict:
+    from steward.core.ids import time_ordered_id
+    return {"id": entry.id or time_ordered_id(), "at": entry.at, "until": entry.until,
+            "type": entry.type, "area": entry.area, "hosts": entry.hosts or [],
+            "subject": entry.subject, "summary": entry.summary, "detail": entry.detail,
+            "actor": entry.actor or "system", "ref_type": entry.ref_type, "ref_id": entry.ref_id}