From 4ea735d87c23244dabb5dbe34fa1a93e34772b77 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 24 Oct 2026 17:03:05 -0400 Subject: [PATCH] feat(timeline): one record of what happened to the infrastructure, newest first (#6348) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A stored log, timeline_entries, with one row per thing that happened. Each source writes its row in the same transaction as its own record, so the history outlives the 30/90/365-day retention of the tables it points to: - alert events (core/alerts._add_event): fired, resolved, acknowledged, went silent, readings back; - finished Ansible runs (executor, and the startup sweep of interrupted runs): changed something, changed nothing, failed, dry run; - audit rows (core/audit.log_audit): Steward's settings, users and access, sign-ins, backups. Actions another writer records are left to it. Entries carry a kind (fixed) and an area (Service, Host OS, Hardware, Storage, Network, Security, Monitoring). Settings → Timeline edits the areas and each type's area and visibility; both are applied when the timeline is read, so a change re-sorts past entries and a hidden type comes back when shown. Kept forever unless a retention is set. The first start fills the timeline from the last 90 days of records once, skipping any already covered. A host rename carries its entries. UI: Timeline in the sidebar (filter by host, area, kind; scrolls on), a History card on each host page. MCP: get_timeline. Co-Authored-By: Claude Opus 5.5 --- README.md | 1 + docs/core/mcp.md | 1 + docs/core/timeline.md | 48 ++++ steward/alerts/service.py | 10 +- steward/ansible/executor.py | 3 + steward/app.py | 26 +- steward/backup/contents.py | 2 + steward/core/alerts.py | 32 ++- steward/core/audit.py | 18 +- steward/core/cleanup.py | 5 + steward/core/settings.py | 11 + steward/hosts/routes.py | 2 + steward/mcp/tools/__init__.py | 5 +- steward/mcp/tools/alerts.py | 5 +- steward/mcp/tools/timeline.py | 76 ++++++ steward/migrations/versions/0047_timeline.py | 43 +++ steward/models/__init__.py | 2 + steward/models/timeline.py | 47 ++++ steward/templates/base.html | 1 + steward/templates/hosts/detail.html | 5 + steward/templates/settings/_tabs.html | 1 + steward/templates/settings/timeline.html | 76 ++++++ steward/templates/timeline/_entry.html | 21 ++ steward/templates/timeline/_host_card.html | 25 ++ steward/templates/timeline/_rows.html | 21 ++ steward/templates/timeline/list.html | 54 ++++ steward/timeline/__init__.py | 2 + steward/timeline/catalog.py | 137 ++++++++++ steward/timeline/config.py | 150 +++++++++++ steward/timeline/record.py | 233 ++++++++++++++++ steward/timeline/routes.py | 103 +++++++ steward/timeline/service.py | 196 ++++++++++++++ tests/core/test_alert_no_data.py | 9 +- tests/core/test_timeline.py | 270 +++++++++++++++++++ tests/integration/test_timeline.py | 265 ++++++++++++++++++ tests/test_breadcrumbs.py | 1 + 36 files changed, 1882 insertions(+), 25 deletions(-) create mode 100644 docs/core/timeline.md create mode 100644 steward/mcp/tools/timeline.py create mode 100644 steward/migrations/versions/0047_timeline.py create mode 100644 steward/models/timeline.py create mode 100644 steward/templates/settings/timeline.html create mode 100644 steward/templates/timeline/_entry.html create mode 100644 steward/templates/timeline/_host_card.html create mode 100644 steward/templates/timeline/_rows.html create mode 100644 steward/templates/timeline/list.html create mode 100644 steward/timeline/__init__.py create mode 100644 steward/timeline/catalog.py create mode 100644 steward/timeline/config.py create mode 100644 steward/timeline/record.py create mode 100644 steward/timeline/routes.py create mode 100644 steward/timeline/service.py create mode 100644 tests/core/test_timeline.py create mode 100644 tests/integration/test_timeline.py diff --git a/README.md b/README.md index e258252..b5b87ec 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,7 @@ See `docs/plugins/` for a full plugin development guide. | `docs/core/alerting.md` | Alert rules, state machine, notification channels, template variables | | `docs/core/ansible.md` | Playbook sources, run lifecycle, SSE streaming | | `docs/core/backups.md` | Steward's daily backup to an S3 bucket (Fabled Archives), what it holds, and `steward restore` | +| `docs/core/timeline.md` | The timeline: what it records from each source, areas and kinds, Settings → Timeline | | `docs/plugins/overview.md` | Plugin system: how loading works, plugin.yaml schema, required exports | | `docs/plugins/writing-a-plugin.md` | Step-by-step plugin development guide | | `docs/plugins/traefik.md` | Traefik plugin: config, metrics, alert rule examples | diff --git a/docs/core/mcp.md b/docs/core/mcp.md index de0a5a4..2ff2ed7 100644 --- a/docs/core/mcp.md +++ b/docs/core/mcp.md @@ -78,6 +78,7 @@ Each role includes the tools of the roles above it in this list. | Metric anomalies | `list_anomalies` (metrics that left their normal for the hour, to review first), `get_anomaly` (one anomaly with each occurrence against what is usual at that hour, the metric around it, what else happened on its host — alerts, Ansible runs, container deaths and starts, maintenance, reboots, other anomalies — and how earlier ones on the same metric were judged), `get_metric_normal` (the learned normal slot by slot, and the level that would be flagged), `get_anomaly_settings` (the review window and each floor) | | Automation | `list_playbooks`, `get_playbook`, `list_inventory`, `list_ansible_runs`, `get_ansible_run`, `wait_for_run`, `list_schedules` | | Audit | `list_audit_events` | +| Timeline | `get_timeline` (everything that happened, newest first, in one list: alerts firing and clearing, playbook runs and what they changed, changes to Steward's settings; by host, area or kind, and kept after the records it came from age out) | | Docker plugin | `list_containers`, `get_container`, `list_swarm_services`, `search_container_logs`, `get_docker_disk_usage` | | Host agent (built in, always on) | `list_agents`, `list_maintenance_plans` (each plan's group, weekly slot, next run and latest run: how far it got, who it skipped and why), `get_maintenance_job` (a host's update jobs or one job: state, packages updated, whether it rebooted, why it failed), `get_host_senders` (who sent or received a host's network traffic over a window: its containers and NFS mounts ranked at the host's peak, and what none of them accounts for) | | Traefik plugin | `list_routers` (each router graded as the Traefik page grades it, with the status codes behind a problem, what they mean and the swarm service behind it), `get_router` (one router's history, codes, normal and anomalies), `list_certificates` (served certificates, days left, the routers they cover), `get_traffic` (the access log's clients, countries and routers, and Traefik's own figures) | diff --git a/docs/core/timeline.md b/docs/core/timeline.md new file mode 100644 index 0000000..d447a89 --- /dev/null +++ b/docs/core/timeline.md @@ -0,0 +1,48 @@ +# Timeline + +The timeline is one list of everything that happened to the infrastructure, newest first: alerts firing and clearing, playbook runs and what they changed, and changes to Steward's own settings. Open it from **Timeline** in the sidebar, from a host's **History** card, or through the `get_timeline` MCP tool. + +Each entry stands on its own. An alert event, an Ansible run or an audit row is pruned after its own retention, but the entry for it stays. Its link then reads "Details no longer kept". + +**Source:** `steward/timeline/` +**Table:** `timeline_entries` (`steward/models/timeline.py`) + +--- + +## What is recorded + +Every source writes its entry in the same transaction as its own record, through `steward/timeline/record.py`: + +| Source | Written by | Entry types | +|---|---|---| +| Alert events | `core/alerts._add_event` | `alert.fired`, `alert.resolved`, `alert.acknowledged`, `alert.silent`, `alert.resumed` | +| Ansible runs, when they finish | `ansible/executor.start_run`, and the startup sweep of interrupted runs | `ansible.changed`, `ansible.unchanged`, `ansible.failed`, `ansible.check` | +| Audit rows | `core/audit.log_audit` | `steward.config`, `steward.access`, `steward.signin`, `steward.backup` | + +Some audit actions are left to another writer, because the event itself is recorded there: starting a run, acknowledging an alert, and starting maintenance. A person's own preferences (time zone, email settings) are not recorded. `catalog.audit_type` holds that mapping. + +On its first start, Steward fills the timeline from the last 90 days of alert events, finished runs and audit rows (`service.backfill`). It does this once, and records when in `timeline.backfilled_at`. An entry for a record that is already covered is skipped. + +## Areas and kinds + +An entry has a **kind**, which says how it came about: Change, Automation, Alert, Anomaly, Ticket or Steward. Kinds are fixed in `catalog.KINDS`. + +An entry also has an **area**, which says what part of the infrastructure it is about. The areas start as Service, Host OS, Hardware, Storage, Network, Security and Monitoring. + +Each entry is written with an area of its own. An alert's area comes from what its rule watches (`catalog.area_of_metric`): drive health is Hardware, a mount is Storage, an HTTP monitor is Service. Other entries use their type's area. + +## Settings → Timeline + +| Setting | Key | Default | +|---|---|---| +| Areas, in order: rename, add, or remove one and move its entries to another | `timeline.areas`, `timeline.area_moved` | the seven above | +| Each type's area ("As recorded" keeps each entry's own) and whether it is shown | `timeline.types` | from the catalog | +| Keep for, in days (0 keeps everything) | `timeline.retention_days` | 0 | + +These are applied when the timeline is read, never written onto an entry. A change re-sorts the whole history. A hidden type is still recorded, so showing it again brings its entries back. + +Hidden by default: dry runs, playbook runs that changed nothing, and sign-ins. + +## Adding a type + +A plugin registers its types in `setup()` with `catalog.register_type(key, label, kind, area, shown, about)`, and writes entries with `record.record(session, type=…, at=…, summary=…, hosts=…, ref=(kind, id))`. Settings → Timeline lists the new type at once. diff --git a/steward/alerts/service.py b/steward/alerts/service.py index 158c1db..7c32bad 100644 --- a/steward/alerts/service.py +++ b/steward/alerts/service.py @@ -833,10 +833,11 @@ class PartOfCoveringRule(ValueError): f"rule per host; change or disable it there.") -async def acknowledge(db, rule_id: str, user_id: str | None) -> AlertState: +async def acknowledge(db, rule_id: str, user_id: str | None, actor: str | None = None) -> AlertState: """Acknowledge a firing alert, in the caller's transaction, and record it in the alert's history as a firing → acknowledged event by `user_id`, so - who saw it outlives the alert (#5463). + who saw it outlives the alert (#5463). `actor` names them on the timeline + when it isn't their username alone (" via MCP"). LookupError when the rule has no state; ValueError when it is not firing. """ @@ -853,8 +854,9 @@ async def acknowledge(db, rule_id: str, user_id: str | None) -> AlertState: state.acknowledged_by = user_id state.acknowledged_at = now state.last_transition_at = now - _add_event(db, rule_id, AlertStateEnum.firing.value, AlertStateEnum.acknowledged.value, - None, now, actor_id=user_id) + rule = await db.get(AlertRule, rule_id) + await _add_event(db, rule, AlertStateEnum.firing.value, AlertStateEnum.acknowledged.value, + None, now, actor_id=user_id, actor=actor) return state diff --git a/steward/ansible/executor.py b/steward/ansible/executor.py index 315feb1..898e400 100644 --- a/steward/ansible/executor.py +++ b/steward/ansible/executor.py @@ -574,6 +574,9 @@ async def start_run( await session.execute( update(AnsibleRun).where(AnsibleRun.id == run_id).values(**values) ) + if final_status is not None: + from steward.timeline.record import run_finished + await run_finished(session, run_id) lines_since_flush = 0 last_flush_at = time.monotonic() diff --git a/steward/app.py b/steward/app.py index 34051c5..f5337cc 100644 --- a/steward/app.py +++ b/steward/app.py @@ -233,6 +233,7 @@ def create_app( from .audit.routes import audit_bp from .ai_access.routes import ai_access_bp from .account.routes import account_bp + from .timeline.routes import timeline_bp app.register_blueprint(auth_bp) app.register_blueprint(dashboard_bp) @@ -247,6 +248,7 @@ def create_app( app.register_blueprint(audit_bp) app.register_blueprint(ai_access_bp) app.register_blueprint(account_bp) + app.register_blueprint(timeline_bp) # Register the unified Monitor status source for the Status page. Plugins # may register additional sources from setup(). Idempotent. @@ -580,15 +582,32 @@ def _register_core_tasks(app: Quart) -> None: ) ) + # The timeline's first fill from the records that predate it, after the + # first boot that has it; once done, each later turn is one settings read. + async def run_timeline_backfill(): + from .timeline.service import backfill + await backfill(app) + + app._task_registry.append( + ScheduledTask( + name="timeline_backfill", + coro_factory=run_timeline_backfill, + interval_seconds=86400, + run_on_startup=True, + ) + ) + async def _mark_interrupted_runs(app: Quart) -> None: from sqlalchemy import update from .models.ansible import AnsibleRun, AnsibleRunStatus from datetime import datetime, timezone + from .timeline.record import run_finished + async with app.db_sessionmaker() as session: async with session.begin(): - await session.execute( + stopped = (await session.execute( update(AnsibleRun) .where(AnsibleRun.status.in_( [AnsibleRunStatus.running, AnsibleRunStatus.queued])) @@ -596,7 +615,10 @@ async def _mark_interrupted_runs(app: Quart) -> None: status=AnsibleRunStatus.interrupted, finished_at=datetime.now(timezone.utc), ) - ) + .returning(AnsibleRun.id) + )).scalars().all() + for run_id in stopped: + await run_finished(session, run_id) async def _run_scheduler(app: Quart) -> None: diff --git a/steward/backup/contents.py b/steward/backup/contents.py index 4ea7e97..167e6b3 100644 --- a/steward/backup/contents.py +++ b/steward/backup/contents.py @@ -35,6 +35,8 @@ KEPT: frozenset[str] = frozenset({ EXCLUDED: frozenset[str] = frozenset({ # Logs and history (the 2026-10-15 ruling): a recovery doesn't need them. "audit_events", "alert_events", + # The timeline of what happened to the infrastructure: history. + "timeline_entries", # Live state: each rule starts inactive, and alert_reconcile gives a # restored rule its state row within a minute (ensure_rule_states). "alert_states", diff --git a/steward/core/alerts.py b/steward/core/alerts.py index c05d752..cbeb64e 100644 --- a/steward/core/alerts.py +++ b/steward/core/alerts.py @@ -571,11 +571,12 @@ async def _evaluate_rule( state.acknowledged_at = None said = "; ".join(n for n in (resumed, note) if n) or None if step.event: - event_id = _add_event(session, rule.id, "no_data" if resumed else from_state.value, - step.event, value, now, note=said) + event_id = await _add_event(session, rule, "no_data" if resumed else from_state.value, + step.event, value, now, note=said) notifications.append((step.event, event_id, None)) elif resumed: - event_id = _add_event(session, rule.id, "no_data", state.state.value, value, now, note=said) + event_id = await _add_event(session, rule, "no_data", state.state.value, value, now, note=said, + resumed=True) notifications.append(("data_resumed", event_id, resumed)) await session.flush() return notifications @@ -595,8 +596,8 @@ async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str, return now = datetime.now(timezone.utc) if state.state in (AlertStateEnum.firing, AlertStateEnum.acknowledged): - _add_event(session, rule.id, state.state.value, "resolved", None, now, note=note, - actor_id=actor_id) + await _add_event(session, rule, state.state.value, "resolved", None, now, note=note, + actor_id=actor_id) # The watch starts over now: silence is measured from here, against the # cadence of whatever the rule watches from now on. state.last_transition_at = now @@ -610,21 +611,27 @@ async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str, state.acknowledged_at = None -def _add_event( +async def _add_event( session: AsyncSession, - rule_id: str, + rule: AlertRule, from_state: str, to_state: str, value: float | None, now: datetime, note: str | None = None, actor_id: str | None = None, + actor: str | None = None, + resumed: bool = False, ) -> str: - """Add an AlertEvent row and return its ID.""" + """Add an AlertEvent row, and its timeline entry, and return its ID. + `actor` names who did it on the timeline when `actor_id`'s username + alone would not say it (" via MCP"). `resumed` marks a series + back from silence that left the alert's state as it was.""" + from steward.timeline.record import from_alert event_id = str(uuid.uuid4()) event = AlertEvent( id=event_id, - rule_id=rule_id, + rule_id=rule.id, from_state=from_state, to_state=to_state, metric_value=value, @@ -633,6 +640,11 @@ def _add_event( actor_id=actor_id, ) session.add(event) + try: + await from_alert(session, rule, event, actor, resumed=resumed) + except Exception: + # The alert's own history matters more than its timeline entry. + logger.exception("Alert event %s: its timeline entry could not be written", event_id) return event_id @@ -825,7 +837,7 @@ async def check_silence(app: "Quart") -> None: muted = _muted_by(windows, rule) # Silent with its agent: recorded, and "agent offline" is the email (#5721). hushed = AGENT_OFFLINE if host_behind(rule) in offline else None - event_id = _add_event(session, rule.id, state.state.value, "no_data", None, now, + event_id = await _add_event(session, rule, state.state.value, "no_data", None, now, note="; ".join(n for n in (note, hushed, muted) if n)) if rule.notify_no_data and not muted and not hushed: notify.append((rule, event_id, note)) diff --git a/steward/core/audit.py b/steward/core/audit.py index bb8eb47..ed4308f 100644 --- a/steward/core/audit.py +++ b/steward/core/audit.py @@ -22,12 +22,17 @@ async def log_audit( entity_id: str | None = None, detail: dict | None = None, ) -> None: - """Write one audit event. Never raises — failures are logged and swallowed.""" + """Write one audit event, and its timeline entry when the action has one + (steward.timeline.catalog.audit_type). Never raises — failures are logged + and swallowed.""" + from steward.core.ids import time_ordered_id from steward.models.audit import AuditEvent + from steward.timeline.record import from_audit try: async with app.db_sessionmaker() as db: async with db.begin(): - db.add(AuditEvent( + event = AuditEvent( + id=time_ordered_id(), user_id=user_id, username=username or "unknown", action=action, @@ -35,7 +40,14 @@ async def log_audit( entity_id=entity_id, detail_json=json.dumps(detail) if detail else None, timestamp=datetime.now(timezone.utc), - )) + ) + db.add(event) + try: + # In a savepoint: an entry that fails leaves the audit row standing. + async with db.begin_nested(): + await from_audit(db, event, detail) + except Exception: + logger.exception("Audit event %r: its timeline entry could not be written", action) except Exception: logger.exception("Failed to write audit event action=%r", action) diff --git a/steward/core/cleanup.py b/steward/core/cleanup.py index 85e2199..75e3805 100644 --- a/steward/core/cleanup.py +++ b/steward/core/cleanup.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING from sqlalchemy import delete from steward.core.anomalies import prune as prune_anomalies +from steward.timeline.service import prune as prune_timeline from steward.models.alerts import AlertEvent from steward.models.ansible import AnsibleRun from steward.models.audit import AuditEvent @@ -81,6 +82,10 @@ async def run_cleanup(app: "Quart") -> dict: pruned = await prune_anomalies(session, cutoff) removed.append([f"metric_anomalies: reviewed, ended over {data_days} days ago", pruned]) + # The timeline keeps its own window, everything by default. + pruned, days = await prune_timeline(session, now) + if days: + removed.append([f"timeline_entries: older than {days} days", pruned]) files = await asyncio.to_thread(prune_run_logs, ARTIFACT_DIR, cutoff) removed.append([f"Ansible run log files: older than {data_days} days", files]) diff --git a/steward/core/settings.py b/steward/core/settings.py index 13338ba..1d21b16 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -52,6 +52,17 @@ DEFAULTS: dict[str, Any] = { "data.retention_days": 90, # The audit log is a security record: kept longer than operational data. "audit.retention_days": 365, + # The timeline is the record of what happened to the infrastructure: kept + # until this many days are set (0 keeps it all). Its areas, and each entry + # type's area and visibility, are Settings → Timeline (steward/timeline). + "timeline.retention_days": 0, + "timeline.areas": [["service", "Service"], ["host_os", "Host OS"], ["hardware", "Hardware"], + ["storage", "Storage"], ["network", "Network"], ["security", "Security"], + ["monitoring", "Monitoring"]], + "timeline.types": {}, + "timeline.area_moved": {}, + # When the timeline was first filled from the records that predate it. + "timeline.backfilled_at": "", "monitors.poll_interval_seconds": 60, # Root log level: DEBUG, INFO, WARNING or ERROR (steward/core/logging_setup.py). # WARNING also hides the per-request access lines. diff --git a/steward/hosts/routes.py b/steward/hosts/routes.py index 657399c..a4c5253 100644 --- a/steward/hosts/routes.py +++ b/steward/hosts/routes.py @@ -13,6 +13,7 @@ from steward.models.ansible import AnsibleRun, AnsibleRunStatus from steward.models.hosts import Host from steward.models.users import UserRole from steward.hosts import service as host_svc +from steward.timeline.service import carry_host_rename hosts_bp = Blueprint("hosts", __name__, url_prefix="/hosts") @@ -202,6 +203,7 @@ async def update_host(host_id: str): # Agent and container metrics, and the rules and windows on them, # go by the host's name: move them with it. moved = await renames.carry_rename(db, renames.HOST_SOURCES, old_name, name) + await carry_host_rename(db, old_name, name) if moved: await audit_request("host.renamed", "host", name, {"host_id": host_id, "renamed_from": old_name, "carried": moved}) diff --git a/steward/mcp/tools/__init__.py b/steward/mcp/tools/__init__.py index 1d02a42..3748857 100644 --- a/steward/mcp/tools/__init__.py +++ b/steward/mcp/tools/__init__.py @@ -12,9 +12,10 @@ each argument's empty value means, and what comes back. """ from __future__ import annotations -from . import alerts, anomalies, audit, automation, backups, hosts, metrics, monitors, overview, session +from . import (alerts, anomalies, audit, automation, backups, hosts, metrics, monitors, overview, session, + timeline) -_MODULES = (session, overview, hosts, monitors, alerts, anomalies, metrics, audit, automation, +_MODULES = (session, overview, hosts, monitors, alerts, anomalies, metrics, audit, timeline, automation, backups) diff --git a/steward/mcp/tools/alerts.py b/steward/mcp/tools/alerts.py index 6772021..77a9297 100644 --- a/steward/mcp/tools/alerts.py +++ b/steward/mcp/tools/alerts.py @@ -11,7 +11,7 @@ from steward.mcp._context import current_caller, mcp_app from steward.models.alerts import AlertOperator, AlertRule, AlertState, MaintenanceWindow from steward.models.users import UserRole -from ._audit import audit_mcp +from ._audit import audit_mcp, mcp_username from ._shape import ( age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd, ) @@ -389,7 +389,8 @@ async def acknowledge_alert(rule: str) -> dict: async with db_session() as db, db.begin(): r = await _rule(db, rule) try: - state = await alert_svc.acknowledge(db, r.id, caller.user_id) + state = await alert_svc.acknowledge(db, r.id, caller.user_id, + mcp_username(caller.username)) except LookupError: raise ValueError(f"{r.name} has never been evaluated, so there is nothing to " "acknowledge.") from None diff --git a/steward/mcp/tools/timeline.py b/steward/mcp/tools/timeline.py new file mode 100644 index 0000000..ec64e7a --- /dev/null +++ b/steward/mcp/tools/timeline.py @@ -0,0 +1,76 @@ +"""The timeline: everything that happened to the infrastructure, in one list.""" +from __future__ import annotations + +from steward.models.timeline import TimelineEntry +from steward.models.users import UserRole +from steward.timeline import catalog, config as tl_config, service as tl + +from ._shape import age_s, bounded, clamp_limit, db_session, iso, lookback + + +async def get_timeline(host: str = "", area: str = "", kind: str = "", type: str = "", + range: str = "7d", include_hidden: bool = False, limit: int = 50) -> dict: + """What happened to the infrastructure, newest first, from every source in + one list: alerts firing, resolving and going silent; playbook runs and + what they changed; changes to Steward's own settings, users and backups. + Each entry outlives the record it came from, so this reaches further back + than get_alert_history, list_ansible_runs or list_audit_events. + + Reach for it first for "what changed on this host lately?" or "what else + was going on when this broke?". Follow an entry's `url` or ids into the + tool for that record while it is still kept (`details_kept`). + + host: a host's name, exactly; empty for every host. + area: what part of the infrastructure an entry is about, as Settings → + Timeline names its areas (`areas` in the reply lists them); empty for all. + kind: how it came about: "change", "automation", "alert", "anomaly", + "ticket" or "steward"; empty for all. + type: one entry type (`types` in the reply); it is returned even if hidden. + range: "1h", "6h", "24h", "7d" (default) or "30d". + include_hidden: also the types Settings hides from the timeline (dry runs, + sign-ins, playbook runs that changed nothing). + limit: default 50, max 500. + """ + if kind and kind not in catalog.KINDS: + raise ValueError(f"kind {kind!r} is not one of {', '.join(catalog.KINDS)}.") + since, key = lookback(range) + limit = clamp_limit(limit, 50, 500) + async with db_session() as db: + cfg = await tl_config.load(db) + if area and area not in cfg.labels and area != catalog.OTHER[0]: + raise ValueError(f"area {area!r} is not one of {', '.join(cfg.labels)}.") + stmt = tl.query(cfg, host=host.strip(), area=area, kind=kind, type_=type, + since=since, include_hidden=include_hidden) + rows = (await db.execute(stmt.order_by(TimelineEntry.at.desc(), TimelineEntry.id.desc()) + .limit(limit + 1))).scalars().all() + items = await tl.views(db, cfg, rows) + entries = [ + { + "at": iso(v["entry"].at), + "age_s": age_s(v["entry"].at), + "until": iso(v["entry"].until), + "type": v["entry"].type, + "kind": v["kind"], + "area": v["area"], + "summary": v["entry"].summary, + "note": v["note"], + "hosts": v["entry"].hosts, + "subject": v["entry"].subject, + "by": v["entry"].actor, + "url": v["url"], + "details_kept": not v["gone"], + "ref": {"type": v["entry"].ref_type, "id": v["entry"].ref_id} if v["entry"].ref_type else None, + } + for v in items + ] + out = {"range": key, **bounded(entries, limit, key="entries"), + "areas": dict(cfg.labels), "types": sorted(catalog.TYPES)} + if out["truncated"]: + out["total"] = None + out["more"] = (f"More than {limit} entries in range. Pass a larger `limit` (max 500), a " + "shorter `range`, or filter by `host`, `area`, `kind` or `type`.") + return out + + +def register(register_tool) -> None: + register_tool(get_timeline, role=UserRole.viewer, read_only=True) diff --git a/steward/migrations/versions/0047_timeline.py b/steward/migrations/versions/0047_timeline.py new file mode 100644 index 0000000..6d0bcb5 --- /dev/null +++ b/steward/migrations/versions/0047_timeline.py @@ -0,0 +1,43 @@ +"""The timeline: one row per thing that happened to the infrastructure. + +Revision ID: 0047_timeline +Revises: 0046_metric_anomalies +Create Date: 2026-10-24 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import ARRAY + +revision: str = "0047_timeline" +down_revision: Union[str, None] = "0046_metric_anomalies" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "timeline_entries", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("at", sa.DateTime(timezone=True), nullable=False), + sa.Column("until", sa.DateTime(timezone=True), nullable=True), + sa.Column("type", sa.String(64), nullable=False), + sa.Column("area", sa.String(32), nullable=False), + sa.Column("hosts", ARRAY(sa.String(128)), nullable=False, server_default="{}"), + sa.Column("subject", sa.String(255), nullable=True), + sa.Column("summary", sa.Text, nullable=False), + sa.Column("detail", sa.JSON, nullable=True), + sa.Column("actor", sa.String(64), nullable=False, server_default="system"), + sa.Column("ref_type", sa.String(32), nullable=True), + sa.Column("ref_id", sa.String(64), nullable=True), + ) + op.create_index("ix_timeline_entries_at", "timeline_entries", ["at", "id"]) + op.create_index("ix_timeline_entries_hosts", "timeline_entries", ["hosts"], + postgresql_using="gin") + op.create_index("ix_timeline_entries_type", "timeline_entries", ["type"]) + op.create_index("uq_timeline_entries_ref", "timeline_entries", ["ref_type", "ref_id"], + unique=True, postgresql_where=sa.text("ref_id IS NOT NULL")) + + +def downgrade() -> None: + op.drop_table("timeline_entries") diff --git a/steward/models/__init__.py b/steward/models/__init__.py index d559ad3..f860889 100644 --- a/steward/models/__init__.py +++ b/steward/models/__init__.py @@ -14,6 +14,7 @@ from .ansible_schedule import AnsibleSchedule from .settings import AppSetting from .dashboard import Dashboard, DashboardWidget, DashboardShareToken from .outbox import EmailOutbox, email_outbox_events +from .timeline import TimelineEntry __all__ = [ "Base", "User", "ApiToken", "UserInvite", "PasswordReset", @@ -27,4 +28,5 @@ __all__ = [ "AppSetting", "Dashboard", "DashboardWidget", "DashboardShareToken", "EmailOutbox", "email_outbox_events", + "TimelineEntry", ] diff --git a/steward/models/timeline.py b/steward/models/timeline.py new file mode 100644 index 0000000..f1c7564 --- /dev/null +++ b/steward/models/timeline.py @@ -0,0 +1,47 @@ +from __future__ import annotations +from datetime import datetime +from sqlalchemy import JSON, DateTime, Index, String, Text, text +from sqlalchemy.dialects.postgresql import ARRAY +from sqlalchemy.orm import Mapped, mapped_column +from .base import Base +from steward.core.ids import time_ordered_id + + +class TimelineEntry(Base): + """One thing that happened to the infrastructure: an alert firing, a + playbook run, a change to Steward's settings. Each row stands on its own, + so it outlives the record it points to (`ref_type` / `ref_id`) once that + record's retention removes it. + + `type` is a key of steward.timeline.catalog. Whether a type is shown, and + which area it sorts under, are settings applied when the timeline is read; + `area` is the area the event itself named when it was written, used unless + Settings → Timeline gives its type one.""" + __tablename__ = "timeline_entries" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id) + at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + # When something that lasted (a run) ended. + until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + type: Mapped[str] = mapped_column(String(64), nullable=False) + area: Mapped[str] = mapped_column(String(32), nullable=False) + # The hosts it happened on, by name, as every other record keyed by host. + hosts: Mapped[list[str]] = mapped_column(ARRAY(String(128)), nullable=False, default=list) + # What it happened to, when that is not a host: a container, a rule's resource. + subject: Mapped[str | None] = mapped_column(String(255), nullable=True) + summary: Mapped[str] = mapped_column(Text, nullable=False) + detail: Mapped[dict | None] = mapped_column(JSON, nullable=True) + # A username, " via MCP", or "system". + actor: Mapped[str] = mapped_column(String(64), nullable=False, default="system") + ref_type: Mapped[str | None] = mapped_column(String(32), nullable=True) + ref_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + + __table_args__ = ( + Index("ix_timeline_entries_at", "at", "id"), + Index("ix_timeline_entries_hosts", "hosts", postgresql_using="gin"), + Index("ix_timeline_entries_type", "type"), + # One entry per source record, so a backfill can run over rows the + # live writers already covered. + Index("uq_timeline_entries_ref", "ref_type", "ref_id", unique=True, + postgresql_where=text("ref_id IS NOT NULL")), + ) diff --git a/steward/templates/base.html b/steward/templates/base.html index f2e04e7..6a4902b 100644 --- a/steward/templates/base.html +++ b/steward/templates/base.html @@ -42,6 +42,7 @@
Overview
Dashboard + Timeline
Infrastructure
diff --git a/steward/templates/hosts/detail.html b/steward/templates/hosts/detail.html index 3d6275b..fd9847e 100644 --- a/steward/templates/hosts/detail.html +++ b/steward/templates/hosts/detail.html @@ -202,6 +202,11 @@ {% endif %}
{# Ansible card #} +{# ── History: what happened to this host lately (timeline) ─────────────────── #} +
+
Loading history…
+
+ {# ── Docker (docker plugin fragment; renders nothing if the host has none). After Ansible: the container list can run long, and Ansible beneath it was easy to miss (#5214). ─────────────────────────────────────────────────────── #} diff --git a/steward/templates/settings/_tabs.html b/steward/templates/settings/_tabs.html index c2206f3..499aff4 100644 --- a/steward/templates/settings/_tabs.html +++ b/steward/templates/settings/_tabs.html @@ -8,6 +8,7 @@ ("notifications", "Notifications", "/settings/notifications/"), ("reports", "Reports", "/settings/reports/"), ("backups", "Backups", "/settings/backups/"), + ("timeline", "Timeline", "/settings/timeline/"), ("auth", "Authentication", "/settings/auth/"), ("users", "Users", "/settings/users/"), ("ansible", "Ansible", "/settings/ansible/"), diff --git a/steward/templates/settings/timeline.html b/steward/templates/settings/timeline.html new file mode 100644 index 0000000..75faea5 --- /dev/null +++ b/steward/templates/settings/timeline.html @@ -0,0 +1,76 @@ +{# settings/timeline.html — the timeline's areas, and for each type of entry + the area it sorts under and whether it is shown. Applied when the timeline + is read, so a change re-sorts entries already written. #} +{% extends "base.html" %} +{% from "_macros.html" import crumbs %} +{% block title %}Settings — Timeline — Steward{% endblock %} +{% from "settings/_tabs.html" import settings_tabs %} +{% block section_tabs %}{{ settings_tabs("timeline") }}{% endblock %} +{% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("Timeline", "")]) }}{% endblock %} +{% block content %} +{% if error %}{% endif %} + +
{{ csrf_field() }} +
+

Areas

+

What part of the infrastructure an entry is about.

+ + + + {% for key, label in cfg.areas %} + + + + + {% endfor %} + + + + + +
NameRemove, moving its entries to
+ +
+
+ +
+

What the timeline shows

+

A hidden type is still recorded; showing it again brings its entries back.

+ + + {% for kind, types in by_kind.items() %} + + + {% for t in types %} + {% set override = cfg.types.get(t.key, {}) %} + + + + + + {% endfor %} + + {% endfor %} +
TypeAreaShown
{{ kinds[kind] }}
{{ t.label }}
{{ t.about }}
 + +
+

“As recorded” sorts each entry under the area it was written with: an alert by what its rule watches, a change by the area its author picked.

+
+ +
+

Keeping it

+
+ + +
+
+ +
+
+{% endblock %} diff --git a/steward/templates/timeline/_entry.html b/steward/templates/timeline/_entry.html new file mode 100644 index 0000000..aa9f313 --- /dev/null +++ b/steward/templates/timeline/_entry.html @@ -0,0 +1,21 @@ +{# One timeline entry's parts, shared by the Timeline page and a host's card. + `item` is a timeline.service.view() row. #} + +{# What happened: its kind, the sentence, a note, and the full record while it is kept. #} +{% macro entry_what(item) -%} +{%- set e = item.entry -%} +
+ {%- if item.kind_label %}{{ item.kind_label }} · {% endif -%} + {{ e.summary }} +
+{%- if item.note %}
{{ item.note }}
{% endif %} +{%- if item.url %}
Details
+{%- elif item.gone %}
Details no longer kept
{% endif %} +{%- endmacro %} + +{# Where: the hosts it happened on, each linked to its timeline, and what it happened to. #} +{% macro entry_where(e) -%} +{%- for h in e.hosts %}{{ h }}{{ ", " if not loop.last }}{% endfor -%} +{%- if e.subject and e.subject not in e.hosts %}{% if e.hosts %}
{% endif %}{{ e.subject }}{% endif -%} +{%- if not e.hosts and not e.subject %}—{% endif -%} +{%- endmacro %} diff --git a/steward/templates/timeline/_host_card.html b/steward/templates/timeline/_host_card.html new file mode 100644 index 0000000..b2a347a --- /dev/null +++ b/steward/templates/timeline/_host_card.html @@ -0,0 +1,25 @@ +{# A host's newest timeline entries, loaded into its page: what happened to it + lately, with the whole of it a click away. #} +{% from "_macros.html" import empty_state %} +{% from "timeline/_entry.html" import entry_what %} +
+ + {% if items %} + + + {% for item in items %} + + + + + + {% endfor %} + +
{{ item.entry.at|when }}{{ entry_what(item) }}{{ item.entry.actor }}
+ {% else %} + {{ empty_state("Nothing has happened to " ~ host.name ~ " that the timeline records yet.") }} + {% endif %} +
diff --git a/steward/templates/timeline/_rows.html b/steward/templates/timeline/_rows.html new file mode 100644 index 0000000..7783a77 --- /dev/null +++ b/steward/templates/timeline/_rows.html @@ -0,0 +1,21 @@ +{# timeline/_rows.html — one page of the timeline's rows, then the sentinel + that loads the next page as the reader nears it (core.paging). A divider + row opens each recency group: Today, Yesterday, Last 7 days, Older. #} +{% from "_macros.html" import scroll_more, scroll_end %} +{% from "timeline/_entry.html" import entry_what, entry_where %} + {% for item in items %} + {% set e = item.entry %} + {% if item.opens_group %}{{ item.group }}{% endif %} + + {{ e.at|when }} + {{ entry_what(item) }} + {{ entry_where(e) }} + {{ e.actor }} + {{ item.area_label }} + + {% endfor %} +{% if after %} +{{ scroll_more("/timeline/?before=" ~ (after|urlencode) ~ "&" ~ (keep|urlencode) ~ "&group=" ~ (last_group|urlencode), cols=5) }} +{% else %} +{{ scroll_end(cols=5) }} +{% endif %} diff --git a/steward/templates/timeline/list.html b/steward/templates/timeline/list.html new file mode 100644 index 0000000..7360d54 --- /dev/null +++ b/steward/templates/timeline/list.html @@ -0,0 +1,54 @@ +{# timeline/list.html — everything that happened to the infrastructure, newest + first, one list: alerts, playbook runs, changes to Steward's settings. + Filters are by host, area (what was affected) and kind (how it came about); + Settings → Timeline decides each type's area and whether it is shown, and + "Include hidden" shows the types it hides. #} +{% extends "base.html" %} +{% from "_macros.html" import page_header, empty_state %} +{% block title %}Timeline — Steward{% endblock %} +{% block content %} +{% call page_header("Timeline", "What happened, newest first: alerts, playbook runs and changes, on every host.") %} + {% if session.user_role == 'admin' %}Areas and types{% endif %} +{% endcall %} + + + +{% if items %} +
+ + + + + + + + + + + + {% include "timeline/_rows.html" %} + +
When ({{ tz_label() }})What happenedWhereWhoArea
+
+{% else %} +
{{ empty_state("Nothing on the timeline matches these filters." if keep else "Nothing has been recorded on the timeline yet.") }}
+{% endif %} +{% endblock %} diff --git a/steward/timeline/__init__.py b/steward/timeline/__init__.py new file mode 100644 index 0000000..07746b1 --- /dev/null +++ b/steward/timeline/__init__.py @@ -0,0 +1,2 @@ +"""The timeline: one record of what happened to the infrastructure, from +every source, newest first (catalog, config, record, service).""" diff --git a/steward/timeline/catalog.py b/steward/timeline/catalog.py new file mode 100644 index 0000000..e312bec --- /dev/null +++ b/steward/timeline/catalog.py @@ -0,0 +1,137 @@ +"""What can appear on the timeline: each entry type, the kind of event it is, +and the area it sorts under until Settings → Timeline says otherwise. + +An area is what part of the infrastructure was affected (Service, Network, +Hardware …); a kind is how the event came about (a change someone made, an +automation run, an alert …). Kinds are fixed here. Areas, and each type's +area and visibility, are settings (steward.timeline.config), so they can be +changed without a release and the change re-sorts entries already written. + +A plugin adds its own types with register_type() from its setup(). +""" +from __future__ import annotations + +from dataclasses import dataclass + +from steward.core.settings import DEFAULTS + +# How an event came about. Fixed: the page's "Kind" filter and the MCP +# tool's `kind` argument take these keys. +KINDS: dict[str, str] = { + "change": "Change", # a person or an agent did it + "automation": "Automation", # a playbook or maintenance run + "alert": "Alert", + "anomaly": "Anomaly", + "ticket": "Ticket", + "steward": "Steward", # Steward's own settings and access +} + +# The starting areas, [key, label]: the timeline.areas setting's default. +# Settings → Timeline renames, adds and removes them. +DEFAULT_AREAS: list[list[str]] = DEFAULTS["timeline.areas"] +# Where an entry goes whose area is unknown: removed without a destination, +# or named by a plugin that is no longer loaded. +OTHER = ("other", "Other") + + +@dataclass(frozen=True) +class EntryType: + key: str + label: str # how Settings → Timeline names it + kind: str # a KINDS key + area: str # the area its entries are written with, unless the writer names one + shown: bool # whether the timeline shows it until Settings says otherwise + about: str # one line for Settings → Timeline + + +TYPES: dict[str, EntryType] = {} + + +def register_type(key: str, label: str, kind: str, area: str, shown: bool, about: str) -> None: + if kind not in KINDS: + raise ValueError(f"timeline type {key!r}: unknown kind {kind!r}") + TYPES[key] = EntryType(key, label, kind, area, shown, about) + + +for _t in ( + ("alert.fired", "Alert fired", "alert", "monitoring", True, + "An alert rule's condition held long enough to fire."), + ("alert.resolved", "Alert resolved", "alert", "monitoring", True, + "A firing alert cleared, or its rule was changed."), + ("alert.acknowledged", "Alert acknowledged", "alert", "monitoring", True, + "Someone took a firing alert."), + ("alert.silent", "Readings stopped", "alert", "monitoring", True, + "What an alert rule watches stopped reporting."), + ("alert.resumed", "Readings resumed", "alert", "monitoring", True, + "What an alert rule watches reported again after a silence."), + ("ansible.changed", "Playbook changed something", "automation", "host_os", True, + "A playbook run finished and changed at least one host."), + ("ansible.unchanged", "Playbook changed nothing", "automation", "host_os", False, + "A playbook run finished with nothing to change."), + ("ansible.failed", "Playbook failed", "automation", "host_os", True, + "A playbook run failed, was cancelled, or was stopped by a restart."), + ("ansible.check", "Playbook dry run", "automation", "host_os", False, + "A playbook run in check mode: it showed what would change."), + ("steward.config", "Steward settings changed", "steward", "monitoring", True, + "Monitors, alert rules, maintenance windows, inventory, plugins, settings."), + ("steward.access", "Users and access", "steward", "security", True, + "Users, roles, passwords and AI access tokens."), + ("steward.signin", "Sign-ins", "steward", "security", False, + "Someone signed in, or failed to."), + ("steward.backup", "Steward backups", "steward", "storage", True, + "Steward backed itself up, or was restored from a backup."), +): + register_type(*_t) + + +# Which area an alert's entry is written with, from what its rule watches. +# A source not listed here is Monitoring. +_SOURCE_AREAS = { + "icmp": "network", "dns": "network", "tcp": "service", "http": "service", + "docker": "service", "prometheus": "service", + "traefik": "network", "unifi": "network", + "snmp": "hardware", +} +# host_agent metrics by prefix, first match wins; the rest are Host OS. +_HOST_METRIC_AREAS = ( + ("updates_security", "security"), + ("temp_", "hardware"), ("drive", "hardware"), + ("disk_", "storage"), ("inodes_", "storage"), ("mounts_", "storage"), ("nfs_", "storage"), + ("net_", "network"), +) + + +def area_of_metric(source: str, metric: str) -> str: + """The area a reading of `metric` from `source` is about.""" + if source == "host_agent": + for prefix, area in _HOST_METRIC_AREAS: + if metric.startswith(prefix): + return area + return "host_os" + if source == "snmp" and any(w in metric for w in ("volume", "disk_used", "storage")): + return "storage" + return _SOURCE_AREAS.get(source, "monitoring") + + +# Audit actions and the type each is entered as. The first prefix that +# matches wins; None leaves the action off the timeline, because another +# writer records the event itself (a run, an acknowledgement) or it is about +# one person's own preferences. +_AUDIT_TYPES = ( + ("ansible.run.", None), ("ansible.host_run", None), + ("alert_rule.acknowledged", None), ("alert_rule.tested", None), + ("settings.notification.tested", None), + ("account.time_zone", None), ("account.email", None), + ("host_agent.maintenance_plan", "steward.config"), ("host_agent.maintenance", None), + ("docker.prune.", None), + ("auth.login", "steward.signin"), ("auth.setup", "steward.config"), + ("account.", "steward.access"), ("user.", "steward.access"), ("api_token.", "steward.access"), + ("backup.", "steward.backup"), +) + + +def audit_type(action: str) -> str | None: + for prefix, type_ in _AUDIT_TYPES: + if action.startswith(prefix): + return type_ + return "steward.config" diff --git a/steward/timeline/config.py b/steward/timeline/config.py new file mode 100644 index 0000000..57b1302 --- /dev/null +++ b/steward/timeline/config.py @@ -0,0 +1,150 @@ +"""Settings → Timeline, as the reader applies it. + +Three settings, read together: +- timeline.areas: [[key, label], …], the areas in the order the page lists them. +- timeline.types: {type: {"area": key, "shown": bool}}, each only where it + departs from the catalog. An "area" there sorts every entry of the type + under it; without one, an entry sorts under the area it was written with. +- timeline.area_moved: {removed key: key its entries now sort under}. + +Nothing here is written onto an entry, so a change re-sorts the whole +history, and a hidden type's entries come back when it is shown again. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +from sqlalchemy import and_, false, or_, true + +from . import catalog +from steward.models.timeline import TimelineEntry + + +@dataclass +class Config: + areas: list[list[str]] + types: dict[str, dict] = field(default_factory=dict) + moved: dict[str, str] = field(default_factory=dict) + + @property + def labels(self) -> dict[str, str]: + return {key: label for key, label in self.areas} + + def follow(self, area: str) -> str: + """Where an area's entries sort now: through any removals, to a live + area, or Other.""" + seen = set() + while area in self.moved and area not in seen: + seen.add(area) + area = self.moved[area] + return area if area in self.labels else catalog.OTHER[0] + + def type_area(self, type_: str) -> str | None: + """The area Settings sorts every entry of this type under, or None.""" + return (self.types.get(type_) or {}).get("area") or None + + def shown(self, type_: str) -> bool: + override = (self.types.get(type_) or {}).get("shown") + if override is not None: + return bool(override) + known = catalog.TYPES.get(type_) + return known.shown if known else True + + def area_of(self, entry) -> str: + return self.follow(self.type_area(entry.type) or entry.area) + + def area_label(self, key: str) -> str: + return self.labels.get(key) or catalog.OTHER[1] + + def hidden_types(self) -> set[str]: + keys = set(catalog.TYPES) | set(self.types) + return {k for k in keys if not self.shown(k)} + + def area_filter(self, area: str): + """A SQL condition for "entries sorting under `area`".""" + sources = [k for k in [*self.labels, *self.moved, catalog.OTHER[0]] + if self.follow(k) == area] + fixed = {t for t, o in self.types.items() if o.get("area")} + to_here = [t for t in fixed if self.follow(self.types[t]["area"]) == area] + written = TimelineEntry.area.in_(sources) if sources else false() + if area == catalog.OTHER[0]: + # An area nobody listed (a plugin's, gone) sorts under Other too. + written = or_(written, TimelineEntry.area.notin_([*self.labels, *self.moved])) + unfixed = TimelineEntry.type.notin_(fixed) if fixed else true() + return or_(TimelineEntry.type.in_(to_here) if to_here else false(), + and_(unfixed, written)) + + +def kind_types(kind: str) -> list[str]: + return [t.key for t in catalog.TYPES.values() if t.kind == kind] + + +async def load(db) -> Config: + from steward.core.settings import get_setting + areas = await get_setting(db, "timeline.areas") or catalog.DEFAULT_AREAS + return Config(areas=[list(a) for a in areas], + types=dict(await get_setting(db, "timeline.types") or {}), + moved=dict(await get_setting(db, "timeline.area_moved") or {})) + + +def slug(label: str) -> str: + return re.sub(r"[^a-z0-9]+", "_", label.strip().lower()).strip("_")[:32] + + +def edited(cfg: Config, form) -> tuple[Config, str | None]: + """The settings Settings → Timeline's form asks for, or the reason it + can't have them. The form carries, per area, `area_label_` and + `area_move_` ("" to keep it, else the key its entries go to); + `new_area` for one more; and per type `type_area_` ("" for as + written) and `type_shown_` (present when ticked).""" + areas, moved = [], dict(cfg.moved) + for key, label in cfg.areas: + dest = (form.get(f"area_move_{key}") or "").strip() + if dest: + if dest == key or dest not in cfg.labels: + return cfg, f"Pick another area for {label}'s entries to go to." + moved[key] = dest + continue + areas.append([key, (form.get(f"area_label_{key}") or "").strip() or label]) + new = (form.get("new_area") or "").strip() + if new: + key = slug(new) + if not key or key == catalog.OTHER[0] or key in {k for k, _ in cfg.areas}: + return cfg, f"There is already an area like “{new}”; pick another name." + areas.append([key, new]) + moved.pop(key, None) + if not areas: + return cfg, "Keep at least one area." + live = {k for k, _ in areas} + # An area listed again is no longer moved; one moved into an area that + # was itself moved goes on to wherever that went (Config.follow). + moved = {k: d for k, d in moved.items() if k not in live} + + after = Config(areas=areas, moved=moved) + types = {} + for key, t in catalog.TYPES.items(): + override = {} + area = (form.get(f"type_area_{key}") or "").strip() + if area: + # A type sorted under an area removed in this same save follows its entries. + area = after.follow(area) + if area not in live: + return cfg, f"“{t.label}” names an area that isn't listed." + override["area"] = area + shown = f"type_shown_{key}" in form + if shown != t.shown: + override["shown"] = shown + if override: + types[key] = override + # Types of a plugin that isn't loaded keep what they had. + types.update({k: v for k, v in cfg.types.items() if k not in catalog.TYPES}) + after.types = types + return after, None + + +async def save(db, cfg: Config) -> None: + from steward.core.settings import set_setting + await set_setting(db, "timeline.areas", cfg.areas) + await set_setting(db, "timeline.types", cfg.types) + await set_setting(db, "timeline.area_moved", cfg.moved) diff --git a/steward/timeline/record.py b/steward/timeline/record.py new file mode 100644 index 0000000..4a9eaae --- /dev/null +++ b/steward/timeline/record.py @@ -0,0 +1,233 @@ +"""Writing the timeline. Every source writes through record(), in the same +transaction as its own record, as the thing happens: the entry stands on its +own, so it is still there after the source's own row ages out. + +from_alert, from_run and from_audit build the entry for one alert event, +finished Ansible run and audit row. The live writers and the backfill use the +same three, so an entry reads the same however it got there. +""" +from __future__ import annotations + +import logging +from datetime import datetime + +from sqlalchemy import select + +from . import catalog +from steward.models.timeline import TimelineEntry + +logger = logging.getLogger(__name__) + + +def record(session, *, type: str, at: datetime, summary: str, area: str | None = None, + hosts=(), subject: str | None = None, detail: dict | None = None, + actor: str = "system", until: datetime | None = None, + ref: tuple[str, str] | None = None) -> TimelineEntry: + """Add one entry to `session`. `area` defaults to its type's; `ref` is + (kind, id) of the full record it points to, which is at most one entry.""" + known = catalog.TYPES.get(type) + entry = TimelineEntry( + at=at, until=until, type=type, + area=area or (known.area if known else catalog.OTHER[0]), + hosts=sorted({h for h in hosts if h}), subject=subject[:255] if subject else None, summary=summary, + detail=detail or None, actor=(actor or "system")[:64], + ref_type=ref[0] if ref else None, ref_id=str(ref[1]) if ref else None) + session.add(entry) + return entry + + +async def username(session, user_id: str | None) -> str | None: + if not user_id: + return None + from steward.models.users import User + user = await session.get(User, user_id) + return user.username if user else None + + +# ── Alerts ────────────────────────────────────────────────────────────────── + +_MONITOR_SOURCES = ("icmp", "tcp", "dns", "http") + + +async def hosts_of_rule(session, source: str, resource: str) -> list[str]: + """The host an alert rule's resource is on, by name, when it has one: an + agent series ("" or ":"), a container ("/") + or a monitor tied to a host.""" + from steward.core.alerts import host_of_resource + if source == "host_agent": + return [host_of_resource(source, resource) or resource.split(":", 1)[0]] + if source == "docker": + return [resource.split("/", 1)[0]] + if source in _MONITOR_SOURCES: + from steward.models.hosts import Host + from steward.models.monitors import Monitor + name = (await session.execute( + select(Host.name).join(Monitor, Monitor.host_id == Host.id) + .where(Monitor.type == source, Monitor.name == resource).limit(1))).scalar() + return [name] if name else [] + return [] + + +def _alert_type(from_state: str, to_state: str) -> str | None: + if to_state == "firing": + return "alert.fired" + if to_state == "resolved": + return "alert.resolved" + if to_state == "acknowledged": + return "alert.acknowledged" + if to_state == "no_data": + return "alert.silent" + if from_state == "no_data": + return "alert.resumed" + return None + + +def alert_summary(type_: str, rule, value: float | None) -> str: + from steward.alerts.service import _BINARY + from steward.core.formatting import metric_label, metric_value + metric, res = rule.metric_name, rule.resource_name + said = metric_label(metric) + # The reading, and on a firing the condition it broke. + if metric in _BINARY: + reading = breach = f"{res} is {metric_value(metric, value)}" + else: + op = getattr(rule.operator, "value", rule.operator) + reading = f"{said} on {res} is {metric_value(metric, value)}" + breach = f"{reading} ({op} {metric_value(metric, rule.threshold)})" + return { + "alert.fired": f"{rule.name} fired: {breach}" if value is not None else f"{rule.name} fired", + "alert.resolved": f"{rule.name} resolved" + (f": {reading}" if value is not None else ""), + "alert.acknowledged": f"{rule.name} acknowledged", + "alert.silent": f"{rule.name}: no {said} reading from {res}", + "alert.resumed": f"{rule.name}: {said} readings from {res} again", + }[type_] + + +async def from_alert(session, rule, event, actor: str | None = None, + resumed: bool = False) -> TimelineEntry | None: + """The entry for one alert event (an AlertEvent row), or None for a + transition the timeline doesn't show (pending and back). `resumed` is a + series back from silence with the alert's state unchanged, which can be + an already-firing alert still firing.""" + type_ = "alert.resumed" if resumed else _alert_type(event.from_state, event.to_state) + if type_ is None: + return None + return record( + session, type=type_, at=event.transitioned_at, + summary=alert_summary(type_, rule, event.metric_value), + area=catalog.area_of_metric(rule.source_module, rule.metric_name), + hosts=await hosts_of_rule(session, rule.source_module, rule.resource_name), + subject=rule.resource_name, + detail={"rule_id": rule.id, "rule": rule.name, "source": rule.source_module, + "metric": rule.metric_name, "value": event.metric_value, "note": event.note, + "url": f"/alerts/history?rule={rule.id}"}, + actor=actor or await username(session, getattr(event, "actor_id", None)) or "system", + ref=("alert_event", event.id)) + + +# ── Ansible runs ──────────────────────────────────────────────────────────── + +def _run_type(run) -> str: + status = getattr(run.status, "value", run.status) + if status != "success": + return "ansible.failed" + if (run.params or {}).get("check"): + return "ansible.check" + hosts = ((run.results or {}).get("hosts") or {}).values() + return "ansible.changed" if any((h or {}).get("changed") for h in hosts) else "ansible.unchanged" + + +async def hosts_of_run(session, names) -> list[str]: + """The Steward hosts behind a run's inventory names: a target linked to a + host is that host; a name that is a host's own is that host.""" + from steward.models.ansible_inventory import AnsibleTarget + from steward.models.hosts import Host + names = set(names) + if not names: + return [] + linked = dict((await session.execute( + select(AnsibleTarget.name, Host.name).join(Host, AnsibleTarget.host_id == Host.id) + .where(AnsibleTarget.name.in_(names)))).all()) + direct = set((await session.execute( + select(Host.name).where(Host.name.in_(names - set(linked))))).scalars()) + return sorted(set(linked.values()) | direct) + + +async def from_run(session, run) -> TimelineEntry: + """The entry for one finished Ansible run.""" + from steward.ansible.inventory_gen import run_title, scope_labels + from steward.ansible.origins import run_origins + type_ = _run_type(run) + per_host = (run.results or {}).get("hosts") or {} + scope = (await scope_labels(session, [run.inventory_scope])).get(run.inventory_scope) + title = run_title(run.playbook_path, scope) + status = getattr(run.status, "value", run.status) + changed = sum(1 for h in per_host.values() if (h or {}).get("changed")) + failed = sum(1 for h in per_host.values() if (h or {}).get("failed") or (h or {}).get("unreachable")) + if type_ == "ansible.failed": + said = {"cancelled": "cancelled", "interrupted": "stopped by a restart"}.get(status, "failed") + tail = f"{said} on {failed} host{'s' if failed != 1 else ''}" if failed else said + elif type_ == "ansible.check": + tail = f"dry run, would change {changed} host{'s' if changed != 1 else ''}" if changed \ + else "dry run, nothing to change" + elif type_ == "ansible.changed": + tail = f"changed {changed} host{'s' if changed != 1 else ''}" + else: + tail = "nothing to change" + origin = (await run_origins(session, [run])).get(run.id) or {} + return record( + session, type=type_, at=run.started_at, until=run.finished_at, + summary=f"{title}: {tail}", hosts=await hosts_of_run(session, per_host), + subject=run.playbook_path, + detail={"status": status, "scope": scope, "origin": origin.get("text"), + "failures": ((run.results or {}).get("failures") or [])[:3], + "url": f"/ansible/runs/{run.id}"}, + actor=origin.get("text") or "Steward", ref=("ansible_run", run.id)) + + +async def run_finished(session, run_id: str) -> None: + """Write the entry for a run that has just recorded its final status, in + a savepoint of the caller's transaction. Never raises: the run's own + record matters more than its entry.""" + from steward.models.ansible import AnsibleRun + try: + async with session.begin_nested(): + run = await session.get(AnsibleRun, run_id) + if run is not None: + await from_run(session, run) + except Exception: + logger.exception("Ansible run %s: its timeline entry could not be written", run_id) + + +# ── The audit log ─────────────────────────────────────────────────────────── + +async def from_audit(session, event, detail: dict | None) -> TimelineEntry | None: + """The entry for one audit row (an AuditEvent), or None for an action the + timeline leaves to another writer (catalog.audit_type).""" + from steward.audit.service import action_text, entity_labels + type_ = catalog.audit_type(event.action) + if type_ is None: + return None + detail = detail or {} + entity = None + if event.entity_type and event.entity_id: + labels = await entity_labels(session, [{"event": event, "detail": detail}]) + entity = labels.get((event.entity_type, str(event.entity_id))) + what = action_text(event.action) + if entity: + what = f"{what} {entity['label']}" + hosts = [] + if event.entity_type == "host" and entity: + hosts = [entity["label"]] + elif event.entity_type == "monitor": + from steward.models.hosts import Host + from steward.models.monitors import Monitor + name = (await session.execute( + select(Host.name).join(Monitor, Monitor.host_id == Host.id) + .where(Monitor.id == str(event.entity_id)))).scalar() + hosts = [name] if name else [] + return record( + session, type=type_, at=event.timestamp, summary=what[:1].upper() + what[1:], + hosts=hosts, subject=entity["label"] if entity else None, + detail={"action": event.action, "url": entity.get("url") if entity else None}, + actor=event.username, ref=("audit_event", event.id)) diff --git a/steward/timeline/routes.py b/steward/timeline/routes.py new file mode 100644 index 0000000..376e077 --- /dev/null +++ b/steward/timeline/routes.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +from quart import Blueprint, current_app, redirect, render_template, request, session, url_for +from sqlalchemy import select + +from steward.audit.service import mark_day_groups +from steward.auth.middleware import require_role +from steward.core import formatting as fmt +from steward.core.audit import audit_request +from steward.core.paging import keyset, next_page +from steward.models.hosts import Host +from steward.models.timeline import TimelineEntry +from steward.models.users import UserRole +from steward.timeline import catalog, config as tl_config, service as tl + +timeline_bp = Blueprint("timeline", __name__) + + +def _filters() -> dict: + a = request.args + return {"host": a.get("host", "").strip(), "area": a.get("area", "").strip(), + "kind": a.get("kind", "").strip(), "hidden": a.get("hidden") == "1"} + + +@timeline_bp.get("/timeline/") +@require_role(UserRole.viewer) +async def timeline(): + """Everything that happened, newest first, a page at a time as the reader + scrolls; ?before= is the next page's rows alone (core.paging).""" + f = _filters() + async with current_app.db_sessionmaker() as db: + cfg = await tl_config.load(db) + stmt = tl.query(cfg, host=f["host"], area=f["area"], kind=f["kind"], + include_hidden=f["hidden"]) + stmt = keyset(stmt, TimelineEntry.at, TimelineEntry.id, request.args.get("before")) + rows, after = next_page((await db.execute(stmt)).scalars().all(), lambda e: (e.at, e.id)) + items = await tl.views(db, cfg, rows) + hosts = [] if request.args.get("before") else \ + list((await db.execute(select(Host.name).order_by(Host.name))).scalars()) + last_group = mark_day_groups(items, fmt.zone(session.get("tz")), request.args.get("group", ""), + at=lambda item: item["entry"].at) + keep = {k: v for k, v in (("host", f["host"]), ("area", f["area"]), ("kind", f["kind"]), + ("hidden", "1" if f["hidden"] else "")) if v} + ctx = {"items": items, "after": after, "last_group": last_group, "filters": f, "keep": keep, + "areas": cfg.areas, "kinds": catalog.KINDS, + "hosts": hosts if not f["host"] or f["host"] in hosts else [f["host"], *hosts]} + if request.args.get("before"): + return await render_template("timeline/_rows.html", **ctx) + return await render_template("timeline/list.html", **ctx) + + +@timeline_bp.get("/timeline/host/") +@require_role(UserRole.viewer) +async def host_card(host_id: str): + """A host's newest entries, for its page (loaded into it).""" + async with current_app.db_sessionmaker() as db: + host = await db.get(Host, host_id) + if host is None: + return "", 404 + cfg = await tl_config.load(db) + items = await tl.recent_for_host(db, cfg, host.name) + return await render_template("timeline/_host_card.html", host=host, items=items) + + +@timeline_bp.get("/settings/timeline/") +@require_role(UserRole.admin) +async def settings_page(): + async with current_app.db_sessionmaker() as db: + cfg = await tl_config.load(db) + from steward.core.settings import get_setting + retention = int(await get_setting(db, "timeline.retention_days") or 0) + return await render_template("settings/timeline.html", **_settings_ctx(cfg, retention)) + + +def _settings_ctx(cfg, retention: int, error: str | None = None) -> dict: + by_kind = {k: [t for t in catalog.TYPES.values() if t.kind == k] for k in catalog.KINDS} + return {"cfg": cfg, "retention": retention, "error": error, "kinds": catalog.KINDS, + "by_kind": {k: v for k, v in by_kind.items() if v}} + + +@timeline_bp.post("/settings/timeline/") +@require_role(UserRole.admin) +async def save_settings(): + from steward.core.settings import set_setting + form = await request.form + try: + retention = max(0, int(form.get("retention_days") or 0)) + except ValueError: + retention = None + async with current_app.db_sessionmaker() as db: + async with db.begin(): + cfg = await tl_config.load(db) + new, error = tl_config.edited(cfg, form) + if error is None and retention is None: + error = "Keep for: a whole number of days, or 0 to keep everything." + if error is not None: + kept = retention if retention is not None else 0 + return await render_template("settings/timeline.html", + **_settings_ctx(cfg, kept, error)), 400 + await tl_config.save(db, new) + await set_setting(db, "timeline.retention_days", retention) + await audit_request("settings.saved", detail={"section": "timeline"}) + return redirect(url_for("timeline.settings_page")) diff --git a/steward/timeline/service.py b/steward/timeline/service.py new file mode 100644 index 0000000..da3961e --- /dev/null +++ b/steward/timeline/service.py @@ -0,0 +1,196 @@ +"""Reading the timeline, and keeping it: the filtered list the page, the host +card and get_timeline share, the first fill from records that predate it, +its retention, and a host rename carried onto its entries.""" +from __future__ import annotations + +import logging +from datetime import datetime, timedelta, timezone + +from sqlalchemy import delete, func, select, text, update +from sqlalchemy.dialects.postgresql import insert + +from . import catalog +from .config import Config, kind_types +from steward.models.timeline import TimelineEntry + +logger = logging.getLogger(__name__) + +# How far back the first fill reaches: the default retention of the records it reads. +BACKFILL_DAYS = 90 + + +def query(cfg: Config, *, host: str = "", area: str = "", kind: str = "", type_: str = "", + since: datetime | None = None, include_hidden: bool = False): + """The entries matching the filters, unordered (core.paging orders them). + Hidden types are left out unless `include_hidden`, or a `type_` asks for one.""" + stmt = select(TimelineEntry) + if host: + stmt = stmt.where(TimelineEntry.hosts.contains([host])) + if area: + stmt = stmt.where(cfg.area_filter(area)) + if kind: + stmt = stmt.where(TimelineEntry.type.in_(kind_types(kind))) + if type_: + stmt = stmt.where(TimelineEntry.type == type_) + elif not include_hidden: + hidden = cfg.hidden_types() + if hidden: + stmt = stmt.where(TimelineEntry.type.notin_(hidden)) + if since is not None: + stmt = stmt.where(TimelineEntry.at >= since) + return stmt + + +# The full records an entry can point to, and whether one is still kept. +def _ref_models() -> dict: + from steward.models.alerts import AlertEvent + from steward.models.ansible import AnsibleRun + from steward.models.audit import AuditEvent + return {"alert_event": AlertEvent, "ansible_run": AnsibleRun, "audit_event": AuditEvent} + + +async def kept_refs(db, entries) -> set[tuple[str, str]]: + """The (ref_type, ref_id) of `entries` whose full record still exists.""" + wanted: dict[str, set[str]] = {} + for e in entries: + if e.ref_type and e.ref_id: + wanted.setdefault(e.ref_type, set()).add(e.ref_id) + kept = set() + models = _ref_models() + for ref_type, ids in wanted.items(): + model = models.get(ref_type) + if model is None: # a kind only a plugin knows: assume it is kept + kept |= {(ref_type, i) for i in ids} + continue + found = (await db.execute(select(model.id).where(model.id.in_(ids)))).scalars() + kept |= {(ref_type, str(i)) for i in found} + return kept + + +def view(entry: TimelineEntry, cfg: Config, kept: set) -> dict: + """One entry as the page and the MCP tool read it.""" + known = catalog.TYPES.get(entry.type) + area = cfg.area_of(entry) + detail = entry.detail or {} + url = detail.get("url") + gone = bool(entry.ref_type) and (entry.ref_type, entry.ref_id) not in kept + return { + "entry": entry, + "type_label": known.label if known else entry.type, + "kind": known.kind if known else None, + "kind_label": catalog.KINDS.get(known.kind) if known else None, + "area": area, + "area_label": cfg.area_label(area), + "note": detail.get("note"), + "url": None if gone else url, + "gone": gone, + } + + +async def views(db, cfg: Config, entries) -> list[dict]: + kept = await kept_refs(db, entries) + return [view(e, cfg, kept) for e in entries] + + +async def recent_for_host(db, cfg: Config, host: str, limit: int = 8) -> list[dict]: + """A host's newest shown entries, for its page.""" + rows = (await db.execute( + query(cfg, host=host).order_by(TimelineEntry.at.desc(), TimelineEntry.id.desc()) + .limit(limit))).scalars().all() + return await views(db, cfg, rows) + + +# ── Keeping it ────────────────────────────────────────────────────────────── + +async def prune(session, now: datetime) -> tuple[int, int]: + """Delete entries past timeline.retention_days; (rows, days), with 0 days + keeping everything. Call inside a transaction.""" + from steward.core.settings import get_setting + days = int(await get_setting(session, "timeline.retention_days") or 0) + if days <= 0: + return 0, 0 + result = await session.execute( + delete(TimelineEntry).where(TimelineEntry.at < now - timedelta(days=days))) + return result.rowcount or 0, days + + +async def carry_host_rename(session, old: str, new: str) -> int: + """Point a renamed host's entries at its new name, so its page still + shows its history.""" + if not old or old == new: + return 0 + result = await session.execute( + update(TimelineEntry).where(TimelineEntry.hosts.contains([old])) + .values(hosts=func.array_replace(TimelineEntry.hosts, old, new)) + .execution_options(synchronize_session=False)) + return result.rowcount or 0 + + +async def backfill(app, now: datetime | None = None) -> int: + """Fill the timeline once from the alert events, Ansible runs and audit + rows that predate it, BACKFILL_DAYS back, through the same builders the + live writers use. An entry for a record already covered is skipped, so + it is safe to run again. Records the run in timeline.backfilled_at.""" + from steward.core.settings import get_setting, set_setting + from steward.models.alerts import AlertEvent, AlertRule + from steward.models.ansible import AnsibleRun + from steward.models.audit import AuditEvent + from .record import from_alert, from_audit, from_run + + now = now or datetime.now(timezone.utc) + async with app.db_sessionmaker() as session: + async with session.begin(): + if await get_setting(session, "timeline.backfilled_at"): + return 0 + since = now - timedelta(days=BACKFILL_DAYS) + built: list[TimelineEntry] = [] + async with session.begin(): + # The builders add each entry to the session; no flush may write + # them before the insert below, which skips any already covered. + session.sync_session.autoflush = False + + async def build(what: str, coro) -> None: + # One record that can't be read is skipped, not the whole fill. + try: + built.append(await coro) + except Exception: + logger.exception("Timeline fill: %s skipped", what) + + for event, rule in (await session.execute( + select(AlertEvent, AlertRule).join(AlertRule, AlertEvent.rule_id == AlertRule.id) + .where(AlertEvent.transitioned_at >= since))).all(): + await build(f"alert event {event.id}", from_alert(session, rule, event)) + for run in (await session.execute( + select(AnsibleRun).where(AnsibleRun.started_at >= since, + AnsibleRun.finished_at.is_not(None)))).scalars().all(): + await build(f"Ansible run {run.id}", from_run(session, run)) + for event in (await session.execute( + select(AuditEvent).where(AuditEvent.timestamp >= since))).scalars().all(): + await build(f"audit row {event.id}", from_audit(session, event, _audit_detail(event))) + rows = [e for e in built if e is not None] + for entry in rows: + session.expunge(entry) + for chunk in range(0, len(rows), 500): + await session.execute( + insert(TimelineEntry).values([_row(e) for e in rows[chunk:chunk + 500]]) + .on_conflict_do_nothing(index_elements=["ref_type", "ref_id"], + index_where=text("ref_id IS NOT NULL"))) + await set_setting(session, "timeline.backfilled_at", now.isoformat()) + logger.info("Timeline: filled %d entries from the last %d days of records", len(rows), BACKFILL_DAYS) + return len(rows) + + +def _audit_detail(event) -> dict: + import json + try: + return json.loads(event.detail_json) if event.detail_json else {} + except (TypeError, ValueError): + return {} + + +def _row(entry: TimelineEntry) -> dict: + from steward.core.ids import time_ordered_id + return {"id": entry.id or time_ordered_id(), "at": entry.at, "until": entry.until, + "type": entry.type, "area": entry.area, "hosts": entry.hosts or [], + "subject": entry.subject, "summary": entry.summary, "detail": entry.detail, + "actor": entry.actor or "system", "ref_type": entry.ref_type, "ref_id": entry.ref_id} diff --git a/tests/core/test_alert_no_data.py b/tests/core/test_alert_no_data.py index 9af8b4a..067b880 100644 --- a/tests/core/test_alert_no_data.py +++ b/tests/core/test_alert_no_data.py @@ -113,7 +113,8 @@ def test_a_sample_after_silence_clears_it_and_says_so(): assert note == "data resumed after 15m silent" -_RULE = NS(id="r", operator=AlertOperator.gt, threshold=90.0, +_RULE = NS(id="r", name="Slow database", source_module="steward", resource_name="steward", + metric_name="db_roundtrip_ms", operator=AlertOperator.gt, threshold=90.0, consecutive_failures_required=1, consecutive_ok_required=1) @@ -132,8 +133,12 @@ def test_a_resumed_series_gets_one_event_for_what_its_first_sample_did( session, st = _Session(), _state(state=before) said = "data resumed after 15m silent" out = asyncio.run(_evaluate_rule(session, _RULE, st, value, NOW, resumed=said)) - [event] = session.added + event, entry = session.added # the alert's event, then its timeline entry assert (event.from_state, event.to_state) == want_event + assert (entry.ref_type, entry.ref_id, entry.at) == ("alert_event", event.id, NOW) + # A firing alert still firing when its series returns is a resume, not a new firing. + assert entry.type == {"firing": "alert.fired", "resolved": "alert.resolved", + "data_resumed": "alert.resumed"}[want_notice] assert event.note == said and event.transitioned_at == NOW assert [(n, e) for n, e, _ in out] == [(want_notice, event.id)] # Without a resume, a sample that changes nothing records nothing. diff --git a/tests/core/test_timeline.py b/tests/core/test_timeline.py new file mode 100644 index 0000000..fb0f953 --- /dev/null +++ b/tests/core/test_timeline.py @@ -0,0 +1,270 @@ +"""The timeline: what each source becomes, how Settings → Timeline sorts and +hides it when it is read, and the pages that show it.""" +from datetime import datetime, timezone +from types import SimpleNamespace as NS + +import pytest +from quart import render_template +from werkzeug.datastructures import MultiDict + +from steward.models.alerts import AlertOperator +from steward.timeline import catalog, record, service +from steward.timeline.config import Config, edited + +NOW = datetime(2026, 10, 24, 12, 0, tzinfo=timezone.utc) +AREAS = [list(a) for a in catalog.DEFAULT_AREAS] + + +def _cfg(**kw) -> Config: + return Config(areas=[list(a) for a in kw.pop("areas", AREAS)], **kw) + + +def _entry(type_="alert.fired", area="hardware", **kw): + base = dict(id="e1", at=NOW, until=None, type=type_, area=area, hosts=["web-1"], + subject="web-1:sda", summary="Disk hot fired", detail={}, actor="system", + ref_type=None, ref_id=None) + return NS(**{**base, **kw}) + + +# ── What a source becomes ─────────────────────────────────────────────────── + +@pytest.mark.parametrize("action,type_", [ + ("monitor.disabled", "steward.config"), + ("settings.saved", "steward.config"), + ("host_agent.maintenance_plan_created", "steward.config"), + ("user.role_changed", "steward.access"), + ("api_token.create", "steward.access"), + ("account.password_changed", "steward.access"), + ("auth.login", "steward.signin"), + ("auth.login_failed", "steward.signin"), + ("backup.made", "steward.backup"), + # Another writer records the event itself, or it is one person's preference. + ("ansible.run.started", None), + ("ansible.host_run", None), + ("alert_rule.acknowledged", None), + ("host_agent.maintenance", None), + ("docker.prune.started", None), + ("account.time_zone", None), +]) +def test_each_audit_action_is_entered_as_its_type_or_left_out(action, type_): + assert catalog.audit_type(action) == type_ + + +@pytest.mark.parametrize("source,metric,area", [ + ("host_agent", "temp_c_max", "hardware"), + ("host_agent", "drive_failing", "hardware"), + ("host_agent", "disk_used_pct", "storage"), + ("host_agent", "mounts_stalled", "storage"), + ("host_agent", "nfs_rtt_ms", "storage"), + ("host_agent", "net_rx_bps", "network"), + ("host_agent", "updates_security", "security"), + ("host_agent", "cpu_pct", "host_os"), + ("host_agent", "updates_pending", "host_os"), + ("http", "is_up", "service"), + ("icmp", "is_up", "network"), + ("docker", "mem_used_pct", "service"), + ("traefik", "error_rate_5xx_pct", "network"), + ("steward", "db_roundtrip_ms", "monitoring"), + ("snmp", "volume_used_pct", "storage"), + ("snmp", "ups_battery_pct", "hardware"), +]) +def test_an_alert_is_written_under_the_area_its_rule_watches(source, metric, area): + assert catalog.area_of_metric(source, metric) == area + + +@pytest.mark.parametrize("from_state,to_state,type_", [ + ("pending", "firing", "alert.fired"), + ("no_data", "firing", "alert.fired"), + ("firing", "resolved", "alert.resolved"), + ("firing", "acknowledged", "alert.acknowledged"), + ("inactive", "no_data", "alert.silent"), + ("no_data", "inactive", "alert.resumed"), + ("inactive", "pending", None), +]) +def test_an_alert_event_becomes_one_entry_type(from_state, to_state, type_): + assert record._alert_type(from_state, to_state) == type_ + + +def test_an_alert_entry_reads_as_a_sentence(): + rule = NS(name="High CPU", metric_name="cpu_pct", resource_name="web-1", + operator=AlertOperator.gt, threshold=90.0) + assert record.alert_summary("alert.fired", rule, 94.0) == "High CPU fired: CPU on web-1 is 94% (> 90%)" + assert record.alert_summary("alert.resolved", rule, 40.0) == "High CPU resolved: CPU on web-1 is 40%" + assert record.alert_summary("alert.resolved", rule, None) == "High CPU resolved" + assert record.alert_summary("alert.silent", rule, None) == "High CPU: no CPU reading from web-1" + up = NS(name="API up", metric_name="is_up", resource_name="api", operator=AlertOperator.lt, threshold=1.0) + assert record.alert_summary("alert.fired", up, 0.0) == "API up fired: api is down" + + +def test_an_entry_takes_its_type_s_area_unless_the_writer_names_one(): + added = [] + session = NS(add=added.append) + e = record.record(session, type="ansible.changed", at=NOW, summary="x", hosts=["b", "a", "a", ""]) + assert e.area == "host_os" and e.hosts == ["a", "b"] and added == [e] + assert record.record(session, type="ansible.changed", at=NOW, summary="x", area="service").area == "service" + assert record.record(session, type="plugin.gone", at=NOW, summary="x").area == "other" + + +# ── Settings → Timeline, applied when read ────────────────────────────────── + +def test_an_entry_sorts_under_its_own_area_until_its_type_is_given_one(): + e = _entry(area="hardware") + assert _cfg().area_of(e) == "hardware" + assert _cfg(types={"alert.fired": {"area": "monitoring"}}).area_of(e) == "monitoring" + + +def test_a_removed_area_s_entries_follow_where_it_was_moved(): + cfg = _cfg(areas=[a for a in AREAS if a[0] not in ("hardware", "storage")], + moved={"hardware": "storage", "storage": "host_os"}) + assert cfg.area_of(_entry(area="hardware")) == "host_os" + # An area nobody lists any more, and a loop, end in Other rather than hang. + assert cfg.area_of(_entry(area="from_a_plugin")) == "other" + assert _cfg(areas=AREAS[:1], moved={"a": "b", "b": "a"}).follow("a") == "other" + assert cfg.area_label("other") == "Other" + + +def test_a_type_is_shown_as_the_catalog_says_until_settings_say_otherwise(): + cfg = _cfg() + assert cfg.shown("alert.fired") and not cfg.shown("ansible.check") + assert {"ansible.check", "ansible.unchanged", "steward.signin"} <= cfg.hidden_types() + cfg = _cfg(types={"ansible.check": {"shown": True}, "alert.fired": {"shown": False}}) + assert cfg.shown("ansible.check") and not cfg.shown("alert.fired") + assert "alert.fired" in cfg.hidden_types() and "ansible.check" not in cfg.hidden_types() + + +def _form(cfg: Config, **extra) -> MultiDict: + """The form as the page posts it unchanged: every type at its current + visibility, every area kept, then `extra`.""" + data = {f"type_shown_{k}": "on" for k in catalog.TYPES if cfg.shown(k)} + data.update({f"area_label_{k}": label for k, label in cfg.areas}) + data.update(extra) + return MultiDict({k: v for k, v in data.items() if v is not None}) + + +def test_saving_the_page_unchanged_stores_no_overrides(): + cfg = _cfg() + new, error = edited(cfg, _form(cfg)) + assert error is None and new.types == {} and new.moved == {} and new.areas == AREAS + + +def test_an_area_is_renamed_added_and_removed_into_another(): + cfg = _cfg(types={"alert.fired": {"area": "hardware"}}) + new, error = edited(cfg, _form(cfg, area_label_network="Networking", new_area="Power & UPS", + area_move_hardware="storage", + **{"type_area_alert.fired": "hardware"})) + assert error is None + assert dict(new.areas)["network"] == "Networking" + assert new.areas[-1] == ["power_ups", "Power & UPS"] + assert "hardware" not in dict(new.areas) and new.moved == {"hardware": "storage"} + # A type sorted under the removed area follows its entries. + assert new.types["alert.fired"] == {"area": "storage"} + + +def test_a_type_is_hidden_or_given_an_area(): + cfg = _cfg() + form = _form(cfg, **{"type_area_ansible.changed": "service"}) + del form["type_shown_alert.resumed"] + new, error = edited(cfg, form) + assert error is None + assert new.types == {"ansible.changed": {"area": "service"}, "alert.resumed": {"shown": False}} + + +@pytest.mark.parametrize("extra,said", [ + ({"area_move_network": "network"}, "Pick another area"), + ({"area_move_network": "nowhere"}, "Pick another area"), + ({"new_area": "Storage"}, "already an area"), + ({"new_area": "Other"}, "already an area"), + ({"type_area_alert.fired": "nowhere"}, "isn't listed"), +]) +def test_a_form_that_can_t_be_saved_says_why_and_changes_nothing(extra, said): + cfg = _cfg() + new, error = edited(cfg, _form(cfg, **extra)) + assert said in error and new is cfg + + +def test_the_last_area_can_t_be_removed(): + cfg = _cfg(areas=[["service", "Service"], ["network", "Network"]]) + form = _form(cfg, area_move_service="network", area_move_network="service") + assert edited(cfg, form)[1] == "Keep at least one area." + + +def test_a_plugin_s_type_keeps_its_settings_while_the_plugin_is_away(): + cfg = _cfg(types={"unifi.client_joined": {"shown": False}}) + new, _ = edited(cfg, _form(cfg)) + assert new.types == {"unifi.client_joined": {"shown": False}} + + +# ── Read back ─────────────────────────────────────────────────────────────── + +def test_an_entry_whose_record_aged_out_says_so_instead_of_linking(): + e = _entry(ref_type="ansible_run", ref_id="r1", detail={"url": "/ansible/runs/r1", "note": "n"}) + kept = service.view(e, _cfg(), {("ansible_run", "r1")}) + gone = service.view(e, _cfg(), set()) + assert kept["url"] == "/ansible/runs/r1" and not kept["gone"] and kept["note"] == "n" + assert gone["url"] is None and gone["gone"] + assert kept["kind_label"] == "Alert" and kept["area_label"] == "Hardware" + # An entry with no record behind it (a hand-written one) is never "gone". + assert not service.view(_entry(), _cfg(), set())["gone"] + + +# ── Pages ─────────────────────────────────────────────────────────────────── + +def _item(e, **kw): + v = service.view(e, _cfg(), {(e.ref_type, e.ref_id)}) + v.update(group="Today", opens_group=True, **kw) + return v + + +async def test_the_page_lists_what_happened_where_and_by_whom(app): + items = [_item(_entry(ref_type="alert_event", ref_id="a1", detail={"url": "/alerts/history?rule=r"})), + _item(_entry(type_="steward.config", area="monitoring", hosts=[], subject=None, + summary="Paused monitor api", actor="bryan via MCP"), opens_group=False)] + async with app.test_request_context("/timeline/"): + page = await render_template("timeline/list.html", items=items, after=None, last_group="Today", + filters={"host": "", "area": "", "kind": "", "hidden": False}, + keep={}, areas=AREAS, kinds=catalog.KINDS, hosts=["web-1"]) + assert "Disk hot fired" in page and "Paused monitor api" in page + assert 'web-1' in page + assert 'Details' in page + assert "bryan via MCP" in page and ">Hardware<" in page and "Alert · " in page + assert page.count('class="table-divider"') == 1 + assert "End of the list." in page + + +async def test_the_next_page_keeps_the_filters(app): + items = [_item(_entry())] + async with app.test_request_context("/timeline/"): + rows = await render_template("timeline/_rows.html", items=items, last_group="Today", + after="2026-10-24T12:00:00+00:00|e1", keep={"host": "web-1"}) + assert 'hx-get="/timeline/?before=2026-10-24T12%3A00%3A00%2B00%3A00%7Ce1&host=web-1&group=Today"' in rows + + +async def test_an_empty_timeline_says_so(app): + async with app.test_request_context("/timeline/"): + page = await render_template("timeline/list.html", items=[], after=None, last_group="", + filters={"host": "x", "area": "", "kind": "", "hidden": False}, + keep={"host": "x"}, areas=AREAS, kinds=catalog.KINDS, hosts=[]) + assert "Nothing on the timeline matches these filters." in page + + +async def test_a_host_s_card_shows_its_latest_and_links_the_rest(app): + host = NS(id="h1", name="web-1") + async with app.test_request_context("/timeline/host/h1"): + card = await render_template("timeline/_host_card.html", host=host, + items=[_item(_entry(ref_type="ansible_run", ref_id="gone"))]) + empty = await render_template("timeline/_host_card.html", host=host, items=[]) + assert "Disk hot fired" in card and 'href="/timeline/?host=web-1"' in card + assert "Nothing has happened to web-1" in empty + + +async def test_settings_lists_each_type_under_its_kind(app): + from steward.timeline.routes import _settings_ctx + cfg = _cfg(types={"alert.fired": {"area": "monitoring"}}) + async with app.test_request_context("/settings/timeline/"): + page = await render_template("settings/timeline.html", **_settings_ctx(cfg, 0)) + for t in catalog.TYPES.values(): + assert t.label in page + assert 'name="type_shown_alert.fired" aria-label="Show Alert fired" checked' in page + assert 'name="type_shown_ansible.check" aria-label="Show Playbook dry run">' in page + assert '' in page + assert 'aria-current="page">Timeline' in page diff --git a/tests/integration/test_timeline.py b/tests/integration/test_timeline.py new file mode 100644 index 0000000..54bde45 --- /dev/null +++ b/tests/integration/test_timeline.py @@ -0,0 +1,265 @@ +"""Integration: the timeline against real Postgres. Each source writes its +entry as the thing happens, the entry outlives its record, Settings re-sorts +and hides entries when they are read, the first fill skips what is covered, +and get_timeline reads it back. The database is shared with other tests, so +each test looks for its own uniquely named host. +Requires STEWARD_DATABASE_URL. +""" +from __future__ import annotations + +import asyncio +import os +import uuid +from datetime import datetime, timedelta, timezone + +import pytest + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +async def _entries(app, host: str): + from sqlalchemy import select + from steward.models.timeline import TimelineEntry + async with app.db_sessionmaker() as s: + return (await s.execute(select(TimelineEntry).where(TimelineEntry.hosts.contains([host])) + .order_by(TimelineEntry.at))).scalars().all() + + +def test_each_source_writes_an_entry_that_outlives_its_record(app): + from sqlalchemy import delete + from steward.alerts import service as alert_svc + from steward.core.alerts import record_metric, record_metrics + from steward.core.audit import log_audit + from steward.models.alerts import AlertEvent + from steward.models.ansible import AnsibleRun, AnsibleRunStatus + from steward.models.hosts import Host + from steward.models.metrics import PluginMetric + from steward.models.users import User, UserRole + from steward.timeline import config as tl_config, service as tl + from steward.timeline.record import run_finished + + async def _go(): + tag = uuid.uuid4().hex[:8] + name, renamed, run_id = f"tl-{tag}", f"tl-{tag}-new", str(uuid.uuid4()) + async with app.db_sessionmaker() as s: + async with s.begin(): + user = User(username=f"tl-{tag}", email=f"tl-{tag}@test", password_hash="x", + role=UserRole.admin) + host = Host(name=name, address="10.1.2.3") + s.add_all([user, host]) + await record_metrics(s, [PluginMetric(source_module="host_agent", resource_name=name, + metric_name="cpu_pct", value=1.0)]) + await s.flush() + fields = await alert_svc.rule_fields( + s, name=f"hot {tag}", source_module="host_agent", resource_name=name, + metric_name="cpu_pct", operator=">", threshold=90, + consecutive_failures_required=1, consecutive_ok_required=1) + rule = await alert_svc.add_rule(s, fields, created_by=user.id) + user_id, host_id, rule_id = user.id, host.id, rule.id + + # An alert firing. + async with app.db_sessionmaker() as s: + async with s.begin(): + await record_metric(s, "host_agent", name, "cpu_pct", 97.0) + # A change to Steward's settings about the host, and one the timeline leaves to the run. + await log_audit(app, user_id, f"tl-{tag}", "host.created", "host", host_id, {"name": name}) + await log_audit(app, user_id, f"tl-{tag}", "ansible.run.started", "ansible_run", run_id, {}) + # A playbook run that changed the host. + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add(AnsibleRun(id=run_id, playbook_path="update.yml", source_name="local", + inventory_scope=f"steward:target:{uuid.uuid4()}", + status=AnsibleRunStatus.success, triggered_by=user_id, + finished_at=datetime.now(timezone.utc), + results={"hosts": {name: {"ok": 3, "changed": 1}}, "failures": []})) + async with s.begin(): + await run_finished(s, run_id) + + got = {e.type: e for e in await _entries(app, name)} + assert set(got) == {"alert.fired", "steward.config", "ansible.changed"} + fired = got["alert.fired"] + assert fired.summary == f"hot {tag} fired: CPU on {name} is 97% (> 90%)" + assert fired.area == "host_os" and fired.ref_type == "alert_event" + assert got["steward.config"].summary == f"Added host {name}" + assert got["steward.config"].actor == f"tl-{tag}" + assert got["ansible.changed"].summary.endswith(": changed 1 host") + assert got["ansible.changed"].actor == f"tl-{tag}" + + # The records age out; the entries stay, saying their details are gone. + async with app.db_sessionmaker() as s: + async with s.begin(): + await s.execute(delete(AlertEvent).where(AlertEvent.rule_id == rule_id)) + await s.execute(delete(AnsibleRun).where(AnsibleRun.id == run_id)) + async with app.db_sessionmaker() as s: + cfg = await tl_config.load(s) + card = {v["entry"].type: v for v in await tl.recent_for_host(s, cfg, name)} + assert set(card) == {"alert.fired", "steward.config", "ansible.changed"} + assert card["alert.fired"]["gone"] and card["alert.fired"]["url"] is None + assert card["ansible.changed"]["gone"] + assert not card["steward.config"]["gone"] + + # Renaming the host carries its history to the new name. + async with app.db_sessionmaker() as s: + async with s.begin(): + assert await tl.carry_host_rename(s, name, renamed) == 3 + assert await _entries(app, name) == [] + assert len(await _entries(app, renamed)) == 3 + + asyncio.run(_go()) + + +def test_settings_re_sort_and_hide_entries_already_written(app): + from sqlalchemy import select + from steward.models.timeline import TimelineEntry + from steward.timeline import catalog + from steward.timeline.config import Config + from steward.timeline.record import record + from steward.timeline.service import query + + async def _go(): + host = f"tl-{uuid.uuid4().hex[:8]}" + now = datetime.now(timezone.utc) + async with app.db_sessionmaker() as s: + async with s.begin(): + record(s, type="ansible.changed", at=now, summary="ran", hosts=[host]) + record(s, type="ansible.check", at=now, summary="dry run", hosts=[host]) + record(s, type="alert.fired", at=now, summary="hot", hosts=[host], area="hardware") + + async def types(cfg, **kw): + async with app.db_sessionmaker() as s: + rows = (await s.execute(query(cfg, host=host, **kw))).scalars().all() + return sorted(e.type for e in rows) + + areas = [list(a) for a in catalog.DEFAULT_AREAS] + cfg = Config(areas=areas) + assert await types(cfg) == ["alert.fired", "ansible.changed"] # the dry run is hidden + assert await types(cfg, include_hidden=True) == ["alert.fired", "ansible.changed", "ansible.check"] + assert await types(Config(areas=areas, types={"ansible.check": {"shown": True}})) == \ + ["alert.fired", "ansible.changed", "ansible.check"] + assert await types(cfg, area="host_os") == ["ansible.changed"] + assert await types(cfg, area="hardware") == ["alert.fired"] + assert await types(cfg, kind="automation", include_hidden=True) == ["ansible.changed", "ansible.check"] + + # A type given an area moves every entry of it, past ones included. + moved = Config(areas=areas, types={"ansible.changed": {"area": "service"}}) + assert await types(moved, area="service") == ["ansible.changed"] + assert await types(moved, area="host_os") == [] + # A removed area's entries sort under the one they were moved to. + merged = Config(areas=[a for a in areas if a[0] != "hardware"], moved={"hardware": "storage"}) + assert await types(merged, area="storage") == ["alert.fired"] + # An area nobody lists any more sorts under Other. + lost = Config(areas=[a for a in areas if a[0] != "hardware"]) + assert await types(lost, area="other") == ["alert.fired"] + + async with app.db_sessionmaker() as s: + async with s.begin(): + for e in (await s.execute(select(TimelineEntry).where( + TimelineEntry.hosts.contains([host])))).scalars(): + await s.delete(e) + + asyncio.run(_go()) + + +def test_the_first_fill_writes_what_predates_the_timeline_once(app): + from steward.core.settings import get_setting, set_setting + from steward.models.alerts import AlertEvent, AlertOperator, AlertRule + from steward.models.users import User, UserRole + from steward.timeline.service import backfill + + async def _go(): + tag = uuid.uuid4().hex[:8] + name = f"tl-{tag}" + async with app.db_sessionmaker() as s: + async with s.begin(): + before = await get_setting(s, "timeline.backfilled_at") + user = User(username=f"tl-{tag}", email=f"tl-{tag}@test", password_hash="x", + role=UserRole.admin) + s.add(user) + await s.flush() + rule = AlertRule(name=f"disk {tag}", source_module="host_agent", resource_name=f"{name}:/", + metric_name="disk_used_pct", operator=AlertOperator.gt, threshold=90, + created_by=user.id) + s.add(rule) + await s.flush() + # Written straight to the table, as before the timeline existed. + s.add(AlertEvent(rule_id=rule.id, from_state="pending", to_state="firing", metric_value=95.0, + transitioned_at=datetime.now(timezone.utc) - timedelta(days=3))) + await set_setting(s, "timeline.backfilled_at", "") + try: + assert await backfill(app) >= 1 + [entry] = await _entries(app, name) + assert entry.type == "alert.fired" and entry.area == "storage" + assert entry.subject == f"{name}:/" + assert await backfill(app) == 0 # once + async with app.db_sessionmaker() as s: + async with s.begin(): + await set_setting(s, "timeline.backfilled_at", "") + await backfill(app) # again: what is covered is skipped + assert len(await _entries(app, name)) == 1 + finally: + async with app.db_sessionmaker() as s: + async with s.begin(): + await set_setting(s, "timeline.backfilled_at", before or "done") + + asyncio.run(_go()) + + +def test_retention_prunes_only_once_it_is_set(app): + from steward.core.settings import set_setting + from steward.timeline.record import record + from steward.timeline.service import prune + + async def _go(): + old = datetime.now(timezone.utc) - timedelta(days=400) + async with app.db_sessionmaker() as s: + record(s, type="steward.config", at=old, summary="old", hosts=[f"tl-{uuid.uuid4().hex[:8]}"]) + await s.flush() + assert await prune(s, datetime.now(timezone.utc)) == (0, 0) # 0 days keeps everything + await set_setting(s, "timeline.retention_days", 365) + removed, days = await prune(s, datetime.now(timezone.utc)) + assert days == 365 and removed >= 1 + await s.rollback() # leave the shared database as it was + + asyncio.run(_go()) + + +def test_get_timeline_reads_a_host_s_history(app): + from steward.core.api_tokens import Caller + from steward.mcp import _context + from steward.mcp.tools.timeline import get_timeline + from steward.models.users import UserRole + from steward.timeline.record import record + + async def _go(): + host = f"tl-{uuid.uuid4().hex[:8]}" + now = datetime.now(timezone.utc) + async with app.db_sessionmaker() as s: + async with s.begin(): + record(s, type="ansible.changed", at=now - timedelta(hours=2), summary="update.yml: changed 1 host", + hosts=[host], detail={"url": "/ansible/runs/x"}, ref=("ansible_run", str(uuid.uuid4()))) + record(s, type="ansible.check", at=now - timedelta(hours=1), summary="dry run", hosts=[host]) + _context._app_ctx.set(app) + _context._caller_ctx.set(Caller(user_id="u", username="viewer", role=UserRole.viewer, token_id=0)) + out = await get_timeline(host=host) + assert [e["summary"] for e in out["entries"]] == ["update.yml: changed 1 host"] + [e] = out["entries"] + assert e["kind"] == "automation" and e["area"] == "host_os" and e["hosts"] == [host] + assert e["details_kept"] is False and e["url"] is None # the run is not in the table + assert out["areas"]["host_os"] == "Host OS" and "alert.fired" in out["types"] + both = await get_timeline(host=host, include_hidden=True) + assert [x["type"] for x in both["entries"]] == ["ansible.check", "ansible.changed"] + assert (await get_timeline(host=host, kind="alert"))["entries"] == [] + with pytest.raises(ValueError, match="kind"): + await get_timeline(kind="nope") + with pytest.raises(ValueError, match="area"): + await get_timeline(area="nope") + + asyncio.run(_go()) diff --git a/tests/test_breadcrumbs.py b/tests/test_breadcrumbs.py index 2c5fcec..cda3ada 100644 --- a/tests/test_breadcrumbs.py +++ b/tests/test_breadcrumbs.py @@ -17,6 +17,7 @@ TOP_LEVEL = { "steward/templates/alerts/list.html", "steward/templates/ansible/index.html", "steward/templates/audit/list.html", + "steward/templates/timeline/list.html", "steward/templates/ai_access/index.html", "steward/templates/account/index.html", "plugins/docker/templates/docker/index.html",