feat(timeline): one record of what happened to the infrastructure, newest first (#6348)
A stored log, timeline_entries, with one row per thing that happened. Each source writes its row in the same transaction as its own record, so the history outlives the 30/90/365-day retention of the tables it points to: - alert events (core/alerts._add_event): fired, resolved, acknowledged, went silent, readings back; - finished Ansible runs (executor, and the startup sweep of interrupted runs): changed something, changed nothing, failed, dry run; - audit rows (core/audit.log_audit): Steward's settings, users and access, sign-ins, backups. Actions another writer records are left to it. Entries carry a kind (fixed) and an area (Service, Host OS, Hardware, Storage, Network, Security, Monitoring). Settings → Timeline edits the areas and each type's area and visibility; both are applied when the timeline is read, so a change re-sorts past entries and a hidden type comes back when shown. Kept forever unless a retention is set. The first start fills the timeline from the last 90 days of records once, skipping any already covered. A host rename carries its entries. UI: Timeline in the sidebar (filter by host, area, kind; scrolls on), a History card on each host page. MCP: get_timeline. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -113,7 +113,8 @@ def test_a_sample_after_silence_clears_it_and_says_so():
|
||||
assert note == "data resumed after 15m silent"
|
||||
|
||||
|
||||
_RULE = NS(id="r", operator=AlertOperator.gt, threshold=90.0,
|
||||
_RULE = NS(id="r", name="Slow database", source_module="steward", resource_name="steward",
|
||||
metric_name="db_roundtrip_ms", operator=AlertOperator.gt, threshold=90.0,
|
||||
consecutive_failures_required=1, consecutive_ok_required=1)
|
||||
|
||||
|
||||
@@ -132,8 +133,12 @@ def test_a_resumed_series_gets_one_event_for_what_its_first_sample_did(
|
||||
session, st = _Session(), _state(state=before)
|
||||
said = "data resumed after 15m silent"
|
||||
out = asyncio.run(_evaluate_rule(session, _RULE, st, value, NOW, resumed=said))
|
||||
[event] = session.added
|
||||
event, entry = session.added # the alert's event, then its timeline entry
|
||||
assert (event.from_state, event.to_state) == want_event
|
||||
assert (entry.ref_type, entry.ref_id, entry.at) == ("alert_event", event.id, NOW)
|
||||
# A firing alert still firing when its series returns is a resume, not a new firing.
|
||||
assert entry.type == {"firing": "alert.fired", "resolved": "alert.resolved",
|
||||
"data_resumed": "alert.resumed"}[want_notice]
|
||||
assert event.note == said and event.transitioned_at == NOW
|
||||
assert [(n, e) for n, e, _ in out] == [(want_notice, event.id)]
|
||||
# Without a resume, a sample that changes nothing records nothing.
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
"""The timeline: what each source becomes, how Settings → Timeline sorts and
|
||||
hides it when it is read, and the pages that show it."""
|
||||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace as NS
|
||||
|
||||
import pytest
|
||||
from quart import render_template
|
||||
from werkzeug.datastructures import MultiDict
|
||||
|
||||
from steward.models.alerts import AlertOperator
|
||||
from steward.timeline import catalog, record, service
|
||||
from steward.timeline.config import Config, edited
|
||||
|
||||
NOW = datetime(2026, 10, 24, 12, 0, tzinfo=timezone.utc)
|
||||
AREAS = [list(a) for a in catalog.DEFAULT_AREAS]
|
||||
|
||||
|
||||
def _cfg(**kw) -> Config:
|
||||
return Config(areas=[list(a) for a in kw.pop("areas", AREAS)], **kw)
|
||||
|
||||
|
||||
def _entry(type_="alert.fired", area="hardware", **kw):
|
||||
base = dict(id="e1", at=NOW, until=None, type=type_, area=area, hosts=["web-1"],
|
||||
subject="web-1:sda", summary="Disk hot fired", detail={}, actor="system",
|
||||
ref_type=None, ref_id=None)
|
||||
return NS(**{**base, **kw})
|
||||
|
||||
|
||||
# ── What a source becomes ───────────────────────────────────────────────────
|
||||
|
||||
@pytest.mark.parametrize("action,type_", [
|
||||
("monitor.disabled", "steward.config"),
|
||||
("settings.saved", "steward.config"),
|
||||
("host_agent.maintenance_plan_created", "steward.config"),
|
||||
("user.role_changed", "steward.access"),
|
||||
("api_token.create", "steward.access"),
|
||||
("account.password_changed", "steward.access"),
|
||||
("auth.login", "steward.signin"),
|
||||
("auth.login_failed", "steward.signin"),
|
||||
("backup.made", "steward.backup"),
|
||||
# Another writer records the event itself, or it is one person's preference.
|
||||
("ansible.run.started", None),
|
||||
("ansible.host_run", None),
|
||||
("alert_rule.acknowledged", None),
|
||||
("host_agent.maintenance", None),
|
||||
("docker.prune.started", None),
|
||||
("account.time_zone", None),
|
||||
])
|
||||
def test_each_audit_action_is_entered_as_its_type_or_left_out(action, type_):
|
||||
assert catalog.audit_type(action) == type_
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source,metric,area", [
|
||||
("host_agent", "temp_c_max", "hardware"),
|
||||
("host_agent", "drive_failing", "hardware"),
|
||||
("host_agent", "disk_used_pct", "storage"),
|
||||
("host_agent", "mounts_stalled", "storage"),
|
||||
("host_agent", "nfs_rtt_ms", "storage"),
|
||||
("host_agent", "net_rx_bps", "network"),
|
||||
("host_agent", "updates_security", "security"),
|
||||
("host_agent", "cpu_pct", "host_os"),
|
||||
("host_agent", "updates_pending", "host_os"),
|
||||
("http", "is_up", "service"),
|
||||
("icmp", "is_up", "network"),
|
||||
("docker", "mem_used_pct", "service"),
|
||||
("traefik", "error_rate_5xx_pct", "network"),
|
||||
("steward", "db_roundtrip_ms", "monitoring"),
|
||||
("snmp", "volume_used_pct", "storage"),
|
||||
("snmp", "ups_battery_pct", "hardware"),
|
||||
])
|
||||
def test_an_alert_is_written_under_the_area_its_rule_watches(source, metric, area):
|
||||
assert catalog.area_of_metric(source, metric) == area
|
||||
|
||||
|
||||
@pytest.mark.parametrize("from_state,to_state,type_", [
|
||||
("pending", "firing", "alert.fired"),
|
||||
("no_data", "firing", "alert.fired"),
|
||||
("firing", "resolved", "alert.resolved"),
|
||||
("firing", "acknowledged", "alert.acknowledged"),
|
||||
("inactive", "no_data", "alert.silent"),
|
||||
("no_data", "inactive", "alert.resumed"),
|
||||
("inactive", "pending", None),
|
||||
])
|
||||
def test_an_alert_event_becomes_one_entry_type(from_state, to_state, type_):
|
||||
assert record._alert_type(from_state, to_state) == type_
|
||||
|
||||
|
||||
def test_an_alert_entry_reads_as_a_sentence():
|
||||
rule = NS(name="High CPU", metric_name="cpu_pct", resource_name="web-1",
|
||||
operator=AlertOperator.gt, threshold=90.0)
|
||||
assert record.alert_summary("alert.fired", rule, 94.0) == "High CPU fired: CPU on web-1 is 94% (> 90%)"
|
||||
assert record.alert_summary("alert.resolved", rule, 40.0) == "High CPU resolved: CPU on web-1 is 40%"
|
||||
assert record.alert_summary("alert.resolved", rule, None) == "High CPU resolved"
|
||||
assert record.alert_summary("alert.silent", rule, None) == "High CPU: no CPU reading from web-1"
|
||||
up = NS(name="API up", metric_name="is_up", resource_name="api", operator=AlertOperator.lt, threshold=1.0)
|
||||
assert record.alert_summary("alert.fired", up, 0.0) == "API up fired: api is down"
|
||||
|
||||
|
||||
def test_an_entry_takes_its_type_s_area_unless_the_writer_names_one():
|
||||
added = []
|
||||
session = NS(add=added.append)
|
||||
e = record.record(session, type="ansible.changed", at=NOW, summary="x", hosts=["b", "a", "a", ""])
|
||||
assert e.area == "host_os" and e.hosts == ["a", "b"] and added == [e]
|
||||
assert record.record(session, type="ansible.changed", at=NOW, summary="x", area="service").area == "service"
|
||||
assert record.record(session, type="plugin.gone", at=NOW, summary="x").area == "other"
|
||||
|
||||
|
||||
# ── Settings → Timeline, applied when read ──────────────────────────────────
|
||||
|
||||
def test_an_entry_sorts_under_its_own_area_until_its_type_is_given_one():
|
||||
e = _entry(area="hardware")
|
||||
assert _cfg().area_of(e) == "hardware"
|
||||
assert _cfg(types={"alert.fired": {"area": "monitoring"}}).area_of(e) == "monitoring"
|
||||
|
||||
|
||||
def test_a_removed_area_s_entries_follow_where_it_was_moved():
|
||||
cfg = _cfg(areas=[a for a in AREAS if a[0] not in ("hardware", "storage")],
|
||||
moved={"hardware": "storage", "storage": "host_os"})
|
||||
assert cfg.area_of(_entry(area="hardware")) == "host_os"
|
||||
# An area nobody lists any more, and a loop, end in Other rather than hang.
|
||||
assert cfg.area_of(_entry(area="from_a_plugin")) == "other"
|
||||
assert _cfg(areas=AREAS[:1], moved={"a": "b", "b": "a"}).follow("a") == "other"
|
||||
assert cfg.area_label("other") == "Other"
|
||||
|
||||
|
||||
def test_a_type_is_shown_as_the_catalog_says_until_settings_say_otherwise():
|
||||
cfg = _cfg()
|
||||
assert cfg.shown("alert.fired") and not cfg.shown("ansible.check")
|
||||
assert {"ansible.check", "ansible.unchanged", "steward.signin"} <= cfg.hidden_types()
|
||||
cfg = _cfg(types={"ansible.check": {"shown": True}, "alert.fired": {"shown": False}})
|
||||
assert cfg.shown("ansible.check") and not cfg.shown("alert.fired")
|
||||
assert "alert.fired" in cfg.hidden_types() and "ansible.check" not in cfg.hidden_types()
|
||||
|
||||
|
||||
def _form(cfg: Config, **extra) -> MultiDict:
|
||||
"""The form as the page posts it unchanged: every type at its current
|
||||
visibility, every area kept, then `extra`."""
|
||||
data = {f"type_shown_{k}": "on" for k in catalog.TYPES if cfg.shown(k)}
|
||||
data.update({f"area_label_{k}": label for k, label in cfg.areas})
|
||||
data.update(extra)
|
||||
return MultiDict({k: v for k, v in data.items() if v is not None})
|
||||
|
||||
|
||||
def test_saving_the_page_unchanged_stores_no_overrides():
|
||||
cfg = _cfg()
|
||||
new, error = edited(cfg, _form(cfg))
|
||||
assert error is None and new.types == {} and new.moved == {} and new.areas == AREAS
|
||||
|
||||
|
||||
def test_an_area_is_renamed_added_and_removed_into_another():
|
||||
cfg = _cfg(types={"alert.fired": {"area": "hardware"}})
|
||||
new, error = edited(cfg, _form(cfg, area_label_network="Networking", new_area="Power & UPS",
|
||||
area_move_hardware="storage",
|
||||
**{"type_area_alert.fired": "hardware"}))
|
||||
assert error is None
|
||||
assert dict(new.areas)["network"] == "Networking"
|
||||
assert new.areas[-1] == ["power_ups", "Power & UPS"]
|
||||
assert "hardware" not in dict(new.areas) and new.moved == {"hardware": "storage"}
|
||||
# A type sorted under the removed area follows its entries.
|
||||
assert new.types["alert.fired"] == {"area": "storage"}
|
||||
|
||||
|
||||
def test_a_type_is_hidden_or_given_an_area():
|
||||
cfg = _cfg()
|
||||
form = _form(cfg, **{"type_area_ansible.changed": "service"})
|
||||
del form["type_shown_alert.resumed"]
|
||||
new, error = edited(cfg, form)
|
||||
assert error is None
|
||||
assert new.types == {"ansible.changed": {"area": "service"}, "alert.resumed": {"shown": False}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("extra,said", [
|
||||
({"area_move_network": "network"}, "Pick another area"),
|
||||
({"area_move_network": "nowhere"}, "Pick another area"),
|
||||
({"new_area": "Storage"}, "already an area"),
|
||||
({"new_area": "Other"}, "already an area"),
|
||||
({"type_area_alert.fired": "nowhere"}, "isn't listed"),
|
||||
])
|
||||
def test_a_form_that_can_t_be_saved_says_why_and_changes_nothing(extra, said):
|
||||
cfg = _cfg()
|
||||
new, error = edited(cfg, _form(cfg, **extra))
|
||||
assert said in error and new is cfg
|
||||
|
||||
|
||||
def test_the_last_area_can_t_be_removed():
|
||||
cfg = _cfg(areas=[["service", "Service"], ["network", "Network"]])
|
||||
form = _form(cfg, area_move_service="network", area_move_network="service")
|
||||
assert edited(cfg, form)[1] == "Keep at least one area."
|
||||
|
||||
|
||||
def test_a_plugin_s_type_keeps_its_settings_while_the_plugin_is_away():
|
||||
cfg = _cfg(types={"unifi.client_joined": {"shown": False}})
|
||||
new, _ = edited(cfg, _form(cfg))
|
||||
assert new.types == {"unifi.client_joined": {"shown": False}}
|
||||
|
||||
|
||||
# ── Read back ───────────────────────────────────────────────────────────────
|
||||
|
||||
def test_an_entry_whose_record_aged_out_says_so_instead_of_linking():
|
||||
e = _entry(ref_type="ansible_run", ref_id="r1", detail={"url": "/ansible/runs/r1", "note": "n"})
|
||||
kept = service.view(e, _cfg(), {("ansible_run", "r1")})
|
||||
gone = service.view(e, _cfg(), set())
|
||||
assert kept["url"] == "/ansible/runs/r1" and not kept["gone"] and kept["note"] == "n"
|
||||
assert gone["url"] is None and gone["gone"]
|
||||
assert kept["kind_label"] == "Alert" and kept["area_label"] == "Hardware"
|
||||
# An entry with no record behind it (a hand-written one) is never "gone".
|
||||
assert not service.view(_entry(), _cfg(), set())["gone"]
|
||||
|
||||
|
||||
# ── Pages ───────────────────────────────────────────────────────────────────
|
||||
|
||||
def _item(e, **kw):
|
||||
v = service.view(e, _cfg(), {(e.ref_type, e.ref_id)})
|
||||
v.update(group="Today", opens_group=True, **kw)
|
||||
return v
|
||||
|
||||
|
||||
async def test_the_page_lists_what_happened_where_and_by_whom(app):
|
||||
items = [_item(_entry(ref_type="alert_event", ref_id="a1", detail={"url": "/alerts/history?rule=r"})),
|
||||
_item(_entry(type_="steward.config", area="monitoring", hosts=[], subject=None,
|
||||
summary="Paused monitor api", actor="bryan via MCP"), opens_group=False)]
|
||||
async with app.test_request_context("/timeline/"):
|
||||
page = await render_template("timeline/list.html", items=items, after=None, last_group="Today",
|
||||
filters={"host": "", "area": "", "kind": "", "hidden": False},
|
||||
keep={}, areas=AREAS, kinds=catalog.KINDS, hosts=["web-1"])
|
||||
assert "Disk hot fired" in page and "Paused monitor api" in page
|
||||
assert '<a href="/timeline/?host=web-1">web-1</a>' in page
|
||||
assert '<a href="/alerts/history?rule=r">Details</a>' in page
|
||||
assert "bryan via MCP" in page and ">Hardware<" in page and "Alert · " in page
|
||||
assert page.count('class="table-divider"') == 1
|
||||
assert "End of the list." in page
|
||||
|
||||
|
||||
async def test_the_next_page_keeps_the_filters(app):
|
||||
items = [_item(_entry())]
|
||||
async with app.test_request_context("/timeline/"):
|
||||
rows = await render_template("timeline/_rows.html", items=items, last_group="Today",
|
||||
after="2026-10-24T12:00:00+00:00|e1", keep={"host": "web-1"})
|
||||
assert 'hx-get="/timeline/?before=2026-10-24T12%3A00%3A00%2B00%3A00%7Ce1&host=web-1&group=Today"' in rows
|
||||
|
||||
|
||||
async def test_an_empty_timeline_says_so(app):
|
||||
async with app.test_request_context("/timeline/"):
|
||||
page = await render_template("timeline/list.html", items=[], after=None, last_group="",
|
||||
filters={"host": "x", "area": "", "kind": "", "hidden": False},
|
||||
keep={"host": "x"}, areas=AREAS, kinds=catalog.KINDS, hosts=[])
|
||||
assert "Nothing on the timeline matches these filters." in page
|
||||
|
||||
|
||||
async def test_a_host_s_card_shows_its_latest_and_links_the_rest(app):
|
||||
host = NS(id="h1", name="web-1")
|
||||
async with app.test_request_context("/timeline/host/h1"):
|
||||
card = await render_template("timeline/_host_card.html", host=host,
|
||||
items=[_item(_entry(ref_type="ansible_run", ref_id="gone"))])
|
||||
empty = await render_template("timeline/_host_card.html", host=host, items=[])
|
||||
assert "Disk hot fired" in card and 'href="/timeline/?host=web-1"' in card
|
||||
assert "Nothing has happened to web-1" in empty
|
||||
|
||||
|
||||
async def test_settings_lists_each_type_under_its_kind(app):
|
||||
from steward.timeline.routes import _settings_ctx
|
||||
cfg = _cfg(types={"alert.fired": {"area": "monitoring"}})
|
||||
async with app.test_request_context("/settings/timeline/"):
|
||||
page = await render_template("settings/timeline.html", **_settings_ctx(cfg, 0))
|
||||
for t in catalog.TYPES.values():
|
||||
assert t.label in page
|
||||
assert 'name="type_shown_alert.fired" aria-label="Show Alert fired" checked' in page
|
||||
assert 'name="type_shown_ansible.check" aria-label="Show Playbook dry run">' in page
|
||||
assert '<option value="monitoring" selected>Monitoring</option>' in page
|
||||
assert 'aria-current="page">Timeline</a>' in page
|
||||
@@ -0,0 +1,265 @@
|
||||
"""Integration: the timeline against real Postgres. Each source writes its
|
||||
entry as the thing happens, the entry outlives its record, Settings re-sorts
|
||||
and hides entries when they are read, the first fill skips what is covered,
|
||||
and get_timeline reads it back. The database is shared with other tests, so
|
||||
each test looks for its own uniquely named host.
|
||||
Requires STEWARD_DATABASE_URL.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
if not os.environ.get("STEWARD_DATABASE_URL"):
|
||||
pytest.skip("needs Postgres")
|
||||
from steward.app import create_app
|
||||
return create_app(testing=False)
|
||||
|
||||
|
||||
async def _entries(app, host: str):
|
||||
from sqlalchemy import select
|
||||
from steward.models.timeline import TimelineEntry
|
||||
async with app.db_sessionmaker() as s:
|
||||
return (await s.execute(select(TimelineEntry).where(TimelineEntry.hosts.contains([host]))
|
||||
.order_by(TimelineEntry.at))).scalars().all()
|
||||
|
||||
|
||||
def test_each_source_writes_an_entry_that_outlives_its_record(app):
|
||||
from sqlalchemy import delete
|
||||
from steward.alerts import service as alert_svc
|
||||
from steward.core.alerts import record_metric, record_metrics
|
||||
from steward.core.audit import log_audit
|
||||
from steward.models.alerts import AlertEvent
|
||||
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.metrics import PluginMetric
|
||||
from steward.models.users import User, UserRole
|
||||
from steward.timeline import config as tl_config, service as tl
|
||||
from steward.timeline.record import run_finished
|
||||
|
||||
async def _go():
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
name, renamed, run_id = f"tl-{tag}", f"tl-{tag}-new", str(uuid.uuid4())
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
user = User(username=f"tl-{tag}", email=f"tl-{tag}@test", password_hash="x",
|
||||
role=UserRole.admin)
|
||||
host = Host(name=name, address="10.1.2.3")
|
||||
s.add_all([user, host])
|
||||
await record_metrics(s, [PluginMetric(source_module="host_agent", resource_name=name,
|
||||
metric_name="cpu_pct", value=1.0)])
|
||||
await s.flush()
|
||||
fields = await alert_svc.rule_fields(
|
||||
s, name=f"hot {tag}", source_module="host_agent", resource_name=name,
|
||||
metric_name="cpu_pct", operator=">", threshold=90,
|
||||
consecutive_failures_required=1, consecutive_ok_required=1)
|
||||
rule = await alert_svc.add_rule(s, fields, created_by=user.id)
|
||||
user_id, host_id, rule_id = user.id, host.id, rule.id
|
||||
|
||||
# An alert firing.
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await record_metric(s, "host_agent", name, "cpu_pct", 97.0)
|
||||
# A change to Steward's settings about the host, and one the timeline leaves to the run.
|
||||
await log_audit(app, user_id, f"tl-{tag}", "host.created", "host", host_id, {"name": name})
|
||||
await log_audit(app, user_id, f"tl-{tag}", "ansible.run.started", "ansible_run", run_id, {})
|
||||
# A playbook run that changed the host.
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
s.add(AnsibleRun(id=run_id, playbook_path="update.yml", source_name="local",
|
||||
inventory_scope=f"steward:target:{uuid.uuid4()}",
|
||||
status=AnsibleRunStatus.success, triggered_by=user_id,
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
results={"hosts": {name: {"ok": 3, "changed": 1}}, "failures": []}))
|
||||
async with s.begin():
|
||||
await run_finished(s, run_id)
|
||||
|
||||
got = {e.type: e for e in await _entries(app, name)}
|
||||
assert set(got) == {"alert.fired", "steward.config", "ansible.changed"}
|
||||
fired = got["alert.fired"]
|
||||
assert fired.summary == f"hot {tag} fired: CPU on {name} is 97% (> 90%)"
|
||||
assert fired.area == "host_os" and fired.ref_type == "alert_event"
|
||||
assert got["steward.config"].summary == f"Added host {name}"
|
||||
assert got["steward.config"].actor == f"tl-{tag}"
|
||||
assert got["ansible.changed"].summary.endswith(": changed 1 host")
|
||||
assert got["ansible.changed"].actor == f"tl-{tag}"
|
||||
|
||||
# The records age out; the entries stay, saying their details are gone.
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await s.execute(delete(AlertEvent).where(AlertEvent.rule_id == rule_id))
|
||||
await s.execute(delete(AnsibleRun).where(AnsibleRun.id == run_id))
|
||||
async with app.db_sessionmaker() as s:
|
||||
cfg = await tl_config.load(s)
|
||||
card = {v["entry"].type: v for v in await tl.recent_for_host(s, cfg, name)}
|
||||
assert set(card) == {"alert.fired", "steward.config", "ansible.changed"}
|
||||
assert card["alert.fired"]["gone"] and card["alert.fired"]["url"] is None
|
||||
assert card["ansible.changed"]["gone"]
|
||||
assert not card["steward.config"]["gone"]
|
||||
|
||||
# Renaming the host carries its history to the new name.
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
assert await tl.carry_host_rename(s, name, renamed) == 3
|
||||
assert await _entries(app, name) == []
|
||||
assert len(await _entries(app, renamed)) == 3
|
||||
|
||||
asyncio.run(_go())
|
||||
|
||||
|
||||
def test_settings_re_sort_and_hide_entries_already_written(app):
|
||||
from sqlalchemy import select
|
||||
from steward.models.timeline import TimelineEntry
|
||||
from steward.timeline import catalog
|
||||
from steward.timeline.config import Config
|
||||
from steward.timeline.record import record
|
||||
from steward.timeline.service import query
|
||||
|
||||
async def _go():
|
||||
host = f"tl-{uuid.uuid4().hex[:8]}"
|
||||
now = datetime.now(timezone.utc)
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
record(s, type="ansible.changed", at=now, summary="ran", hosts=[host])
|
||||
record(s, type="ansible.check", at=now, summary="dry run", hosts=[host])
|
||||
record(s, type="alert.fired", at=now, summary="hot", hosts=[host], area="hardware")
|
||||
|
||||
async def types(cfg, **kw):
|
||||
async with app.db_sessionmaker() as s:
|
||||
rows = (await s.execute(query(cfg, host=host, **kw))).scalars().all()
|
||||
return sorted(e.type for e in rows)
|
||||
|
||||
areas = [list(a) for a in catalog.DEFAULT_AREAS]
|
||||
cfg = Config(areas=areas)
|
||||
assert await types(cfg) == ["alert.fired", "ansible.changed"] # the dry run is hidden
|
||||
assert await types(cfg, include_hidden=True) == ["alert.fired", "ansible.changed", "ansible.check"]
|
||||
assert await types(Config(areas=areas, types={"ansible.check": {"shown": True}})) == \
|
||||
["alert.fired", "ansible.changed", "ansible.check"]
|
||||
assert await types(cfg, area="host_os") == ["ansible.changed"]
|
||||
assert await types(cfg, area="hardware") == ["alert.fired"]
|
||||
assert await types(cfg, kind="automation", include_hidden=True) == ["ansible.changed", "ansible.check"]
|
||||
|
||||
# A type given an area moves every entry of it, past ones included.
|
||||
moved = Config(areas=areas, types={"ansible.changed": {"area": "service"}})
|
||||
assert await types(moved, area="service") == ["ansible.changed"]
|
||||
assert await types(moved, area="host_os") == []
|
||||
# A removed area's entries sort under the one they were moved to.
|
||||
merged = Config(areas=[a for a in areas if a[0] != "hardware"], moved={"hardware": "storage"})
|
||||
assert await types(merged, area="storage") == ["alert.fired"]
|
||||
# An area nobody lists any more sorts under Other.
|
||||
lost = Config(areas=[a for a in areas if a[0] != "hardware"])
|
||||
assert await types(lost, area="other") == ["alert.fired"]
|
||||
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
for e in (await s.execute(select(TimelineEntry).where(
|
||||
TimelineEntry.hosts.contains([host])))).scalars():
|
||||
await s.delete(e)
|
||||
|
||||
asyncio.run(_go())
|
||||
|
||||
|
||||
def test_the_first_fill_writes_what_predates_the_timeline_once(app):
|
||||
from steward.core.settings import get_setting, set_setting
|
||||
from steward.models.alerts import AlertEvent, AlertOperator, AlertRule
|
||||
from steward.models.users import User, UserRole
|
||||
from steward.timeline.service import backfill
|
||||
|
||||
async def _go():
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
name = f"tl-{tag}"
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
before = await get_setting(s, "timeline.backfilled_at")
|
||||
user = User(username=f"tl-{tag}", email=f"tl-{tag}@test", password_hash="x",
|
||||
role=UserRole.admin)
|
||||
s.add(user)
|
||||
await s.flush()
|
||||
rule = AlertRule(name=f"disk {tag}", source_module="host_agent", resource_name=f"{name}:/",
|
||||
metric_name="disk_used_pct", operator=AlertOperator.gt, threshold=90,
|
||||
created_by=user.id)
|
||||
s.add(rule)
|
||||
await s.flush()
|
||||
# Written straight to the table, as before the timeline existed.
|
||||
s.add(AlertEvent(rule_id=rule.id, from_state="pending", to_state="firing", metric_value=95.0,
|
||||
transitioned_at=datetime.now(timezone.utc) - timedelta(days=3)))
|
||||
await set_setting(s, "timeline.backfilled_at", "")
|
||||
try:
|
||||
assert await backfill(app) >= 1
|
||||
[entry] = await _entries(app, name)
|
||||
assert entry.type == "alert.fired" and entry.area == "storage"
|
||||
assert entry.subject == f"{name}:/"
|
||||
assert await backfill(app) == 0 # once
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await set_setting(s, "timeline.backfilled_at", "")
|
||||
await backfill(app) # again: what is covered is skipped
|
||||
assert len(await _entries(app, name)) == 1
|
||||
finally:
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await set_setting(s, "timeline.backfilled_at", before or "done")
|
||||
|
||||
asyncio.run(_go())
|
||||
|
||||
|
||||
def test_retention_prunes_only_once_it_is_set(app):
|
||||
from steward.core.settings import set_setting
|
||||
from steward.timeline.record import record
|
||||
from steward.timeline.service import prune
|
||||
|
||||
async def _go():
|
||||
old = datetime.now(timezone.utc) - timedelta(days=400)
|
||||
async with app.db_sessionmaker() as s:
|
||||
record(s, type="steward.config", at=old, summary="old", hosts=[f"tl-{uuid.uuid4().hex[:8]}"])
|
||||
await s.flush()
|
||||
assert await prune(s, datetime.now(timezone.utc)) == (0, 0) # 0 days keeps everything
|
||||
await set_setting(s, "timeline.retention_days", 365)
|
||||
removed, days = await prune(s, datetime.now(timezone.utc))
|
||||
assert days == 365 and removed >= 1
|
||||
await s.rollback() # leave the shared database as it was
|
||||
|
||||
asyncio.run(_go())
|
||||
|
||||
|
||||
def test_get_timeline_reads_a_host_s_history(app):
|
||||
from steward.core.api_tokens import Caller
|
||||
from steward.mcp import _context
|
||||
from steward.mcp.tools.timeline import get_timeline
|
||||
from steward.models.users import UserRole
|
||||
from steward.timeline.record import record
|
||||
|
||||
async def _go():
|
||||
host = f"tl-{uuid.uuid4().hex[:8]}"
|
||||
now = datetime.now(timezone.utc)
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
record(s, type="ansible.changed", at=now - timedelta(hours=2), summary="update.yml: changed 1 host",
|
||||
hosts=[host], detail={"url": "/ansible/runs/x"}, ref=("ansible_run", str(uuid.uuid4())))
|
||||
record(s, type="ansible.check", at=now - timedelta(hours=1), summary="dry run", hosts=[host])
|
||||
_context._app_ctx.set(app)
|
||||
_context._caller_ctx.set(Caller(user_id="u", username="viewer", role=UserRole.viewer, token_id=0))
|
||||
out = await get_timeline(host=host)
|
||||
assert [e["summary"] for e in out["entries"]] == ["update.yml: changed 1 host"]
|
||||
[e] = out["entries"]
|
||||
assert e["kind"] == "automation" and e["area"] == "host_os" and e["hosts"] == [host]
|
||||
assert e["details_kept"] is False and e["url"] is None # the run is not in the table
|
||||
assert out["areas"]["host_os"] == "Host OS" and "alert.fired" in out["types"]
|
||||
both = await get_timeline(host=host, include_hidden=True)
|
||||
assert [x["type"] for x in both["entries"]] == ["ansible.check", "ansible.changed"]
|
||||
assert (await get_timeline(host=host, kind="alert"))["entries"] == []
|
||||
with pytest.raises(ValueError, match="kind"):
|
||||
await get_timeline(kind="nope")
|
||||
with pytest.raises(ValueError, match="area"):
|
||||
await get_timeline(area="nope")
|
||||
|
||||
asyncio.run(_go())
|
||||
@@ -17,6 +17,7 @@ TOP_LEVEL = {
|
||||
"steward/templates/alerts/list.html",
|
||||
"steward/templates/ansible/index.html",
|
||||
"steward/templates/audit/list.html",
|
||||
"steward/templates/timeline/list.html",
|
||||
"steward/templates/ai_access/index.html",
|
||||
"steward/templates/account/index.html",
|
||||
"plugins/docker/templates/docker/index.html",
|
||||
|
||||
Reference in New Issue
Block a user