bvandeusenandClaude Opus 5.5
4ea735d87c
A stored log, timeline_entries, with one row per thing that happened. Each source writes its row in the same transaction as its own record, so the history outlives the 30/90/365-day retention of the tables it points to: - alert events (core/alerts._add_event): fired, resolved, acknowledged, went silent, readings back; - finished Ansible runs (executor, and the startup sweep of interrupted runs): changed something, changed nothing, failed, dry run; - audit rows (core/audit.log_audit): Steward's settings, users and access, sign-ins, backups. Actions another writer records are left to it. Entries carry a kind (fixed) and an area (Service, Host OS, Hardware, Storage, Network, Security, Monitoring). Settings → Timeline edits the areas and each type's area and visibility; both are applied when the timeline is read, so a change re-sorts past entries and a hidden type comes back when shown. Kept forever unless a retention is set. The first start fills the timeline from the last 90 days of records once, skipping any already covered. A host rename carries its entries. UI: Timeline in the sidebar (filter by host, area, kind; scrolls on), a History card on each host page. MCP: get_timeline. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
271 lines
13 KiB
Python
271 lines
13 KiB
Python
"""The timeline: what each source becomes, how Settings → Timeline sorts and
|
|
hides it when it is read, and the pages that show it."""
|
|
from datetime import datetime, timezone
|
|
from types import SimpleNamespace as NS
|
|
|
|
import pytest
|
|
from quart import render_template
|
|
from werkzeug.datastructures import MultiDict
|
|
|
|
from steward.models.alerts import AlertOperator
|
|
from steward.timeline import catalog, record, service
|
|
from steward.timeline.config import Config, edited
|
|
|
|
NOW = datetime(2026, 10, 24, 12, 0, tzinfo=timezone.utc)
|
|
AREAS = [list(a) for a in catalog.DEFAULT_AREAS]
|
|
|
|
|
|
def _cfg(**kw) -> Config:
|
|
return Config(areas=[list(a) for a in kw.pop("areas", AREAS)], **kw)
|
|
|
|
|
|
def _entry(type_="alert.fired", area="hardware", **kw):
|
|
base = dict(id="e1", at=NOW, until=None, type=type_, area=area, hosts=["web-1"],
|
|
subject="web-1:sda", summary="Disk hot fired", detail={}, actor="system",
|
|
ref_type=None, ref_id=None)
|
|
return NS(**{**base, **kw})
|
|
|
|
|
|
# ── What a source becomes ───────────────────────────────────────────────────
|
|
|
|
@pytest.mark.parametrize("action,type_", [
|
|
("monitor.disabled", "steward.config"),
|
|
("settings.saved", "steward.config"),
|
|
("host_agent.maintenance_plan_created", "steward.config"),
|
|
("user.role_changed", "steward.access"),
|
|
("api_token.create", "steward.access"),
|
|
("account.password_changed", "steward.access"),
|
|
("auth.login", "steward.signin"),
|
|
("auth.login_failed", "steward.signin"),
|
|
("backup.made", "steward.backup"),
|
|
# Another writer records the event itself, or it is one person's preference.
|
|
("ansible.run.started", None),
|
|
("ansible.host_run", None),
|
|
("alert_rule.acknowledged", None),
|
|
("host_agent.maintenance", None),
|
|
("docker.prune.started", None),
|
|
("account.time_zone", None),
|
|
])
|
|
def test_each_audit_action_is_entered_as_its_type_or_left_out(action, type_):
|
|
assert catalog.audit_type(action) == type_
|
|
|
|
|
|
@pytest.mark.parametrize("source,metric,area", [
|
|
("host_agent", "temp_c_max", "hardware"),
|
|
("host_agent", "drive_failing", "hardware"),
|
|
("host_agent", "disk_used_pct", "storage"),
|
|
("host_agent", "mounts_stalled", "storage"),
|
|
("host_agent", "nfs_rtt_ms", "storage"),
|
|
("host_agent", "net_rx_bps", "network"),
|
|
("host_agent", "updates_security", "security"),
|
|
("host_agent", "cpu_pct", "host_os"),
|
|
("host_agent", "updates_pending", "host_os"),
|
|
("http", "is_up", "service"),
|
|
("icmp", "is_up", "network"),
|
|
("docker", "mem_used_pct", "service"),
|
|
("traefik", "error_rate_5xx_pct", "network"),
|
|
("steward", "db_roundtrip_ms", "monitoring"),
|
|
("snmp", "volume_used_pct", "storage"),
|
|
("snmp", "ups_battery_pct", "hardware"),
|
|
])
|
|
def test_an_alert_is_written_under_the_area_its_rule_watches(source, metric, area):
|
|
assert catalog.area_of_metric(source, metric) == area
|
|
|
|
|
|
@pytest.mark.parametrize("from_state,to_state,type_", [
|
|
("pending", "firing", "alert.fired"),
|
|
("no_data", "firing", "alert.fired"),
|
|
("firing", "resolved", "alert.resolved"),
|
|
("firing", "acknowledged", "alert.acknowledged"),
|
|
("inactive", "no_data", "alert.silent"),
|
|
("no_data", "inactive", "alert.resumed"),
|
|
("inactive", "pending", None),
|
|
])
|
|
def test_an_alert_event_becomes_one_entry_type(from_state, to_state, type_):
|
|
assert record._alert_type(from_state, to_state) == type_
|
|
|
|
|
|
def test_an_alert_entry_reads_as_a_sentence():
|
|
rule = NS(name="High CPU", metric_name="cpu_pct", resource_name="web-1",
|
|
operator=AlertOperator.gt, threshold=90.0)
|
|
assert record.alert_summary("alert.fired", rule, 94.0) == "High CPU fired: CPU on web-1 is 94% (> 90%)"
|
|
assert record.alert_summary("alert.resolved", rule, 40.0) == "High CPU resolved: CPU on web-1 is 40%"
|
|
assert record.alert_summary("alert.resolved", rule, None) == "High CPU resolved"
|
|
assert record.alert_summary("alert.silent", rule, None) == "High CPU: no CPU reading from web-1"
|
|
up = NS(name="API up", metric_name="is_up", resource_name="api", operator=AlertOperator.lt, threshold=1.0)
|
|
assert record.alert_summary("alert.fired", up, 0.0) == "API up fired: api is down"
|
|
|
|
|
|
def test_an_entry_takes_its_type_s_area_unless_the_writer_names_one():
|
|
added = []
|
|
session = NS(add=added.append)
|
|
e = record.record(session, type="ansible.changed", at=NOW, summary="x", hosts=["b", "a", "a", ""])
|
|
assert e.area == "host_os" and e.hosts == ["a", "b"] and added == [e]
|
|
assert record.record(session, type="ansible.changed", at=NOW, summary="x", area="service").area == "service"
|
|
assert record.record(session, type="plugin.gone", at=NOW, summary="x").area == "other"
|
|
|
|
|
|
# ── Settings → Timeline, applied when read ──────────────────────────────────
|
|
|
|
def test_an_entry_sorts_under_its_own_area_until_its_type_is_given_one():
|
|
e = _entry(area="hardware")
|
|
assert _cfg().area_of(e) == "hardware"
|
|
assert _cfg(types={"alert.fired": {"area": "monitoring"}}).area_of(e) == "monitoring"
|
|
|
|
|
|
def test_a_removed_area_s_entries_follow_where_it_was_moved():
|
|
cfg = _cfg(areas=[a for a in AREAS if a[0] not in ("hardware", "storage")],
|
|
moved={"hardware": "storage", "storage": "host_os"})
|
|
assert cfg.area_of(_entry(area="hardware")) == "host_os"
|
|
# An area nobody lists any more, and a loop, end in Other rather than hang.
|
|
assert cfg.area_of(_entry(area="from_a_plugin")) == "other"
|
|
assert _cfg(areas=AREAS[:1], moved={"a": "b", "b": "a"}).follow("a") == "other"
|
|
assert cfg.area_label("other") == "Other"
|
|
|
|
|
|
def test_a_type_is_shown_as_the_catalog_says_until_settings_say_otherwise():
|
|
cfg = _cfg()
|
|
assert cfg.shown("alert.fired") and not cfg.shown("ansible.check")
|
|
assert {"ansible.check", "ansible.unchanged", "steward.signin"} <= cfg.hidden_types()
|
|
cfg = _cfg(types={"ansible.check": {"shown": True}, "alert.fired": {"shown": False}})
|
|
assert cfg.shown("ansible.check") and not cfg.shown("alert.fired")
|
|
assert "alert.fired" in cfg.hidden_types() and "ansible.check" not in cfg.hidden_types()
|
|
|
|
|
|
def _form(cfg: Config, **extra) -> MultiDict:
|
|
"""The form as the page posts it unchanged: every type at its current
|
|
visibility, every area kept, then `extra`."""
|
|
data = {f"type_shown_{k}": "on" for k in catalog.TYPES if cfg.shown(k)}
|
|
data.update({f"area_label_{k}": label for k, label in cfg.areas})
|
|
data.update(extra)
|
|
return MultiDict({k: v for k, v in data.items() if v is not None})
|
|
|
|
|
|
def test_saving_the_page_unchanged_stores_no_overrides():
|
|
cfg = _cfg()
|
|
new, error = edited(cfg, _form(cfg))
|
|
assert error is None and new.types == {} and new.moved == {} and new.areas == AREAS
|
|
|
|
|
|
def test_an_area_is_renamed_added_and_removed_into_another():
|
|
cfg = _cfg(types={"alert.fired": {"area": "hardware"}})
|
|
new, error = edited(cfg, _form(cfg, area_label_network="Networking", new_area="Power & UPS",
|
|
area_move_hardware="storage",
|
|
**{"type_area_alert.fired": "hardware"}))
|
|
assert error is None
|
|
assert dict(new.areas)["network"] == "Networking"
|
|
assert new.areas[-1] == ["power_ups", "Power & UPS"]
|
|
assert "hardware" not in dict(new.areas) and new.moved == {"hardware": "storage"}
|
|
# A type sorted under the removed area follows its entries.
|
|
assert new.types["alert.fired"] == {"area": "storage"}
|
|
|
|
|
|
def test_a_type_is_hidden_or_given_an_area():
|
|
cfg = _cfg()
|
|
form = _form(cfg, **{"type_area_ansible.changed": "service"})
|
|
del form["type_shown_alert.resumed"]
|
|
new, error = edited(cfg, form)
|
|
assert error is None
|
|
assert new.types == {"ansible.changed": {"area": "service"}, "alert.resumed": {"shown": False}}
|
|
|
|
|
|
@pytest.mark.parametrize("extra,said", [
|
|
({"area_move_network": "network"}, "Pick another area"),
|
|
({"area_move_network": "nowhere"}, "Pick another area"),
|
|
({"new_area": "Storage"}, "already an area"),
|
|
({"new_area": "Other"}, "already an area"),
|
|
({"type_area_alert.fired": "nowhere"}, "isn't listed"),
|
|
])
|
|
def test_a_form_that_can_t_be_saved_says_why_and_changes_nothing(extra, said):
|
|
cfg = _cfg()
|
|
new, error = edited(cfg, _form(cfg, **extra))
|
|
assert said in error and new is cfg
|
|
|
|
|
|
def test_the_last_area_can_t_be_removed():
|
|
cfg = _cfg(areas=[["service", "Service"], ["network", "Network"]])
|
|
form = _form(cfg, area_move_service="network", area_move_network="service")
|
|
assert edited(cfg, form)[1] == "Keep at least one area."
|
|
|
|
|
|
def test_a_plugin_s_type_keeps_its_settings_while_the_plugin_is_away():
|
|
cfg = _cfg(types={"unifi.client_joined": {"shown": False}})
|
|
new, _ = edited(cfg, _form(cfg))
|
|
assert new.types == {"unifi.client_joined": {"shown": False}}
|
|
|
|
|
|
# ── Read back ───────────────────────────────────────────────────────────────
|
|
|
|
def test_an_entry_whose_record_aged_out_says_so_instead_of_linking():
|
|
e = _entry(ref_type="ansible_run", ref_id="r1", detail={"url": "/ansible/runs/r1", "note": "n"})
|
|
kept = service.view(e, _cfg(), {("ansible_run", "r1")})
|
|
gone = service.view(e, _cfg(), set())
|
|
assert kept["url"] == "/ansible/runs/r1" and not kept["gone"] and kept["note"] == "n"
|
|
assert gone["url"] is None and gone["gone"]
|
|
assert kept["kind_label"] == "Alert" and kept["area_label"] == "Hardware"
|
|
# An entry with no record behind it (a hand-written one) is never "gone".
|
|
assert not service.view(_entry(), _cfg(), set())["gone"]
|
|
|
|
|
|
# ── Pages ───────────────────────────────────────────────────────────────────
|
|
|
|
def _item(e, **kw):
|
|
v = service.view(e, _cfg(), {(e.ref_type, e.ref_id)})
|
|
v.update(group="Today", opens_group=True, **kw)
|
|
return v
|
|
|
|
|
|
async def test_the_page_lists_what_happened_where_and_by_whom(app):
|
|
items = [_item(_entry(ref_type="alert_event", ref_id="a1", detail={"url": "/alerts/history?rule=r"})),
|
|
_item(_entry(type_="steward.config", area="monitoring", hosts=[], subject=None,
|
|
summary="Paused monitor api", actor="bryan via MCP"), opens_group=False)]
|
|
async with app.test_request_context("/timeline/"):
|
|
page = await render_template("timeline/list.html", items=items, after=None, last_group="Today",
|
|
filters={"host": "", "area": "", "kind": "", "hidden": False},
|
|
keep={}, areas=AREAS, kinds=catalog.KINDS, hosts=["web-1"])
|
|
assert "Disk hot fired" in page and "Paused monitor api" in page
|
|
assert '<a href="/timeline/?host=web-1">web-1</a>' in page
|
|
assert '<a href="/alerts/history?rule=r">Details</a>' in page
|
|
assert "bryan via MCP" in page and ">Hardware<" in page and "Alert · " in page
|
|
assert page.count('class="table-divider"') == 1
|
|
assert "End of the list." in page
|
|
|
|
|
|
async def test_the_next_page_keeps_the_filters(app):
|
|
items = [_item(_entry())]
|
|
async with app.test_request_context("/timeline/"):
|
|
rows = await render_template("timeline/_rows.html", items=items, last_group="Today",
|
|
after="2026-10-24T12:00:00+00:00|e1", keep={"host": "web-1"})
|
|
assert 'hx-get="/timeline/?before=2026-10-24T12%3A00%3A00%2B00%3A00%7Ce1&host=web-1&group=Today"' in rows
|
|
|
|
|
|
async def test_an_empty_timeline_says_so(app):
|
|
async with app.test_request_context("/timeline/"):
|
|
page = await render_template("timeline/list.html", items=[], after=None, last_group="",
|
|
filters={"host": "x", "area": "", "kind": "", "hidden": False},
|
|
keep={"host": "x"}, areas=AREAS, kinds=catalog.KINDS, hosts=[])
|
|
assert "Nothing on the timeline matches these filters." in page
|
|
|
|
|
|
async def test_a_host_s_card_shows_its_latest_and_links_the_rest(app):
|
|
host = NS(id="h1", name="web-1")
|
|
async with app.test_request_context("/timeline/host/h1"):
|
|
card = await render_template("timeline/_host_card.html", host=host,
|
|
items=[_item(_entry(ref_type="ansible_run", ref_id="gone"))])
|
|
empty = await render_template("timeline/_host_card.html", host=host, items=[])
|
|
assert "Disk hot fired" in card and 'href="/timeline/?host=web-1"' in card
|
|
assert "Nothing has happened to web-1" in empty
|
|
|
|
|
|
async def test_settings_lists_each_type_under_its_kind(app):
|
|
from steward.timeline.routes import _settings_ctx
|
|
cfg = _cfg(types={"alert.fired": {"area": "monitoring"}})
|
|
async with app.test_request_context("/settings/timeline/"):
|
|
page = await render_template("settings/timeline.html", **_settings_ctx(cfg, 0))
|
|
for t in catalog.TYPES.values():
|
|
assert t.label in page
|
|
assert 'name="type_shown_alert.fired" aria-label="Show Alert fired" checked' in page
|
|
assert 'name="type_shown_ansible.check" aria-label="Show Playbook dry run">' in page
|
|
assert '<option value="monitoring" selected>Monitoring</option>' in page
|
|
assert 'aria-current="page">Timeline</a>' in page
|