feat(timeline): one record of what happened to the infrastructure, newest first (#6348)
A stored log, timeline_entries, with one row per thing that happened. Each source writes its row in the same transaction as its own record, so the history outlives the 30/90/365-day retention of the tables it points to: - alert events (core/alerts._add_event): fired, resolved, acknowledged, went silent, readings back; - finished Ansible runs (executor, and the startup sweep of interrupted runs): changed something, changed nothing, failed, dry run; - audit rows (core/audit.log_audit): Steward's settings, users and access, sign-ins, backups. Actions another writer records are left to it. Entries carry a kind (fixed) and an area (Service, Host OS, Hardware, Storage, Network, Security, Monitoring). Settings → Timeline edits the areas and each type's area and visibility; both are applied when the timeline is read, so a change re-sorts past entries and a hidden type comes back when shown. Kept forever unless a retention is set. The first start fills the timeline from the last 90 days of records once, skipping any already covered. A host rename carries its entries. UI: Timeline in the sidebar (filter by host, area, kind; scrolls on), a History card on each host page. MCP: get_timeline. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+22
-10
@@ -571,11 +571,12 @@ async def _evaluate_rule(
|
||||
state.acknowledged_at = None
|
||||
said = "; ".join(n for n in (resumed, note) if n) or None
|
||||
if step.event:
|
||||
event_id = _add_event(session, rule.id, "no_data" if resumed else from_state.value,
|
||||
step.event, value, now, note=said)
|
||||
event_id = await _add_event(session, rule, "no_data" if resumed else from_state.value,
|
||||
step.event, value, now, note=said)
|
||||
notifications.append((step.event, event_id, None))
|
||||
elif resumed:
|
||||
event_id = _add_event(session, rule.id, "no_data", state.state.value, value, now, note=said)
|
||||
event_id = await _add_event(session, rule, "no_data", state.state.value, value, now, note=said,
|
||||
resumed=True)
|
||||
notifications.append(("data_resumed", event_id, resumed))
|
||||
await session.flush()
|
||||
return notifications
|
||||
@@ -595,8 +596,8 @@ async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str,
|
||||
return
|
||||
now = datetime.now(timezone.utc)
|
||||
if state.state in (AlertStateEnum.firing, AlertStateEnum.acknowledged):
|
||||
_add_event(session, rule.id, state.state.value, "resolved", None, now, note=note,
|
||||
actor_id=actor_id)
|
||||
await _add_event(session, rule, state.state.value, "resolved", None, now, note=note,
|
||||
actor_id=actor_id)
|
||||
# The watch starts over now: silence is measured from here, against the
|
||||
# cadence of whatever the rule watches from now on.
|
||||
state.last_transition_at = now
|
||||
@@ -610,21 +611,27 @@ async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str,
|
||||
state.acknowledged_at = None
|
||||
|
||||
|
||||
def _add_event(
|
||||
async def _add_event(
|
||||
session: AsyncSession,
|
||||
rule_id: str,
|
||||
rule: AlertRule,
|
||||
from_state: str,
|
||||
to_state: str,
|
||||
value: float | None,
|
||||
now: datetime,
|
||||
note: str | None = None,
|
||||
actor_id: str | None = None,
|
||||
actor: str | None = None,
|
||||
resumed: bool = False,
|
||||
) -> str:
|
||||
"""Add an AlertEvent row and return its ID."""
|
||||
"""Add an AlertEvent row, and its timeline entry, and return its ID.
|
||||
`actor` names who did it on the timeline when `actor_id`'s username
|
||||
alone would not say it ("<user> via MCP"). `resumed` marks a series
|
||||
back from silence that left the alert's state as it was."""
|
||||
from steward.timeline.record import from_alert
|
||||
event_id = str(uuid.uuid4())
|
||||
event = AlertEvent(
|
||||
id=event_id,
|
||||
rule_id=rule_id,
|
||||
rule_id=rule.id,
|
||||
from_state=from_state,
|
||||
to_state=to_state,
|
||||
metric_value=value,
|
||||
@@ -633,6 +640,11 @@ def _add_event(
|
||||
actor_id=actor_id,
|
||||
)
|
||||
session.add(event)
|
||||
try:
|
||||
await from_alert(session, rule, event, actor, resumed=resumed)
|
||||
except Exception:
|
||||
# The alert's own history matters more than its timeline entry.
|
||||
logger.exception("Alert event %s: its timeline entry could not be written", event_id)
|
||||
return event_id
|
||||
|
||||
|
||||
@@ -825,7 +837,7 @@ async def check_silence(app: "Quart") -> None:
|
||||
muted = _muted_by(windows, rule)
|
||||
# Silent with its agent: recorded, and "agent offline" is the email (#5721).
|
||||
hushed = AGENT_OFFLINE if host_behind(rule) in offline else None
|
||||
event_id = _add_event(session, rule.id, state.state.value, "no_data", None, now,
|
||||
event_id = await _add_event(session, rule, state.state.value, "no_data", None, now,
|
||||
note="; ".join(n for n in (note, hushed, muted) if n))
|
||||
if rule.notify_no_data and not muted and not hushed:
|
||||
notify.append((rule, event_id, note))
|
||||
|
||||
Reference in New Issue
Block a user