FabledSteward/steward/timeline/service.py

"""Reading the timeline, and keeping it: the filtered list the page, the host
card and get_timeline share, the first fill from records that predate it,
its retention, and a host rename carried onto its entries."""
from __future__ import annotations

import logging
from datetime import datetime, timedelta, timezone

from sqlalchemy import delete, func, select, text, update
from sqlalchemy.dialects.postgresql import insert

from . import catalog
from .config import Config, kind_types
from steward.models.timeline import TimelineEntry

logger = logging.getLogger(__name__)

# How far back the first fill reaches: the default retention of the records it reads.
BACKFILL_DAYS = 90


def query(cfg: Config, *, host: str = "", area: str = "", kind: str = "", type_: str = "",
          since: datetime | None = None, include_hidden: bool = False):
    """The entries matching the filters, unordered (core.paging orders them).
    Hidden types are left out unless `include_hidden`, or a `type_` asks for one."""
    stmt = select(TimelineEntry)
    if host:
        stmt = stmt.where(TimelineEntry.hosts.contains([host]))
    if area:
        stmt = stmt.where(cfg.area_filter(area))
    if kind:
        stmt = stmt.where(TimelineEntry.type.in_(kind_types(kind)))
    if type_:
        stmt = stmt.where(TimelineEntry.type == type_)
    elif not include_hidden:
        hidden = cfg.hidden_types()
        if hidden:
            stmt = stmt.where(TimelineEntry.type.notin_(hidden))
    if since is not None:
        stmt = stmt.where(TimelineEntry.at >= since)
    return stmt


def _ref_models() -> dict:
    """The full records an entry can point to: core's, and the ones loaded
    plugins registered (catalog.register_ref)."""
    from steward.models.alerts import AlertEvent
    from steward.models.ansible import AnsibleRun
    from steward.models.audit import AuditEvent
    from steward.models.norms import MetricAnomaly
    return {"alert_event": AlertEvent, "ansible_run": AnsibleRun, "audit_event": AuditEvent,
            "metric_anomaly": MetricAnomaly, **catalog.REFS}


async def kept_refs(db, entries) -> set[tuple[str, str]]:
    """The (ref_type, ref_id) of `entries` whose full record still exists."""
    wanted: dict[str, set[str]] = {}
    for e in entries:
        if e.ref_type and e.ref_id:
            wanted.setdefault(e.ref_type, set()).add(e.ref_id)
    kept = set()
    models = _ref_models()
    for ref_type, ids in wanted.items():
        model = models.get(ref_type)
        if model is None:           # a plugin no longer loaded: assume it is kept
            kept |= {(ref_type, i) for i in ids}
            continue
        found = (await db.execute(select(model.id).where(model.id.in_(ids)))).scalars()
        kept |= {(ref_type, str(i)) for i in found}
    return kept


def view(entry: TimelineEntry, cfg: Config, kept: set) -> dict:
    """One entry as the page and the MCP tool read it."""
    known = catalog.TYPES.get(entry.type)
    area = cfg.area_of(entry)
    detail = entry.detail or {}
    url = detail.get("url")
    gone = bool(entry.ref_type) and (entry.ref_type, entry.ref_id) not in kept
    return {
        "entry": entry,
        "type_label": known.label if known else entry.type,
        "kind": known.kind if known else None,
        "kind_label": catalog.KINDS.get(known.kind) if known else None,
        "area": area,
        "area_label": cfg.area_label(area),
        "note": detail.get("note"),
        "url": None if gone else url,
        "gone": gone,
    }


async def views(db, cfg: Config, entries) -> list[dict]:
    kept = await kept_refs(db, entries)
    return [view(e, cfg, kept) for e in entries]


async def recent_for_host(db, cfg: Config, host: str, limit: int = 8) -> list[dict]:
    """A host's newest shown entries, for its page."""
    rows = (await db.execute(
        query(cfg, host=host).order_by(TimelineEntry.at.desc(), TimelineEntry.id.desc())
        .limit(limit))).scalars().all()
    return await views(db, cfg, rows)


# ── Keeping it ──────────────────────────────────────────────────────────────

async def prune(session, now: datetime) -> tuple[int, int]:
    """Delete entries past timeline.retention_days; (rows, days), with 0 days
    keeping everything. Call inside a transaction."""
    from steward.core.settings import get_setting
    days = int(await get_setting(session, "timeline.retention_days") or 0)
    if days <= 0:
        return 0, 0
    result = await session.execute(
        delete(TimelineEntry).where(TimelineEntry.at < now - timedelta(days=days)))
    return result.rowcount or 0, days


async def carry_host_rename(session, old: str, new: str) -> int:
    """Point a renamed host's entries at its new name, so its page still
    shows its history."""
    if not old or old == new:
        return 0
    result = await session.execute(
        update(TimelineEntry).where(TimelineEntry.hosts.contains([old]))
        .values(hosts=func.array_replace(TimelineEntry.hosts, old, new))
        .execution_options(synchronize_session=False))
    return result.rowcount or 0


async def _core_entries(session, since: datetime) -> list:
    """Entries for the alert events, finished Ansible runs and audit rows
    since `since`, built by the live writers' own builders. One record that
    can't be read is skipped, not the whole fill."""
    from steward.models.alerts import AlertEvent, AlertRule
    from steward.models.ansible import AnsibleRun
    from steward.models.audit import AuditEvent
    from .record import from_alert, from_audit, from_run
    built: list = []

    async def build(what: str, coro) -> None:
        try:
            built.append(await coro)
        except Exception:
            logger.exception("Timeline fill: %s skipped", what)

    for event, rule in (await session.execute(
            select(AlertEvent, AlertRule).join(AlertRule, AlertEvent.rule_id == AlertRule.id)
            .where(AlertEvent.transitioned_at >= since))).all():
        await build(f"alert event {event.id}", from_alert(session, rule, event))
    for run in (await session.execute(
            select(AnsibleRun).where(AnsibleRun.started_at >= since,
                                     AnsibleRun.finished_at.is_not(None)))).scalars().all():
        await build(f"Ansible run {run.id}", from_run(session, run))
    for event in (await session.execute(
            select(AuditEvent).where(AuditEvent.timestamp >= since))).scalars().all():
        await build(f"audit row {event.id}", from_audit(session, event, _audit_detail(event)))
    return built


def fill_sources() -> dict:
    """{source: its `(session, since) -> [entries]`}: core's, and each loaded
    plugin's `timeline_backfill`."""
    from steward.core.plugin_manager import get_loaded_plugin_modules
    sources = {"steward": _core_entries}
    for name, module in sorted(get_loaded_plugin_modules().items()):
        hook = getattr(module, "timeline_backfill", None)
        if callable(hook):
            sources[name] = hook
    return sources


async def backfill(app, now: datetime | None = None, sources: dict | None = None) -> int:
    """Fill the timeline from the records that predate it, BACKFILL_DAYS
    back, once per source: core's, then each loaded plugin's. Which sources
    are done, and when, is kept in timeline.filled, so a plugin loaded later
    fills on the next turn. An entry for a record already covered is
    skipped, so a source filled again adds nothing twice. Returns how many
    entries were added."""
    from steward.core.settings import get_setting, set_setting

    now = now or datetime.now(timezone.utc)
    since = now - timedelta(days=BACKFILL_DAYS)
    sources = fill_sources() if sources is None else sources
    added = 0
    async with app.db_sessionmaker() as session:
        async with session.begin():
            filled = dict(await get_setting(session, "timeline.filled") or {})
        # The builders add each entry to the session; no flush may write
        # them before the insert below, which skips any already covered.
        session.sync_session.autoflush = False
        for name, source in sources.items():
            if name in filled:
                continue
            try:
                async with session.begin():
                    rows = [e for e in await source(session, since) if e is not None]
                    for entry in rows:
                        session.expunge(entry)
                    for chunk in range(0, len(rows), 500):
                        result = await session.execute(
                            insert(TimelineEntry).values([_row(e) for e in rows[chunk:chunk + 500]])
                            .on_conflict_do_nothing(index_elements=["ref_type", "ref_id"],
                                                    index_where=text("ref_id IS NOT NULL")))
                        added += result.rowcount or 0
                    filled[name] = now.isoformat()
                    await set_setting(session, "timeline.filled", filled)
                logger.info("Timeline: filled from %s, the last %d days", name, BACKFILL_DAYS)
            except Exception:
                logger.exception("Timeline: the fill from %s failed; it is tried again tomorrow", name)
    return added


def _audit_detail(event) -> dict:
    import json
    try:
        return json.loads(event.detail_json) if event.detail_json else {}
    except (TypeError, ValueError):
        return {}


def _row(entry: TimelineEntry) -> dict:
    from steward.core.ids import time_ordered_id
    return {"id": entry.id or time_ordered_id(), "at": entry.at, "until": entry.until,
            "type": entry.type, "area": entry.area, "hosts": entry.hosts or [],
            "subject": entry.subject, "summary": entry.summary, "detail": entry.detail,
            "actor": entry.actor or "system", "ref_type": entry.ref_type, "ref_id": entry.ref_id}