FabledSteward/steward/core/alerts.py

from __future__ import annotations
import asyncio
import logging
import uuid
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING

from sqlalchemy import event as sa_event, select
from sqlalchemy.ext.asyncio import AsyncSession

from steward.models.alerts import (
    AlertEvent,
    AlertOperator,
    AlertRule,
    AlertState,
    AlertStateEnum,
    MaintenanceWindow,
)
from steward.core.formatting import human_duration
from steward.models.metrics import PluginMetric

if TYPE_CHECKING:
    from quart import Quart

logger = logging.getLogger(__name__)

_app: "Quart | None" = None


def init_alerts(app: "Quart") -> None:
    """Store app reference for use in notification dispatch tasks."""
    global _app
    _app = app


def alert_extra_vars(rule: AlertRule, value: float) -> list[str]:
    """Ansible extra-vars describing the alert that fired, as `key=value` strings.

    Injected into every alert-triggered playbook run so the playbook can target
    the exact resource/metric that fired without templating. Pure function.
    """
    return [
        f"steward_alert_rule={rule.name}",
        f"steward_alert_source={rule.source_module}",
        f"steward_alert_resource={rule.resource_name}",
        f"steward_alert_metric={rule.metric_name}",
        f"steward_alert_value={value}",
        f"steward_alert_threshold={rule.threshold}",
        f"steward_alert_operator={rule.operator.value}",
    ]


async def record_metric(
    session: AsyncSession,
    source_module: str,
    resource_name: str,
    metric_name: str,
    value: float,
) -> None:
    """One metric sample: record_metrics with a single row."""
    await record_metrics(session, [PluginMetric(
        source_module=source_module, resource_name=resource_name,
        metric_name=metric_name, value=value)])


class MetricBatch:
    """Gathers one transaction's samples so they are recorded, and their alert
    rules evaluated, in a single record_metrics call:

        batch = MetricBatch()
        batch.add(source_module="unifi", resource_name="wan", metric_name="is_up", value=1.0)
        await batch.record(session)
    """

    def __init__(self) -> None:
        self.rows: list[PluginMetric] = []

    def add(self, source_module: str, resource_name: str, metric_name: str, value: float,
            recorded_at: datetime | None = None) -> None:
        self.rows.append(PluginMetric(source_module=source_module, resource_name=resource_name,
                                      metric_name=metric_name, value=float(value),
                                      recorded_at=recorded_at))

    async def record(self, session: AsyncSession) -> None:
        rows, self.rows = self.rows, []
        await record_metrics(session, rows)


def _in_window(w: MaintenanceWindow, source_module: str, resource_name: str) -> bool:
    if w.scope_source is None:
        return True
    return w.scope_source == source_module and w.scope_resource in (None, resource_name)


MUTED = "notification muted: maintenance window"
RELEASED = "notified when the window ended"


def _muted_by(windows, rule: AlertRule) -> str | None:
    """The note an event carries when a maintenance window holds back its
    notification, or None when no window covers the rule."""
    w = next((w for w in windows if _in_window(w, rule.source_module, rule.resource_name)), None)
    return f"{MUTED} {w.name}" if w else None


# Notification and action tasks in flight. Held here so the event loop's weak
# references are not the only ones (a dropped task is silently never run).
_background: set[asyncio.Task] = set()

_AFTER_COMMIT = "steward_alert_after_commit"


def _spawn(coro) -> None:
    task = asyncio.create_task(coro)
    _background.add(task)
    task.add_done_callback(_background.discard)


def _after_commit(session: AsyncSession, fn, *args) -> None:
    """Run fn(app, *args) once the session's transaction commits, and never if
    it rolls back: a notification or a playbook run must describe what was
    actually stored."""
    if _app is None:
        return
    info = session.info
    if _AFTER_COMMIT not in info:
        info[_AFTER_COMMIT] = []
        app = _app

        def _committed(_sync_session) -> None:
            jobs, info[_AFTER_COMMIT] = info[_AFTER_COMMIT], []
            for job_fn, job_args in jobs:
                _spawn(job_fn(app, *job_args))

        def _rolled_back(_sync_session) -> None:
            info[_AFTER_COMMIT] = []

        sa_event.listen(session.sync_session, "after_commit", _committed)
        sa_event.listen(session.sync_session, "after_rollback", _rolled_back)
    info[_AFTER_COMMIT].append((fn, args))


async def record_metrics(session: AsyncSession, rows: list[PluginMetric]) -> None:
    """Write metric rows and evaluate the alert rules that watch them.

    Every ingest path comes through here, so a rule on any metric Steward
    records can fire. Must be called inside an active transaction:
        async with session.begin():
            await record_metrics(session, rows)

    Maintenance windows, rules and their states are read once per batch, not
    once per row. Each rule sees its key's samples oldest first, so a backlog
    (an agent catching up after an outage) counts consecutive checks as they
    happened. A maintenance window holds back notifications and actions, not
    evaluation: the state still moves, and the event says it was muted.
    Notifications and actions start only once the transaction commits.
    """
    if not rows:
        return
    now = datetime.now(timezone.utc)
    for row in rows:
        if row.recorded_at is None:
            row.recorded_at = now
    session.add_all(rows)
    await session.flush()

    by_key: dict[tuple[str, str, str], list[PluginMetric]] = {}
    for row in rows:
        by_key.setdefault((row.source_module, row.resource_name, row.metric_name), []).append(row)
    sources = {k[0] for k in by_key}
    metric_names = {k[2] for k in by_key}

    rules = (await session.execute(
        select(AlertRule).where(
            AlertRule.source_module.in_(sources),
            AlertRule.metric_name.in_(metric_names),
            AlertRule.enabled.is_(True),
        )
    )).scalars().all()
    rules = [r for r in rules if (r.source_module, r.resource_name, r.metric_name) in by_key]
    if not rules:
        return

    windows = (await session.execute(
        select(MaintenanceWindow).where(
            MaintenanceWindow.start_at <= now, MaintenanceWindow.end_at >= now)
    )).scalars().all()
    states = {st.rule_id: st for st in (await session.execute(
        select(AlertState).where(AlertState.rule_id.in_([r.id for r in rules]))
    )).scalars()}

    for rule in rules:
        alert_state = states.get(rule.id)
        if alert_state is None:
            logger.warning("No alert_state row for rule %s, skipping", rule.id)
            continue
        samples = sorted(by_key[(rule.source_module, rule.resource_name, rule.metric_name)],
                         key=lambda r: r.recorded_at)
        muted = _muted_by(windows, rule)
        resumed = _track_samples(session, rule, alert_state, samples, now, muted)
        if resumed and rule.notify_no_data and not muted:
            event_id, value, note = resumed
            _after_commit(session, _dispatch_notification, "data_resumed", rule, value,
                          event_id, note)
        for sample in samples:
            notifications = await _evaluate_rule(session, rule, alert_state, sample.value,
                                                 sample.recorded_at, note=muted)
            if muted:
                continue
            for notif_type, event_id in notifications:
                _after_commit(session, _dispatch_notification, notif_type, rule, sample.value,
                              event_id)
                # Admin-configured Ansible action runs only on transition-to-firing.
                if notif_type == "firing" and rule.ansible_action:
                    _after_commit(session, _run_ansible_action, rule, sample.value, event_id)


# A series is silent once it has gone this long without a sample, or three of
# its usual gaps if that is longer. Until its cadence is known (fewer than two
# samples seen), it gets an hour: a monitor may check only every ten minutes.
NO_DATA_FLOOR_S = 300.0
NO_DATA_GAPS = 3
NO_DATA_UNKNOWN_S = 3600.0


def next_interval(interval: float | None, gap: float) -> float:
    """A series' usual gap between samples, updated with one more gap.

    A smoothed average, so one late sample barely moves it. A gap longer than
    the silence threshold counts only as that threshold: an outage is not the
    series' cadence. A real change of cadence still gets learned, a step at a
    time (each step at most 1.5x).
    """
    gap = max(0.0, gap)
    if interval is None or interval <= 0:
        return gap
    return 0.75 * interval + 0.25 * min(gap, NO_DATA_GAPS * interval)


def no_data_after(interval: float | None) -> float:
    """Seconds without a sample before a series counts as silent."""
    if not interval:
        return NO_DATA_UNKNOWN_S
    return max(NO_DATA_FLOOR_S, NO_DATA_GAPS * interval)


def silent_since(last_sample_at: datetime | None, interval: float | None,
                 watching_since: datetime, now: datetime) -> datetime | None:
    """When a series went silent, or None while it is still reporting.

    A series that has never had a sample counts from when the rule started
    watching it (created, changed or re-enabled).
    """
    last = last_sample_at or watching_since
    limit = timedelta(seconds=no_data_after(interval))
    return last + limit if now - last > limit else None


def _track_samples(session: AsyncSession, rule: AlertRule, state: AlertState,
                   samples: list[PluginMetric], now: datetime, muted: str | None = None
                   ) -> tuple[str, float, str] | None:
    """Note a rule's new samples (oldest first) for silence tracking. When the
    series had been marked no_data, clear it and record a resume event,
    returned as (event_id, value, note) for the caller to notify. `muted` is
    added to the event's note when a maintenance window holds it back."""
    for sample in samples:
        if state.last_sample_at is not None:
            if sample.recorded_at <= state.last_sample_at:
                continue  # late or duplicate: not a gap
            gap = (sample.recorded_at - state.last_sample_at).total_seconds()
            state.sample_interval_s = next_interval(state.sample_interval_s, gap)
        state.last_sample_at = sample.recorded_at
    if state.no_data_since is None or not samples:
        return None
    note = f"data resumed after {human_duration((now - state.no_data_since).total_seconds())} silent"
    value = samples[-1].value
    event_id = _add_event(session, rule.id, "no_data", state.state.value, value, now,
                          note=f"{note}; {muted}" if muted else note)
    state.no_data_since = None
    return event_id, value, note


def _is_breached(value: float, operator: AlertOperator, threshold: float) -> bool:
    op = operator.value
    if op == ">":
        return value > threshold
    if op == "<":
        return value < threshold
    if op == ">=":
        return value >= threshold
    if op == "<=":
        return value <= threshold
    if op == "==":
        return value == threshold
    if op == "!=":
        return value != threshold
    return False


@dataclass(frozen=True)
class Step:
    """One sample's effect on an alert: the state after it, the two run
    counters, and the event to record ("firing" / "resolved") if any."""
    state: AlertStateEnum
    fail_count: int
    ok_count: int
    event: str | None = None


def next_step(state: AlertStateEnum, fail_count: int, ok_count: int, breached: bool,
              fail_required: int, ok_required: int) -> Step:
    """The alert state machine, as a pure function of one sample.

    inactive → pending → firing when `fail_required` breaching checks run in a
    row; a non-breaching check while pending goes back to inactive.
    firing and acknowledged resolve (to inactive, with a resolved event) only
    after `ok_required` non-breaching checks in a row; a breaching check
    resets that run. An acknowledgement holds while the alert is still
    breached: only a resolve clears it.
    """
    fail_required, ok_required = max(1, fail_required), max(1, ok_required)
    S = AlertStateEnum
    if state in (S.firing, S.acknowledged):
        if breached:
            return Step(state, fail_count + 1, 0)
        if ok_count + 1 >= ok_required:
            return Step(S.inactive, 0, 0, "resolved")
        return Step(state, fail_count, ok_count + 1)
    # inactive, pending (and the transient resolved, read as inactive)
    if not breached:
        return Step(S.inactive, 0, 0)
    fails = fail_count + 1 if state == S.pending else 1
    if fails >= fail_required:
        return Step(S.firing, fails, 0, "firing")
    return Step(S.pending, fails, 0)


async def _evaluate_rule(
    session: AsyncSession,
    rule: AlertRule,
    state: AlertState,
    value: float,
    now: datetime,
    note: str | None = None,
) -> list[tuple[str, str]]:
    """Apply one sample to a rule's state. Returns [(notif_type, event_id)];
    an event it records carries `note`."""
    from_state = state.state
    step = next_step(from_state, state.consecutive_failure_count, state.consecutive_ok_count,
                     _is_breached(value, rule.operator, rule.threshold),
                     rule.consecutive_failures_required, rule.consecutive_ok_required)
    state.consecutive_failure_count = step.fail_count
    state.consecutive_ok_count = step.ok_count
    notifications: list[tuple[str, str]] = []
    if step.state != from_state:
        state.state = step.state
        state.last_transition_at = now
        if step.state == AlertStateEnum.inactive:
            state.acknowledged_by = None
            state.acknowledged_at = None
    if step.event:
        event_id = _add_event(session, rule.id, from_state.value, step.event, value, now,
                              note=note)
        notifications.append((step.event, event_id))
    await session.flush()
    return notifications


async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str) -> None:
    """Put a rule's alert back to inactive because the rule itself changed
    (what it watches, its threshold, or it was disabled or re-enabled), and
    start its silence watch over. An alert that was
    firing or acknowledged gets a resolved event carrying `note`; nobody is
    notified, since nothing about the watched resource changed."""
    state = (await session.execute(
        select(AlertState).where(AlertState.rule_id == rule.id))).scalar_one_or_none()
    if state is None:
        return
    now = datetime.now(timezone.utc)
    if state.state in (AlertStateEnum.firing, AlertStateEnum.acknowledged):
        _add_event(session, rule.id, state.state.value, "resolved", None, now, note=note)
    # The watch starts over now: silence is measured from here, against the
    # cadence of whatever the rule watches from now on.
    state.last_transition_at = now
    state.last_sample_at = None
    state.sample_interval_s = None
    state.no_data_since = None
    state.state = AlertStateEnum.inactive
    state.consecutive_failure_count = 0
    state.consecutive_ok_count = 0
    state.acknowledged_by = None
    state.acknowledged_at = None


def _add_event(
    session: AsyncSession,
    rule_id: str,
    from_state: str,
    to_state: str,
    value: float | None,
    now: datetime,
    note: str | None = None,
) -> str:
    """Add an AlertEvent row and return its ID."""
    event_id = str(uuid.uuid4())
    event = AlertEvent(
        id=event_id,
        rule_id=rule_id,
        from_state=from_state,
        to_state=to_state,
        metric_value=value,
        transitioned_at=now,
        note=note,
    )
    session.add(event)
    return event_id


async def _dispatch_notification(
    app: "Quart",
    notif_type: str,
    rule: AlertRule,
    value: float | None,
    event_id: str,
    note: str | None = None,
) -> None:
    """Run after the transaction commits. Sends notifications and updates the event row."""
    from steward.core.notifications import dispatch_notifications

    alert_data = {
        "rule_name": rule.name,
        "state": notif_type.upper(),
        "metric": rule.metric_name,
        "value": value,
        "threshold": rule.threshold,
        "resource": rule.resource_name,
        "source_module": rule.source_module,
        "note": note,
        "timestamp": datetime.now(timezone.utc).isoformat(),
    }

    smtp_cfg = app.config.get("SMTP", {})
    webhook_cfg = app.config.get("WEBHOOK", {})
    results = await dispatch_notifications(smtp_cfg, webhook_cfg, alert_data)

    # Update event row with delivery results in a new transaction
    async with app.db_sessionmaker() as db:
        async with db.begin():
            event_result = await db.execute(
                select(AlertEvent).where(AlertEvent.id == event_id)
            )
            event = event_result.scalar_one_or_none()
            if event:
                email_r = results.get("email")
                if email_r:
                    event.email_sent = email_r["sent"]
                    event.email_error = email_r.get("error")
                webhook_r = results.get("webhook")
                if webhook_r:
                    event.webhook_sent = webhook_r["sent"]
                    event.webhook_error = webhook_r.get("error")


async def _run_ansible_action(
    app: "Quart",
    rule: AlertRule,
    value: float,
    event_id: str,
) -> None:
    """Run the rule's configured Ansible playbook (system-triggered).

    Starts after the evaluation transaction commits. Goes through
    runner.trigger_run like every other run path, so the playbook and the
    inventory must be ones the source lists (ansible/guards.py). The firing
    event links to the run, or says why the action did not start.
    """
    from steward.ansible.runner import trigger_run

    action = rule.ansible_action or {}
    playbook = action.get("playbook") or ""
    inventory = action.get("inventory") or ""
    source_name = action.get("source") or ""
    params: dict = {
        "extra_vars": list(action.get("extra_vars") or []) + alert_extra_vars(rule, value),
    }
    if action.get("limit"):
        params["limit"] = action["limit"]
    if action.get("tags"):
        params["tags"] = action["tags"]
    if action.get("check"):
        params["check"] = True
    scope = inventory if inventory.startswith("steward:") else f"repo:{source_name}:{inventory}"

    if not playbook or not inventory:
        run, err = None, "the action has no playbook or inventory"
    else:
        try:
            run, _source, err = await trigger_run(
                app, source_name=source_name, playbook_path=playbook,
                inventory_scope=scope, params=params, triggered_by=None)
        except Exception as exc:
            logger.exception("Ansible action for rule %r failed to start", rule.name)
            run, err = None, str(exc)
    if err:
        logger.error("Ansible action for rule %r not started: %s", rule.name, err)

    async with app.db_sessionmaker() as db:
        async with db.begin():
            ev = (await db.execute(
                select(AlertEvent).where(AlertEvent.id == event_id))).scalar_one_or_none()
            if ev is None:
                return
            if run is not None:
                ev.ansible_run_id = run.id
            else:
                why = f"Ansible action not started: {err}"
                ev.note = f"{ev.note}; {why}" if ev.note else why


async def check_silence(app: "Quart") -> None:
    """Mark the enabled rules whose series has gone silent (no_data).

    Run every minute by the core scheduler. A rule is silent once its series
    has had no sample for no_data_after(its usual gap); one that never had a
    sample counts from when the rule started watching. The mark sits beside
    the state, so a firing alert that goes silent stays firing. Each silence
    is recorded once, as an event, and notified once (rule.notify_no_data,
    outside maintenance windows), after the transaction commits.
    """
    now = datetime.now(timezone.utc)
    notify: list[tuple[AlertRule, str, str]] = []
    async with app.db_sessionmaker() as session:
        async with session.begin():
            rows = (await session.execute(
                select(AlertState, AlertRule)
                .join(AlertRule, AlertState.rule_id == AlertRule.id)
                .where(AlertRule.enabled.is_(True), AlertState.no_data_since.is_(None))
            )).all()
            if not rows:
                return
            windows = (await session.execute(
                select(MaintenanceWindow).where(
                    MaintenanceWindow.start_at <= now, MaintenanceWindow.end_at >= now)
            )).scalars().all()
            for state, rule in rows:
                watching_since = max(rule.created_at, state.last_transition_at)
                since = silent_since(state.last_sample_at, state.sample_interval_s,
                                     watching_since, now)
                if since is None:
                    continue
                state.no_data_since = since
                if state.last_sample_at is None:
                    note = (f"no sample yet, {human_duration((now - watching_since).total_seconds())}"
                            " after the rule started watching")
                else:
                    note = f"no sample for {human_duration((now - state.last_sample_at).total_seconds())}"
                    if state.sample_interval_s:
                        note += f" (usually every {human_duration(state.sample_interval_s)})"
                muted = _muted_by(windows, rule)
                event_id = _add_event(session, rule.id, state.state.value, "no_data", None,
                                      now, note=f"{note}; {muted}" if muted else note)
                if rule.notify_no_data and not muted:
                    notify.append((rule, event_id, note))
    for rule, event_id, note in notify:
        _spawn(_dispatch_notification(app, "no_data", rule, None, event_id, note))


async def notify_after_maintenance(app: "Quart") -> None:
    """Notify the alerts that started firing inside a maintenance window and
    are still firing now that no window covers them, and run their actions:
    a window delays what it mutes, it does not discard it. Run every minute
    by the core scheduler; each muted firing is released once."""
    now = datetime.now(timezone.utc)
    release: list[tuple[AlertRule, str, float | None]] = []
    async with app.db_sessionmaker() as session:
        async with session.begin():
            rows = (await session.execute(
                select(AlertState, AlertRule)
                .join(AlertRule, AlertState.rule_id == AlertRule.id)
                .where(AlertRule.enabled.is_(True), AlertState.state == AlertStateEnum.firing)
            )).all()
            if not rows:
                return
            windows = (await session.execute(
                select(MaintenanceWindow).where(
                    MaintenanceWindow.start_at <= now, MaintenanceWindow.end_at >= now)
            )).scalars().all()
            for state, rule in rows:
                if _muted_by(windows, rule):
                    continue
                fired = (await session.execute(
                    select(AlertEvent)
                    .where(AlertEvent.rule_id == rule.id, AlertEvent.to_state == "firing")
                    .order_by(AlertEvent.transitioned_at.desc()).limit(1)
                )).scalar_one_or_none()
                note = (fired.note or "") if fired else ""
                if MUTED not in note or RELEASED in note:
                    continue
                fired.note = f"{note}; {RELEASED}"
                release.append((rule, fired.id, fired.metric_value))
    for rule, event_id, value in release:
        _spawn(_dispatch_notification(app, "firing", rule, value, event_id))
        if rule.ansible_action:
            _spawn(_run_ansible_action(app, rule, value, event_id))