FabledSteward/steward/alerts/routes.py

from __future__ import annotations
from datetime import datetime, timezone
from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
from steward.core.audit import log_audit
from sqlalchemy import func, select
from steward.auth.middleware import require_role
from steward.alerts import service as alert_svc
from steward.core.paging import PAGE_SIZE, next_page
from steward.core.time_range import parse_range
from steward.ansible import sources as src_module
from steward.models.alerts import AlertEvent, AlertRule, AlertOperator, AlertState, MaintenanceWindow
from steward.models.users import UserRole

alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")

_OPERATORS = [(op.value, op.value) for op in AlertOperator]

def _form_condition(form) -> dict:
    """The operator and threshold the form asks for. An up/down metric's form
    sends the word it fires on (binary_when) instead: down is < 0.5, up is
    >= 0.5, the midpoint of the 1.0 / 0.0 such a metric records."""
    when = form.get("binary_when")
    if when in ("down", "up") and form.get("metric_name") in ("is_up", "is_healthy"):
        return {"operator": "<" if when == "down" else ">=", "threshold": "0.5"}
    return {"operator": form.get("operator", ""), "threshold": form.get("threshold")}


def _rule_form(form) -> dict:
    """The rule form's fields, as alert_svc.rule_fields takes them."""
    return {
        "name": form.get("name", ""),
        "source_module": form.get("source_module", ""),
        "resource_name": form.get("resource_name", ""),
        "metric_name": form.get("metric_name", ""),
        **_form_condition(form),
        "consecutive_failures_required": form.get("consecutive_failures_required"),
        "consecutive_ok_required": form.get("consecutive_ok_required"),
        "notify_no_data": "notify_no_data" in form,
        "covers": form.get("covers") or "resource",
        "cover_group": form.get("cover_group", ""),
    }


def _form_time(value) -> datetime:
    """A datetime-local field, read as UTC."""
    try:
        return datetime.fromisoformat((value or "").strip()).replace(tzinfo=timezone.utc)
    except ValueError:
        raise ValueError(f"{value!r} is not a date and time.") from None


async def _child_conflict(db, rule: AlertRule):
    """A 409 for a host's rule under a covering rule, which is changed
    through its parent (#5464); None for any other rule."""
    if not rule.parent_id:
        return None
    parent = await db.get(AlertRule, rule.parent_id)
    return str(alert_svc.PartOfCoveringRule(rule, parent.name if parent else "?")), 409


def _is_admin() -> bool:
    return session.get("user_role") == UserRole.admin.value


def _ansible_source_names() -> list[str]:
    try:
        return [s["name"] for s in src_module.get_sources(current_app.config.get("ANSIBLE", {}))]
    except Exception:
        return []


def _parse_ansible_action(form) -> tuple[dict | None, str | None]:
    """Parse + validate the admin-only Ansible action from a rule form.

    Returns (action_or_None, error_or_None). action is None when the action
    checkbox is unchecked (i.e. the action is being cleared).
    """
    if "ansible_enabled" not in form:
        return None, None
    source_name = (form.get("ansible_source", "") or "").strip()
    playbook = (form.get("ansible_playbook", "") or "").strip()
    inventory = (form.get("ansible_inventory", "") or "").strip()
    if not (source_name and playbook and inventory):
        return None, "Ansible action requires source, playbook, and inventory"
    try:
        srcs = src_module.get_sources(current_app.config.get("ANSIBLE", {}))
    except Exception:
        return None, "Ansible sources are misconfigured"
    source = next((s for s in srcs if s["name"] == source_name), None)
    if source is None:
        return None, f"Ansible source '{source_name}' not found"
    if playbook not in src_module.discover_playbooks(source["path"]):
        return None, f"Playbook '{playbook}' not found in source '{source_name}'"
    extra_vars: list[str] = []
    for line in (form.get("ansible_extra_vars", "") or "").splitlines():
        line = line.strip()
        if not line:
            continue
        if "=" not in line:
            return None, f"Invalid extra var (expected key=value): {line!r}"
        extra_vars.append(line)
    action: dict = {"source": source_name, "playbook": playbook, "inventory": inventory}
    if extra_vars:
        action["extra_vars"] = extra_vars
    limit = (form.get("ansible_limit", "") or "").strip()
    tags = (form.get("ansible_tags", "") or "").strip()
    if limit:
        action["limit"] = limit
    if tags:
        action["tags"] = tags
    if "ansible_check" in form:
        action["check"] = True
    return action, None


async def _notify() -> list[str]:
    """Where alerts' notifications go now, from the settings the app runs with
    and the accounts that get alert email."""
    from steward.core.mail import audience
    async with current_app.db_sessionmaker() as db:
        accounts = len((await audience(db))["alerts"])
    return alert_svc.notify_targets(current_app.config.get("SMTP", {}), current_app.config.get("WEBHOOK", {}),
                                    accounts)


@alerts_bp.get("/")
@require_role(UserRole.viewer)
async def list_alerts():
    """The verdict and what is active, then every rule with what it is doing
    now and when it last fired (#5315)."""
    async with current_app.db_sessionmaker() as db:
        active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
        rules = await alert_svc.list_rules(db)
        states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
        fired = await alert_svc.last_fired(db)

    # Every rule with what it is doing now; disabled rules last. A covering
    # rule's per-host children sit under it (#5464), and it reads as the most
    # pressing of their states.
    rule_state = {r.id: alert_svc.alert_label(states[r.id]) for r in rules if r.id in states}
    children: dict[str, list] = {}
    for r in rules:
        if r.parent_id:
            children.setdefault(r.parent_id, []).append(r)
    for kids in children.values():
        kids.sort(key=lambda r: r.resource_name.lower())
        parent_id = kids[0].parent_id
        labels = [rule_state.get(k.id) for k in kids]
        rule_state[parent_id] = min(
            (lb for lb in labels if lb and lb not in ("inactive", "resolved")),
            key=lambda lb: alert_svc.ACTIVE_ORDER.get(lb, 9), default="inactive")
        fired[parent_id] = max((fired[k.id] for k in kids if k.id in fired), default=None)
    rules = sorted((r for r in rules if not r.parent_id),
                   key=lambda r: (not r.enabled, r.name.lower()))
    return await render_template(
        "alerts/list.html", active=active, rules=rules, children=children,
        rule_state=rule_state, last_fired={k: v for k, v in fired.items() if v},
        notify=await _notify(), operators=_OPERATORS,
        poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60))


@alerts_bp.get("/active")
@require_role(UserRole.viewer)
async def active_fragment():
    """The verdict and Active section alone, which the alerts page polls."""
    async with current_app.db_sessionmaker() as db:
        active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
        # Rules as people made them: a covering rule counts once, not per host.
        rules = [r for r in await alert_svc.list_rules(db) if not r.parent_id]
    return await render_template(
        "alerts/_active.html", active=active, notify=await _notify(), rules_total=len(rules),
        rules_off=sum(1 for r in rules if not r.enabled))


def _history_row(event, rule, names: dict | None = None) -> dict:
    """One transition as the history page shows it. `names` maps user ids to
    usernames: a transition a person made says who (#5463)."""
    from steward.core.alerts import MUTED, RELEASED
    from steward.core.formatting import metric_label, metric_value
    notes, muted_by, released = [], None, False
    for part in (event.note or "").split("; "):
        if part.startswith(MUTED):
            muted_by = part[len(MUTED):].strip() or "a maintenance window"
        elif part == RELEASED:
            released = True
        elif part:
            notes.append(part)
    actor = getattr(event, "actor_id", None)
    by = (names or {}).get(actor) or ("a deleted user" if actor else None)
    if event.to_state == "acknowledged":
        notes.insert(0, f"acknowledged by {by}" if by else "acknowledged")
    elif by and notes:
        notes[0] = f"{notes[0]} by {by}"
    metric = rule.metric_name
    if event.metric_value is None:
        value = f"No {metric_label(metric)} reading"
    elif metric in ("is_up", "is_healthy"):
        value = metric_value(metric, event.metric_value).capitalize()
    else:
        value = f"{metric_label(metric)} {metric_value(metric, event.metric_value)}"
    return {
        "at": event.transitioned_at, "rule": rule,
        "from_state": event.from_state, "to_state": event.to_state,
        "value": value, "raw": event.metric_value, "condition": alert_svc.rule_condition(rule),
        "notes": notes, "muted_by": muted_by, "released": released,
        "deliveries": [(label, sent, error) for label, sent, error in (
            ("email", event.email_sent, event.email_error),
            ("webhook", event.webhook_sent, event.webhook_error)) if sent is not None],
        "ansible_run_id": event.ansible_run_id,
    }


@alerts_bp.get("/history")
@require_role(UserRole.viewer)
async def alert_history():
    """Every alert episode in a range, newest first; one rule's with ?rule=,
    which also says how often it fired and for how long in all (#5316). A
    page of transitions at a time as the reader scrolls: ?before=<cursor> is
    the next page's episodes alone (core.paging). The Notified and Action
    columns show only when something in the range was notified or acted on."""
    from steward.alerts.episodes import episodes, summary
    now = datetime.now(timezone.utc)
    since, range_key = parse_range(request.args.get("range") or "7d")
    before = request.args.get("before")
    async with current_app.db_sessionmaker() as db:
        # A covering rule has no history of its own; its hosts' rules do.
        rules = [r for r in await alert_svc.list_rules(db) if r.covers == "resource"]
        chosen = next((r for r in rules if r.id == request.args.get("rule")), None)
        rule_id = chosen.id if chosen else None
        rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before)
        show_notified, show_action = await alert_svc.history_columns(db, since, rule_id)
        names = await alert_svc.usernames(db, (e.actor_id for e, _ in rows))
        verdict = None
        if chosen and not before:
            every = await alert_svc.history(db, since, rule_id, HISTORY_SUMMARY_ROWS)
            verdict = summary(episodes([_history_row(e, r) for e, r in every], newest_page=True,
                                       now=now, since=since))
    rows, after = next_page(rows, lambda row: (row[0].transitioned_at, row[0].id))
    ctx = {"episodes": episodes([_history_row(e, r, names) for e, r in rows],
                                newest_page=not before, now=now, since=since),
           "after": after, "rules": rules, "chosen": chosen, "current_range": range_key,
           "show_notified": show_notified, "show_action": show_action, "verdict": verdict}
    if before:
        return await render_template("alerts/_history_page.html", **ctx)
    if request.headers.get("HX-Request"):
        return await render_template("alerts/_history_rows.html", **ctx)
    return await render_template("alerts/history.html", **ctx)


# How many of one rule's transitions the history verdict reads ("fired N
# times, X in all"); a rule that flapped more often than this in the range
# is summed over its newest ones.
HISTORY_SUMMARY_ROWS = 2000


@alerts_bp.get("/api/rule_fields")
@require_role(UserRole.viewer)
async def rule_fields_api():
    source = request.args.get("source_module", "")
    current_resource = request.args.get("current_resource", "")
    current_metric = request.args.get("current_metric", "")
    async with current_app.db_sessionmaker() as db:
        resources = await alert_svc.resources_for(db, source) if source else []
        metrics = await alert_svc.metrics_for(db, source)
        cover = await _cover_choices(db, source)
    return await render_template(
        "alerts/_rule_fields.html",
        resources=resources,
        metrics=metrics,
        current_resource=current_resource,
        current_metric=current_metric,
        **cover,
    )


async def _cover_choices(db, source: str, rule: AlertRule | None = None) -> dict:
    """What the rule form offers for covering hosts (#5464): whether this
    source's resources are hosts, the inventory groups, and the rule's own
    choice, fixed once it exists."""
    from steward.hosts.service import group_names
    host_source = source in alert_svc.HOST_SOURCES
    return {"host_source": host_source, "groups": await group_names(db) if host_source else [],
            "current_covers": rule.covers if rule else "resource",
            "current_group": (rule.cover_group or "") if rule else "",
            "covers_locked": rule is not None}


@alerts_bp.get("/rules/new")
@require_role(UserRole.operator)
async def new_rule():
    first_source = alert_svc.source_modules()[0]
    async with current_app.db_sessionmaker() as db:
        resources = await alert_svc.resources_for(db, first_source)
        cover = await _cover_choices(db, first_source)
    return await render_template(
        "alerts/rules_form.html",
        rule=None, **cover,
        operators=_OPERATORS,
        source_modules=alert_svc.source_modules(),
        initial_source=first_source,
        resources=resources,
        metrics=alert_svc.metric_catalog().get(first_source, []),
        catalog=alert_svc.metric_catalog(),
        ansible_sources=_ansible_source_names(),
        notify=await _notify(),
    )


@alerts_bp.get("/rules/<rule_id>/edit")
@require_role(UserRole.operator)
async def edit_rule(rule_id: str):
    async with current_app.db_sessionmaker() as db:
        result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
        rule = result.scalar_one_or_none()
        if rule is None:
            return "Not found", 404
        if rule.parent_id:
            # A host's rule under a covering rule is changed through it.
            return redirect(f"/alerts/rules/{rule.parent_id}/edit")
        resources = await alert_svc.resources_for(db, rule.source_module)
        cover = await _cover_choices(db, rule.source_module, rule)
        hosts = sorted((c.resource_name for c in (await db.execute(
            select(AlertRule).where(AlertRule.parent_id == rule.id))).scalars()), key=str.lower)
        history_count = (await db.execute(
            select(func.count()).select_from(AlertEvent).where(AlertEvent.rule_id == rule.id))).scalar_one()
        fired = (await alert_svc.last_fired(db, rule.id)).get(rule.id)
    return await render_template(
        "alerts/rules_form.html",
        rule=rule, history_count=history_count, fired=fired, notify=await _notify(),
        covered=hosts, **cover,
        operators=_OPERATORS,
        source_modules=alert_svc.source_modules(),
        initial_source=rule.source_module,
        resources=resources,
        metrics=alert_svc.metric_catalog().get(rule.source_module, []),
        catalog=alert_svc.metric_catalog(),
        ansible_sources=_ansible_source_names(),
    )


@alerts_bp.post("/rules")
@require_role(UserRole.operator)
async def create_rule():
    form = await request.form
    user_id = session.get("user_id")
    # Ansible action is admin-only; non-admins simply can't set one.
    ansible_action = None
    if _is_admin():
        ansible_action, err = _parse_ansible_action(form)
        if err:
            return err, 400
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            try:
                fields = await alert_svc.rule_fields(db, **_rule_form(form))
            except ValueError as exc:
                return str(exc), 400
            rule = await alert_svc.add_rule(db, fields, created_by=user_id,
                                            ansible_action=ansible_action)
    await log_audit(current_app, user_id, session.get("username", ""), "alert_rule.created",
                    entity_type="alert_rule", entity_id=rule.name,
                    detail={"source": rule.source_module, "resource": rule.resource_name,
                            "metric": rule.metric_name})
    return redirect(url_for("alerts.list_alerts"))


@alerts_bp.post("/rules/<rule_id>")
@require_role(UserRole.operator)
async def update_rule(rule_id: str):
    form = await request.form
    # Validate the admin-only action before opening the transaction. Non-admins
    # leave the existing action untouched (they can still edit everything else).
    apply_action = _is_admin()
    if apply_action:
        ansible_action, err = _parse_ansible_action(form)
        if err:
            return err, 400
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
            rule = result.scalar_one_or_none()
            if rule is None:
                return "Not found", 404
            if (conflict := await _child_conflict(db, rule)):
                return conflict
            try:
                # What a rule covers is fixed once it exists; its group may change.
                fields = await alert_svc.rule_fields(db, **{**_rule_form(form),
                                                            "covers": rule.covers})
            except ValueError as exc:
                return str(exc), 400
            await alert_svc.apply_rule_change(db, rule, fields, actor_id=session.get("user_id"))
            if apply_action:
                rule.ansible_action = ansible_action
            if rule.covers != "resource":
                await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id"))
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "alert_rule.updated", entity_type="alert_rule", entity_id=rule.name)
    return redirect(url_for("alerts.list_alerts"))


@alerts_bp.post("/rules/<rule_id>/toggle")
@require_role(UserRole.operator)
async def toggle_rule(rule_id: str):
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
            rule = result.scalar_one_or_none()
            if rule is None:
                return "Not found", 404
            if (conflict := await _child_conflict(db, rule)):
                return conflict
            await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled,
                                              actor_id=session.get("user_id"))
            if rule.covers != "resource":
                await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id"))
    action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    action, entity_type="alert_rule", entity_id=rule.name)
    return redirect(url_for("alerts.list_alerts"))


@alerts_bp.post("/rules/<rule_id>/test")
@require_role(UserRole.operator)
async def test_rule(rule_id: str):
    """Send test (#5466): a TEST notification for this rule, now, by every
    channel its alerts use; the result shows under the button."""
    try:
        sent = await alert_svc.send_test(current_app, rule_id)
    except LookupError:
        return "Not found", 404
    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                    "alert_rule.tested", entity_type="alert_rule", entity_id=sent["rule"],
                    detail={"rule_id": rule_id, "ok": sent["ok"], "lines": sent["lines"]})
    return await render_template("alerts/_test_result.html", ok=sent["ok"], lines=sent["lines"])


@alerts_bp.post("/rules/<rule_id>/delete")
@require_role(UserRole.admin)
async def delete_rule(rule_id: str):
    rule_name = None
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
            rule = result.scalar_one_or_none()
            if rule:
                if (conflict := await _child_conflict(db, rule)):
                    return conflict
                rule_name = rule.name
                await db.delete(rule)
    if rule_name:
        await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                        "alert_rule.deleted", entity_type="alert_rule", entity_id=rule_name)
    return redirect(url_for("alerts.list_alerts"))


@alerts_bp.get("/maintenance")
@require_role(UserRole.viewer)
async def list_maintenance():
    now = datetime.now(timezone.utc)
    async with current_app.db_sessionmaker() as db:
        windows = await alert_svc.list_maintenance_windows(db)
    # Active first (ending soonest first), then upcoming (starting soonest
    # first); expired ones are history, newest first, folded away.
    active = sorted((w for w in windows if w.start_at <= now <= w.end_at), key=lambda w: w.end_at)
    upcoming = sorted((w for w in windows if w.start_at > now), key=lambda w: w.start_at)
    expired = sorted((w for w in windows if w.end_at < now), key=lambda w: w.end_at, reverse=True)
    return await render_template(
        "alerts/maintenance.html",
        windows=windows, active=active, upcoming=upcoming, expired=expired,
        now=now,
        source_modules=alert_svc.source_modules(),
    )


@alerts_bp.get("/maintenance/new")
@require_role(UserRole.operator)
async def new_maintenance():
    return await render_template(
        "alerts/maintenance_form.html",
        window=None,
        source_modules=alert_svc.source_modules(),
    )


@alerts_bp.post("/maintenance")
@require_role(UserRole.operator)
async def create_maintenance():
    form = await request.form
    user_id = session.get("user_id")
    try:
        fields = alert_svc.window_fields(
            name=form.get("name", ""),
            start_at=_form_time(form.get("start_at")), end_at=_form_time(form.get("end_at")),
            scope_source=form.get("scope_source"), scope_resource=form.get("scope_resource"))
    except ValueError as exc:
        return str(exc), 400
    window = MaintenanceWindow(created_by=user_id, **fields)
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            db.add(window)
    scope_desc = alert_svc.window_scope(window)
    await log_audit(current_app, user_id, session.get("username", ""),
                    "maintenance_window.created", entity_type="maintenance_window",
                    entity_id=window.name, detail={"scope": scope_desc})
    return redirect(url_for("alerts.list_maintenance"))


@alerts_bp.post("/maintenance/<window_id>/delete")
@require_role(UserRole.operator)
async def delete_maintenance(window_id: str):
    window_name = None
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            result = await db.execute(
                select(MaintenanceWindow).where(MaintenanceWindow.id == window_id)
            )
            window = result.scalar_one_or_none()
            if window:
                window_name = window.name
                await db.delete(window)
    if window_name:
        await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                        "maintenance_window.deleted", entity_type="maintenance_window",
                        entity_id=window_name)
    return redirect(url_for("alerts.list_maintenance"))


@alerts_bp.post("/<rule_id>/acknowledge")
@require_role(UserRole.operator)
async def acknowledge(rule_id: str):
    user_id = session.get("user_id")
    async with current_app.db_sessionmaker() as db:
        async with db.begin():
            try:
                await alert_svc.acknowledge(db, rule_id, user_id)
            except LookupError:
                return "Not found", 404
            except ValueError as exc:
                return f"Conflict: {exc}", 409
    return redirect(url_for("alerts.list_alerts"))