Files
FabledSteward/steward/alerts/routes.py
T
bvandeusenandClaude Opus 5.5 c368e93e2c
CI / lint (push) Successful in 2s
CI / unit (push) Failing after 1m14s
CI / integration (push) Successful in 1m14s
CI / publish (push) Skipped
feat(alerts): silence is alertable — agent offline and no-data (#5062)
Agent offline is an ordinary rule now. The host_agent scheduler records
host_agent/<host>/seconds_since_seen every minute (from last_seen_at, or
registration time for an agent that never reported), replacing the task
that only logged stale agents. The catalog offers the metric, and picking
it on a new rule fills "> 180".

No data: each rule's state tracks its last sample and its usual gap,
smoothed so that an outage counts as at most three gaps. A core task,
alert_silence_check, runs every 60s and marks a rule no_data_since once its
series has had no sample for three usual gaps (at least 5 minutes; an hour
while the cadence is unknown). A never-sampled rule counts from when it
started watching. The mark sits beside the state, so a firing alert that
goes silent stays firing. Each silence records one event and sends one
notification (rule option notify_no_data, default on; muted in maintenance
windows). The next sample clears it with a "data resumed" event and
notification. Creating, changing, disabling or re-enabling a rule restarts
its watch.

Migration 0029 adds the columns and seeds each existing rule's last sample
and gap from its last day of data, so an upgrade does not flag every rule.

Shown on the Alerts page (a NO DATA badge with its start time), in MCP
list_alerts, list_alert_rules and get_host (state "no_data", no_data_since),
in get_overview findings (alert_no_data, or a firing alert marked silent)
and in the weekly report. The rule form has the notify option, and its
metric reference now matches what traefik records and lists host_agent.
Emails say "no data" and carry the event's note.

The metric-catalog guard also reads positional MetricBatch.add arguments.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 13:06:54 -04:00

330 lines
13 KiB
Python

from __future__ import annotations
from datetime import datetime, timezone
from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
from steward.core.audit import log_audit
from sqlalchemy import select
from steward.auth.middleware import require_role
from steward.alerts import service as alert_svc
from steward.ansible import sources as src_module
from steward.models.alerts import AlertRule, AlertOperator, MaintenanceWindow
from steward.models.users import UserRole
alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")
_OPERATORS = [(op.value, op.value) for op in AlertOperator]
def _rule_form(form) -> dict:
"""The rule form's fields, as alert_svc.rule_fields takes them."""
return {
"name": form.get("name", ""),
"source_module": form.get("source_module", ""),
"resource_name": form.get("resource_name", ""),
"metric_name": form.get("metric_name", ""),
"operator": form.get("operator", ""),
"threshold": form.get("threshold"),
"consecutive_failures_required": form.get("consecutive_failures_required"),
"consecutive_ok_required": form.get("consecutive_ok_required"),
"notify_no_data": "notify_no_data" in form,
}
def _form_time(value) -> datetime:
"""A datetime-local field, read as UTC."""
try:
return datetime.fromisoformat((value or "").strip()).replace(tzinfo=timezone.utc)
except ValueError:
raise ValueError(f"{value!r} is not a date and time.") from None
def _is_admin() -> bool:
return session.get("user_role") == UserRole.admin.value
def _ansible_source_names() -> list[str]:
try:
return [s["name"] for s in src_module.get_sources(current_app.config.get("ANSIBLE", {}))]
except Exception:
return []
def _parse_ansible_action(form) -> tuple[dict | None, str | None]:
"""Parse + validate the admin-only Ansible action from a rule form.
Returns (action_or_None, error_or_None). action is None when the action
checkbox is unchecked (i.e. the action is being cleared).
"""
if "ansible_enabled" not in form:
return None, None
source_name = (form.get("ansible_source", "") or "").strip()
playbook = (form.get("ansible_playbook", "") or "").strip()
inventory = (form.get("ansible_inventory", "") or "").strip()
if not (source_name and playbook and inventory):
return None, "Ansible action requires source, playbook, and inventory"
try:
srcs = src_module.get_sources(current_app.config.get("ANSIBLE", {}))
except Exception:
return None, "Ansible sources are misconfigured"
source = next((s for s in srcs if s["name"] == source_name), None)
if source is None:
return None, f"Ansible source '{source_name}' not found"
if playbook not in src_module.discover_playbooks(source["path"]):
return None, f"Playbook '{playbook}' not found in source '{source_name}'"
extra_vars: list[str] = []
for line in (form.get("ansible_extra_vars", "") or "").splitlines():
line = line.strip()
if not line:
continue
if "=" not in line:
return None, f"Invalid extra var (expected key=value): {line!r}"
extra_vars.append(line)
action: dict = {"source": source_name, "playbook": playbook, "inventory": inventory}
if extra_vars:
action["extra_vars"] = extra_vars
limit = (form.get("ansible_limit", "") or "").strip()
tags = (form.get("ansible_tags", "") or "").strip()
if limit:
action["limit"] = limit
if tags:
action["tags"] = tags
if "ansible_check" in form:
action["check"] = True
return action, None
@alerts_bp.get("/")
@require_role(UserRole.viewer)
async def list_alerts():
async with current_app.db_sessionmaker() as db:
active = await alert_svc.active_alerts(db)
rules = await alert_svc.list_rules(db)
return await render_template("alerts/list.html", active=active, rules=rules, operators=_OPERATORS)
@alerts_bp.get("/api/rule_fields")
@require_role(UserRole.viewer)
async def rule_fields_api():
source = request.args.get("source_module", "")
current_resource = request.args.get("current_resource", "")
current_metric = request.args.get("current_metric", "")
async with current_app.db_sessionmaker() as db:
resources = await alert_svc.resources_for(db, source) if source else []
async with current_app.db_sessionmaker() as db:
metrics = await alert_svc.metrics_for(db, source)
return await render_template(
"alerts/_rule_fields.html",
resources=resources,
metrics=metrics,
current_resource=current_resource,
current_metric=current_metric,
)
@alerts_bp.get("/rules/new")
@require_role(UserRole.operator)
async def new_rule():
first_source = alert_svc.source_modules()[0]
async with current_app.db_sessionmaker() as db:
resources = await alert_svc.resources_for(db, first_source)
return await render_template(
"alerts/rules_form.html",
rule=None,
operators=_OPERATORS,
source_modules=alert_svc.source_modules(),
initial_source=first_source,
resources=resources,
metrics=alert_svc.metric_catalog().get(first_source, []),
ansible_sources=_ansible_source_names(),
)
@alerts_bp.get("/rules/<rule_id>/edit")
@require_role(UserRole.operator)
async def edit_rule(rule_id: str):
async with current_app.db_sessionmaker() as db:
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if rule is None:
return "Not found", 404
resources = await alert_svc.resources_for(db, rule.source_module)
return await render_template(
"alerts/rules_form.html",
rule=rule,
operators=_OPERATORS,
source_modules=alert_svc.source_modules(),
initial_source=rule.source_module,
resources=resources,
metrics=alert_svc.metric_catalog().get(rule.source_module, []),
ansible_sources=_ansible_source_names(),
)
@alerts_bp.post("/rules")
@require_role(UserRole.operator)
async def create_rule():
form = await request.form
user_id = session.get("user_id")
# Ansible action is admin-only; non-admins simply can't set one.
ansible_action = None
if _is_admin():
ansible_action, err = _parse_ansible_action(form)
if err:
return err, 400
async with current_app.db_sessionmaker() as db:
async with db.begin():
try:
fields = await alert_svc.rule_fields(db, **_rule_form(form))
except ValueError as exc:
return str(exc), 400
rule = await alert_svc.add_rule(db, fields, created_by=user_id,
ansible_action=ansible_action)
await log_audit(current_app, user_id, session.get("username", ""), "alert_rule.created",
entity_type="alert_rule", entity_id=rule.name,
detail={"source": rule.source_module, "resource": rule.resource_name,
"metric": rule.metric_name})
return redirect(url_for("alerts.list_alerts"))
@alerts_bp.post("/rules/<rule_id>")
@require_role(UserRole.operator)
async def update_rule(rule_id: str):
form = await request.form
# Validate the admin-only action before opening the transaction. Non-admins
# leave the existing action untouched (they can still edit everything else).
apply_action = _is_admin()
if apply_action:
ansible_action, err = _parse_ansible_action(form)
if err:
return err, 400
async with current_app.db_sessionmaker() as db:
async with db.begin():
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if rule is None:
return "Not found", 404
try:
fields = await alert_svc.rule_fields(db, **_rule_form(form))
except ValueError as exc:
return str(exc), 400
await alert_svc.apply_rule_change(db, rule, fields)
if apply_action:
rule.ansible_action = ansible_action
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
"alert_rule.updated", entity_type="alert_rule", entity_id=rule.name)
return redirect(url_for("alerts.list_alerts"))
@alerts_bp.post("/rules/<rule_id>/toggle")
@require_role(UserRole.operator)
async def toggle_rule(rule_id: str):
async with current_app.db_sessionmaker() as db:
async with db.begin():
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if rule is None:
return "Not found", 404
await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled)
action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
action, entity_type="alert_rule", entity_id=rule.name)
return redirect(url_for("alerts.list_alerts"))
@alerts_bp.post("/rules/<rule_id>/delete")
@require_role(UserRole.admin)
async def delete_rule(rule_id: str):
rule_name = None
async with current_app.db_sessionmaker() as db:
async with db.begin():
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if rule:
rule_name = rule.name
await db.delete(rule)
if rule_name:
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
"alert_rule.deleted", entity_type="alert_rule", entity_id=rule_name)
return redirect(url_for("alerts.list_alerts"))
@alerts_bp.get("/maintenance")
@require_role(UserRole.viewer)
async def list_maintenance():
now = datetime.now(timezone.utc)
async with current_app.db_sessionmaker() as db:
windows = await alert_svc.list_maintenance_windows(db)
return await render_template(
"alerts/maintenance.html",
windows=windows,
now=now,
source_modules=alert_svc.source_modules(),
)
@alerts_bp.get("/maintenance/new")
@require_role(UserRole.operator)
async def new_maintenance():
return await render_template(
"alerts/maintenance_form.html",
window=None,
source_modules=alert_svc.source_modules(),
)
@alerts_bp.post("/maintenance")
@require_role(UserRole.operator)
async def create_maintenance():
form = await request.form
user_id = session.get("user_id")
try:
fields = alert_svc.window_fields(
name=form.get("name", ""),
start_at=_form_time(form.get("start_at")), end_at=_form_time(form.get("end_at")),
scope_source=form.get("scope_source"), scope_resource=form.get("scope_resource"))
except ValueError as exc:
return str(exc), 400
window = MaintenanceWindow(created_by=user_id, **fields)
async with current_app.db_sessionmaker() as db:
async with db.begin():
db.add(window)
scope_desc = alert_svc.window_scope(window)
await log_audit(current_app, user_id, session.get("username", ""),
"maintenance_window.created", entity_type="maintenance_window",
entity_id=window.name, detail={"scope": scope_desc})
return redirect(url_for("alerts.list_maintenance"))
@alerts_bp.post("/maintenance/<window_id>/delete")
@require_role(UserRole.operator)
async def delete_maintenance(window_id: str):
window_name = None
async with current_app.db_sessionmaker() as db:
async with db.begin():
result = await db.execute(
select(MaintenanceWindow).where(MaintenanceWindow.id == window_id)
)
window = result.scalar_one_or_none()
if window:
window_name = window.name
await db.delete(window)
if window_name:
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
"maintenance_window.deleted", entity_type="maintenance_window",
entity_id=window_name)
return redirect(url_for("alerts.list_maintenance"))
@alerts_bp.post("/<rule_id>/acknowledge")
@require_role(UserRole.operator)
async def acknowledge(rule_id: str):
user_id = session.get("user_id")
async with current_app.db_sessionmaker() as db:
async with db.begin():
try:
await alert_svc.acknowledge(db, rule_id, user_id)
except LookupError:
return "Not found", 404
except ValueError as exc:
return f"Conflict: {exc}", 409
return redirect(url_for("alerts.list_alerts"))