331 lines
13 KiB
Python
331 lines
13 KiB
Python
from __future__ import annotations
|
|
from datetime import datetime, timezone
|
|
from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
|
|
from fabledscryer.core.audit import log_audit
|
|
from sqlalchemy import select
|
|
from fabledscryer.auth.middleware import require_role
|
|
from fabledscryer.models.alerts import AlertRule, AlertState, AlertStateEnum, AlertOperator, MaintenanceWindow
|
|
from fabledscryer.models.hosts import Host
|
|
from fabledscryer.models.metrics import PluginMetric
|
|
from fabledscryer.models.users import UserRole
|
|
|
|
alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")
|
|
|
|
_OPERATORS = [(op.value, op.value) for op in AlertOperator]
|
|
|
|
# Static catalog: source_module → available metric names
|
|
METRIC_CATALOG: dict[str, list[str]] = {
|
|
"ping": ["up", "response_time_ms"],
|
|
"dns": ["resolved", "ip_changed"],
|
|
"traefik": ["request_rate", "error_rate", "latency_p50_ms", "latency_p95_ms",
|
|
"latency_p99_ms", "response_bytes_rate", "cert_expiry_days"],
|
|
"unifi": ["is_up", "latency_ms", "total_clients"],
|
|
"ups": ["battery_charge_pct", "on_battery", "battery_runtime_secs"],
|
|
"docker": ["cpu_pct", "mem_pct"],
|
|
"http": ["is_up", "response_ms"],
|
|
# snmp metrics are user-defined OID labels — populated dynamically from plugin_metrics
|
|
"snmp": [],
|
|
}
|
|
|
|
# Ordered list for source module picker
|
|
_SOURCE_MODULES = list(METRIC_CATALOG.keys())
|
|
|
|
|
|
async def _get_resources(db, source_module: str) -> list[str]:
|
|
"""Return sorted distinct resource names for a source module.
|
|
|
|
For ping/dns, supplements with host names so the list is populated
|
|
even before any metrics have been recorded.
|
|
"""
|
|
result = await db.execute(
|
|
select(PluginMetric.resource_name)
|
|
.where(PluginMetric.source_module == source_module)
|
|
.distinct()
|
|
.order_by(PluginMetric.resource_name)
|
|
)
|
|
resources: set[str] = {r for (r,) in result}
|
|
|
|
if source_module in ("ping", "dns"):
|
|
host_result = await db.execute(select(Host.name).order_by(Host.name))
|
|
for (name,) in host_result:
|
|
resources.add(name)
|
|
|
|
return sorted(resources)
|
|
|
|
|
|
@alerts_bp.get("/")
|
|
@require_role(UserRole.viewer)
|
|
async def list_alerts():
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(
|
|
select(AlertState, AlertRule)
|
|
.join(AlertRule, AlertState.rule_id == AlertRule.id)
|
|
.where(AlertState.state.in_([
|
|
AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending
|
|
]))
|
|
.order_by(AlertState.last_transition_at.desc())
|
|
)
|
|
active = result.all()
|
|
|
|
rules_result = await db.execute(
|
|
select(AlertRule).order_by(AlertRule.name)
|
|
)
|
|
rules = rules_result.scalars().all()
|
|
|
|
return await render_template("alerts/list.html", active=active, rules=rules, operators=_OPERATORS)
|
|
|
|
|
|
@alerts_bp.get("/api/rule_fields")
|
|
@require_role(UserRole.viewer)
|
|
async def rule_fields_api():
|
|
source = request.args.get("source_module", "")
|
|
current_resource = request.args.get("current_resource", "")
|
|
current_metric = request.args.get("current_metric", "")
|
|
async with current_app.db_sessionmaker() as db:
|
|
resources = await _get_resources(db, source) if source else []
|
|
metrics = METRIC_CATALOG.get(source, [])
|
|
# Dynamic sources (e.g. snmp) have empty static catalog — query live metric names
|
|
if source and not metrics:
|
|
async with current_app.db_sessionmaker() as db:
|
|
m_result = await db.execute(
|
|
select(PluginMetric.metric_name)
|
|
.where(PluginMetric.source_module == source)
|
|
.distinct()
|
|
.order_by(PluginMetric.metric_name)
|
|
)
|
|
metrics = [r for (r,) in m_result]
|
|
return await render_template(
|
|
"alerts/_rule_fields.html",
|
|
resources=resources,
|
|
metrics=metrics,
|
|
current_resource=current_resource,
|
|
current_metric=current_metric,
|
|
)
|
|
|
|
|
|
@alerts_bp.get("/rules/new")
|
|
@require_role(UserRole.operator)
|
|
async def new_rule():
|
|
first_source = _SOURCE_MODULES[0]
|
|
async with current_app.db_sessionmaker() as db:
|
|
resources = await _get_resources(db, first_source)
|
|
return await render_template(
|
|
"alerts/rules_form.html",
|
|
rule=None,
|
|
operators=_OPERATORS,
|
|
source_modules=_SOURCE_MODULES,
|
|
initial_source=first_source,
|
|
resources=resources,
|
|
metrics=METRIC_CATALOG.get(first_source, []),
|
|
)
|
|
|
|
|
|
@alerts_bp.get("/rules/<rule_id>/edit")
|
|
@require_role(UserRole.operator)
|
|
async def edit_rule(rule_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule is None:
|
|
return "Not found", 404
|
|
resources = await _get_resources(db, rule.source_module)
|
|
return await render_template(
|
|
"alerts/rules_form.html",
|
|
rule=rule,
|
|
operators=_OPERATORS,
|
|
source_modules=_SOURCE_MODULES,
|
|
initial_source=rule.source_module,
|
|
resources=resources,
|
|
metrics=METRIC_CATALOG.get(rule.source_module, []),
|
|
)
|
|
|
|
|
|
@alerts_bp.post("/rules")
|
|
@require_role(UserRole.operator)
|
|
async def create_rule():
|
|
form = await request.form
|
|
user_id = session.get("user_id")
|
|
now = datetime.now(timezone.utc)
|
|
rule = AlertRule(
|
|
name=form["name"].strip(),
|
|
source_module=form["source_module"].strip(),
|
|
resource_name=form["resource_name"].strip(),
|
|
metric_name=form["metric_name"].strip(),
|
|
operator=AlertOperator(form["operator"]),
|
|
threshold=float(form["threshold"]),
|
|
consecutive_failures_required=int(form.get("consecutive_failures_required") or 1),
|
|
enabled=True,
|
|
created_by=user_id,
|
|
)
|
|
state = AlertState(
|
|
rule_id=rule.id,
|
|
state=AlertStateEnum.inactive,
|
|
last_transition_at=now,
|
|
)
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
db.add(rule)
|
|
await db.flush()
|
|
state.rule_id = rule.id
|
|
db.add(state)
|
|
await log_audit(current_app, user_id, session.get("username", ""), "alert_rule.created",
|
|
entity_type="alert_rule", entity_id=rule.name,
|
|
detail={"source": rule.source_module, "resource": rule.resource_name,
|
|
"metric": rule.metric_name})
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>")
|
|
@require_role(UserRole.operator)
|
|
async def update_rule(rule_id: str):
|
|
form = await request.form
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule is None:
|
|
return "Not found", 404
|
|
rule.name = form["name"].strip()
|
|
rule.source_module = form["source_module"].strip()
|
|
rule.resource_name = form["resource_name"].strip()
|
|
rule.metric_name = form["metric_name"].strip()
|
|
rule.operator = AlertOperator(form["operator"])
|
|
rule.threshold = float(form["threshold"])
|
|
rule.consecutive_failures_required = int(form.get("consecutive_failures_required") or 1)
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"alert_rule.updated", entity_type="alert_rule", entity_id=rule.name)
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>/toggle")
|
|
@require_role(UserRole.operator)
|
|
async def toggle_rule(rule_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule is None:
|
|
return "Not found", 404
|
|
rule.enabled = not rule.enabled
|
|
action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
action, entity_type="alert_rule", entity_id=rule.name)
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>/delete")
|
|
@require_role(UserRole.admin)
|
|
async def delete_rule(rule_id: str):
|
|
rule_name = None
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule:
|
|
rule_name = rule.name
|
|
await db.delete(rule)
|
|
if rule_name:
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"alert_rule.deleted", entity_type="alert_rule", entity_id=rule_name)
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.get("/maintenance")
|
|
@require_role(UserRole.viewer)
|
|
async def list_maintenance():
|
|
now = datetime.now(timezone.utc)
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(
|
|
select(MaintenanceWindow).order_by(MaintenanceWindow.start_at.desc())
|
|
)
|
|
windows = result.scalars().all()
|
|
return await render_template(
|
|
"alerts/maintenance.html",
|
|
windows=windows,
|
|
now=now,
|
|
source_modules=_SOURCE_MODULES,
|
|
)
|
|
|
|
|
|
@alerts_bp.get("/maintenance/new")
|
|
@require_role(UserRole.operator)
|
|
async def new_maintenance():
|
|
return await render_template(
|
|
"alerts/maintenance_form.html",
|
|
window=None,
|
|
source_modules=_SOURCE_MODULES,
|
|
)
|
|
|
|
|
|
@alerts_bp.post("/maintenance")
|
|
@require_role(UserRole.operator)
|
|
async def create_maintenance():
|
|
form = await request.form
|
|
user_id = session.get("user_id")
|
|
start_at = datetime.fromisoformat(form["start_at"]).replace(tzinfo=timezone.utc)
|
|
end_at = datetime.fromisoformat(form["end_at"]).replace(tzinfo=timezone.utc)
|
|
scope_source = form.get("scope_source", "").strip() or None
|
|
scope_resource = form.get("scope_resource", "").strip() or None
|
|
if scope_source is None:
|
|
scope_resource = None # can't have resource without source
|
|
window = MaintenanceWindow(
|
|
name=form["name"].strip(),
|
|
start_at=start_at,
|
|
end_at=end_at,
|
|
scope_source=scope_source,
|
|
scope_resource=scope_resource,
|
|
created_by=user_id,
|
|
)
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
db.add(window)
|
|
scope_desc = "all" if not scope_source else (
|
|
f"{scope_source}/{scope_resource}" if scope_resource else f"{scope_source}/*"
|
|
)
|
|
await log_audit(current_app, user_id, session.get("username", ""),
|
|
"maintenance_window.created", entity_type="maintenance_window",
|
|
entity_id=window.name, detail={"scope": scope_desc})
|
|
return redirect(url_for("alerts.list_maintenance"))
|
|
|
|
|
|
@alerts_bp.post("/maintenance/<window_id>/delete")
|
|
@require_role(UserRole.operator)
|
|
async def delete_maintenance(window_id: str):
|
|
window_name = None
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(
|
|
select(MaintenanceWindow).where(MaintenanceWindow.id == window_id)
|
|
)
|
|
window = result.scalar_one_or_none()
|
|
if window:
|
|
window_name = window.name
|
|
await db.delete(window)
|
|
if window_name:
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"maintenance_window.deleted", entity_type="maintenance_window",
|
|
entity_id=window_name)
|
|
return redirect(url_for("alerts.list_maintenance"))
|
|
|
|
|
|
@alerts_bp.post("/<rule_id>/acknowledge")
|
|
@require_role(UserRole.operator)
|
|
async def acknowledge(rule_id: str):
|
|
user_id = session.get("user_id")
|
|
now = datetime.now(timezone.utc)
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(
|
|
select(AlertState).where(AlertState.rule_id == rule_id)
|
|
)
|
|
state = result.scalar_one_or_none()
|
|
if state is None:
|
|
return "Not found", 404
|
|
if state.state != AlertStateEnum.firing:
|
|
return "Conflict: alert is not in FIRING state", 409
|
|
state.state = AlertStateEnum.acknowledged
|
|
state.acknowledged_by = user_id
|
|
state.acknowledged_at = now
|
|
state.last_transition_at = now
|
|
return redirect(url_for("alerts.list_alerts"))
|