bvandeusenandClaude Opus 5.5
c38fd68e48
The host page's link to the full metrics sat in the Agent card, under the anomaly card and the vitals strip, and was missed. It now heads the vitals strip at the top of the page. The anomaly card on the host, monitor, container and Traefik pages listed every anomaly of the last 30 days with its review form, reviewed ones included, and took over the top of the page. It is now one line: how many wait for review, which are out of their normal now, and a Review button to Alerts → Anomalies narrowed to that resource (?source=&resource=), where a review returns. It is hidden when nothing waits and nothing is going on. The callers fetch up to 100 rows so the count isn't capped at 10 or 20. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
687 lines
32 KiB
Python
687 lines
32 KiB
Python
from __future__ import annotations
|
|
from datetime import datetime, timezone
|
|
from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
|
|
from steward.core.audit import log_audit
|
|
from steward.core.settings import get_setting
|
|
from steward.core import outbox
|
|
from sqlalchemy import func, select
|
|
from steward.auth.middleware import require_role
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core.paging import PAGE_SIZE, next_page
|
|
from steward.core.time_range import parse_range
|
|
from steward.ansible import sources as src_module
|
|
from steward.models.alerts import AlertEvent, AlertRule, AlertOperator, AlertState, MaintenanceWindow
|
|
from steward.models.users import UserRole
|
|
|
|
alerts_bp = Blueprint("alerts", __name__, url_prefix="/alerts")
|
|
|
|
_OPERATORS = [(op.value, op.said) for op in AlertOperator]
|
|
|
|
def _form_condition(form) -> dict:
|
|
"""The operator and threshold the form asks for. An up/down metric's form
|
|
sends the word it fires on (binary_when) instead: down is < 0.5, up is
|
|
>= 0.5, the midpoint of the 1.0 / 0.0 such a metric records."""
|
|
when = form.get("binary_when")
|
|
if when in ("down", "up") and form.get("metric_name") in ("is_up", "is_healthy"):
|
|
return {"operator": "<" if when == "down" else ">=", "threshold": "0.5"}
|
|
return {"operator": form.get("operator", ""), "threshold": form.get("threshold")}
|
|
|
|
|
|
def _rule_form(form) -> dict:
|
|
"""The rule form's fields, as alert_svc.rule_fields takes them."""
|
|
return {
|
|
"name": form.get("name", ""),
|
|
"source_module": form.get("source_module", ""),
|
|
"resource_name": form.get("resource_name", ""),
|
|
"metric_name": form.get("metric_name", ""),
|
|
**_form_condition(form),
|
|
"consecutive_failures_required": form.get("consecutive_failures_required"),
|
|
"consecutive_ok_required": form.get("consecutive_ok_required"),
|
|
"notify_no_data": "notify_no_data" in form,
|
|
"covers": form.get("covers") or "resource",
|
|
"cover_group": form.get("cover_group", ""),
|
|
}
|
|
|
|
|
|
def _form_time(value) -> datetime:
|
|
"""A datetime-local field, read as UTC."""
|
|
try:
|
|
return datetime.fromisoformat((value or "").strip()).replace(tzinfo=timezone.utc)
|
|
except ValueError:
|
|
raise ValueError(f"{value!r} is not a date and time.") from None
|
|
|
|
|
|
async def _child_conflict(db, rule: AlertRule):
|
|
"""A 409 for a host's rule under a covering rule, which is changed
|
|
through its parent (#5464); None for any other rule."""
|
|
if not rule.parent_id:
|
|
return None
|
|
parent = await db.get(AlertRule, rule.parent_id)
|
|
return str(alert_svc.PartOfCoveringRule(rule, parent.name if parent else "?")), 409
|
|
|
|
|
|
def _is_admin() -> bool:
|
|
return session.get("user_role") == UserRole.admin.value
|
|
|
|
|
|
def _ansible_source_names() -> list[str]:
|
|
try:
|
|
return [s["name"] for s in src_module.get_sources(current_app.config.get("ANSIBLE", {}))]
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def _parse_ansible_action(form) -> tuple[dict | None, str | None]:
|
|
"""Parse + validate the admin-only Ansible action from a rule form.
|
|
|
|
Returns (action_or_None, error_or_None). action is None when the action
|
|
checkbox is unchecked (i.e. the action is being cleared).
|
|
"""
|
|
if "ansible_enabled" not in form:
|
|
return None, None
|
|
source_name = (form.get("ansible_source", "") or "").strip()
|
|
playbook = (form.get("ansible_playbook", "") or "").strip()
|
|
inventory = (form.get("ansible_inventory", "") or "").strip()
|
|
if not (source_name and playbook and inventory):
|
|
return None, "Ansible action requires source, playbook, and inventory"
|
|
try:
|
|
srcs = src_module.get_sources(current_app.config.get("ANSIBLE", {}))
|
|
except Exception:
|
|
return None, "Ansible sources are misconfigured"
|
|
source = next((s for s in srcs if s["name"] == source_name), None)
|
|
if source is None:
|
|
return None, f"Ansible source '{source_name}' not found"
|
|
if playbook not in src_module.discover_playbooks(source["path"]):
|
|
return None, f"Playbook '{playbook}' not found in source '{source_name}'"
|
|
extra_vars: list[str] = []
|
|
for line in (form.get("ansible_extra_vars", "") or "").splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
if "=" not in line:
|
|
return None, f"Invalid extra var (expected key=value): {line!r}"
|
|
extra_vars.append(line)
|
|
action: dict = {"source": source_name, "playbook": playbook, "inventory": inventory}
|
|
if extra_vars:
|
|
action["extra_vars"] = extra_vars
|
|
limit = (form.get("ansible_limit", "") or "").strip()
|
|
tags = (form.get("ansible_tags", "") or "").strip()
|
|
if limit:
|
|
action["limit"] = limit
|
|
if tags:
|
|
action["tags"] = tags
|
|
if "ansible_check" in form:
|
|
action["check"] = True
|
|
return action, None
|
|
|
|
|
|
async def _notify() -> list[str]:
|
|
"""Where alerts' notifications go now, from the settings the app runs with
|
|
and the accounts that get alert email."""
|
|
from steward.core.mail import audience
|
|
async with current_app.db_sessionmaker() as db:
|
|
accounts = len((await audience(db))["alerts"])
|
|
return alert_svc.notify_targets(current_app.config.get("SMTP", {}), current_app.config.get("WEBHOOK", {}),
|
|
accounts)
|
|
|
|
|
|
@alerts_bp.get("/")
|
|
@require_role(UserRole.viewer)
|
|
async def list_alerts():
|
|
"""The verdict and what is active, then every rule with what it is doing
|
|
now and when it last fired (#5315)."""
|
|
async with current_app.db_sessionmaker() as db:
|
|
active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
|
|
rules = await alert_svc.list_rules(db)
|
|
states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
|
|
fired = await alert_svc.last_fired(db)
|
|
|
|
# Every rule with what it is doing now; disabled rules last. A covering
|
|
# rule's per-host children sit under it (#5464), and it reads as the most
|
|
# pressing of their states.
|
|
rule_state = {r.id: alert_svc.alert_label(states[r.id]) for r in rules if r.id in states}
|
|
children: dict[str, list] = {}
|
|
uneasy: dict[str, list[str]] = {}
|
|
for r in rules:
|
|
if r.parent_id:
|
|
children.setdefault(r.parent_id, []).append(r)
|
|
for kids in children.values():
|
|
kids.sort(key=lambda r: r.resource_name.lower())
|
|
parent_id = kids[0].parent_id
|
|
labels = [rule_state.get(k.id) for k in kids]
|
|
rule_state[parent_id] = min(
|
|
(lb for lb in labels if lb and lb not in ("inactive", "resolved")),
|
|
key=lambda lb: alert_svc.ACTIVE_ORDER.get(lb, 9), default="inactive")
|
|
# The hosts that aren't quiet: named on the rule's row, and they keep
|
|
# its hosts open (#5483).
|
|
uneasy[parent_id] = [k.resource_name for k, lb in zip(kids, labels)
|
|
if k.enabled and lb and lb not in ("inactive", "resolved")]
|
|
fired[parent_id] = max((fired[k.id] for k in kids if k.id in fired), default=None)
|
|
rules = sorted((r for r in rules if not r.parent_id),
|
|
key=lambda r: (not r.enabled, r.name.lower()))
|
|
return await render_template(
|
|
"alerts/list.html", active=active, rules=rules, children=children, uneasy=uneasy,
|
|
rule_state=rule_state, last_fired={k: v for k, v in fired.items() if v},
|
|
notify=await _notify(), operators=_OPERATORS,
|
|
poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60))
|
|
|
|
|
|
@alerts_bp.get("/active")
|
|
@require_role(UserRole.viewer)
|
|
async def active_fragment():
|
|
"""The verdict and Active section alone, which the alerts page polls."""
|
|
async with current_app.db_sessionmaker() as db:
|
|
active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
|
|
# Rules as people made them: a covering rule counts once, not per host.
|
|
rules = [r for r in await alert_svc.list_rules(db) if not r.parent_id]
|
|
return await render_template(
|
|
"alerts/_active.html", active=active, notify=await _notify(), rules_total=len(rules),
|
|
rules_off=sum(1 for r in rules if not r.enabled))
|
|
|
|
|
|
@alerts_bp.get("/widget")
|
|
@require_role(UserRole.viewer)
|
|
async def widget():
|
|
"""The dashboard Alerts widget (#5510): a verdict, what is firing or
|
|
silent, then what someone has taken, a line when alert email is failing,
|
|
and the quiet rules counted."""
|
|
async with current_app.db_sessionmaker() as db:
|
|
active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
|
|
view = alert_svc.widget_view(active, await alert_svc.list_rules(db))
|
|
email = await outbox.alert_trouble(db)
|
|
return await render_template("alerts/widget.html", **view, email=email)
|
|
|
|
|
|
def _delivery(label: str, sent: bool | None, error: str | None) -> dict | None:
|
|
"""A channel's outcome as recorded on the event; None when not tried."""
|
|
if sent is None:
|
|
return None
|
|
return {"label": label, "state": "sent" if sent else "failed", "at": None, "attempts": None,
|
|
"error": None if sent else error}
|
|
|
|
|
|
def _history_row(event, rule, names: dict | None = None, emails: dict | None = None) -> dict:
|
|
"""One transition as the history page shows it. `names` maps user ids to
|
|
usernames: a transition a person made says who (#5463). `emails` is
|
|
outbox.email_states for the page: the email's live state (gathering,
|
|
retrying, sent, failed) where it went through the outbox (#5509), else
|
|
what the event recorded."""
|
|
from steward.core.alerts import MUTED, RELEASED
|
|
from steward.core.formatting import metric_label, metric_value
|
|
notes, muted_by, released = [], None, False
|
|
for part in (event.note or "").split("; "):
|
|
if part.startswith(MUTED):
|
|
muted_by = part[len(MUTED):].strip() or "a maintenance window"
|
|
elif part == RELEASED:
|
|
released = True
|
|
elif part:
|
|
notes.append(part)
|
|
actor = getattr(event, "actor_id", None)
|
|
by = (names or {}).get(actor) or ("a deleted user" if actor else None)
|
|
if event.to_state == "acknowledged":
|
|
notes.insert(0, f"acknowledged by {by}" if by else "acknowledged")
|
|
elif by and notes:
|
|
notes[0] = f"{notes[0]} by {by}"
|
|
metric = rule.metric_name
|
|
if event.metric_value is None:
|
|
value = f"No {metric_label(metric)} reading"
|
|
elif metric in ("is_up", "is_healthy"):
|
|
value = metric_value(metric, event.metric_value).capitalize()
|
|
else:
|
|
value = f"{metric_label(metric)} {metric_value(metric, event.metric_value)}"
|
|
return {
|
|
"at": event.transitioned_at, "rule": rule,
|
|
"from_state": event.from_state, "to_state": event.to_state,
|
|
"value": value, "raw": event.metric_value, "condition": alert_svc.rule_condition(rule),
|
|
"notes": notes, "muted_by": muted_by, "released": released,
|
|
"deliveries": [d for d in (
|
|
{"label": "email", **(emails or {})[event.id]} if event.id in (emails or {})
|
|
else _delivery("email", event.email_sent, event.email_error),
|
|
_delivery("webhook", event.webhook_sent, event.webhook_error)) if d],
|
|
"ansible_run_id": event.ansible_run_id,
|
|
}
|
|
|
|
|
|
@alerts_bp.get("/history")
|
|
@require_role(UserRole.viewer)
|
|
async def alert_history():
|
|
"""Every alert episode in a range, newest first; one rule's with ?rule=,
|
|
which also says how often it fired and for how long in all (#5316). A
|
|
page of transitions at a time as the reader scrolls: ?before=<cursor> is
|
|
the next page's episodes alone (core.paging). The Notified and Action
|
|
columns show only when something in the range was notified or acted on."""
|
|
from steward.alerts.episodes import episodes, summary
|
|
now = datetime.now(timezone.utc)
|
|
since, range_key = parse_range(request.args.get("range") or "7d")
|
|
before = request.args.get("before")
|
|
async with current_app.db_sessionmaker() as db:
|
|
# A covering rule has no history of its own; its hosts' rules do.
|
|
rules = [r for r in await alert_svc.list_rules(db) if r.covers == "resource"]
|
|
chosen = next((r for r in rules if r.id == request.args.get("rule")), None)
|
|
rule_id = chosen.id if chosen else None
|
|
rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before)
|
|
show_notified, show_action = await alert_svc.history_columns(db, since, rule_id)
|
|
names = await alert_svc.usernames(db, (e.actor_id for e, _ in rows))
|
|
emails = await outbox.email_states(db, [e.id for e, _ in rows])
|
|
verdict = None
|
|
if chosen and not before:
|
|
every = await alert_svc.history(db, since, rule_id, HISTORY_SUMMARY_ROWS)
|
|
verdict = summary(episodes([_history_row(e, r) for e, r in every], newest_page=True,
|
|
now=now, since=since))
|
|
rows, after = next_page(rows, lambda row: (row[0].transitioned_at, row[0].id))
|
|
ctx = {"episodes": episodes([_history_row(e, r, names, emails) for e, r in rows],
|
|
newest_page=not before, now=now, since=since),
|
|
"after": after, "rules": rules, "chosen": chosen, "current_range": range_key,
|
|
"show_notified": show_notified, "show_action": show_action, "verdict": verdict}
|
|
if before:
|
|
return await render_template("alerts/_history_page.html", **ctx)
|
|
if request.headers.get("HX-Request"):
|
|
return await render_template("alerts/_history_rows.html", **ctx)
|
|
return await render_template("alerts/history.html", **ctx)
|
|
|
|
|
|
@alerts_bp.get("/anomalies")
|
|
@require_role(UserRole.viewer)
|
|
async def anomalies():
|
|
"""Metric anomalies (#5740): those to review, newest occurrence first,
|
|
then the reviewed and lapsed ones under day dividers. `source` and
|
|
`resource` narrow it to one resource's, its parts included: a resource's
|
|
page links here with them."""
|
|
from urllib.parse import urlencode
|
|
from steward.audit.service import mark_day_groups
|
|
from steward.core import anomalies as anom
|
|
from steward.core import formatting as fmt
|
|
source = request.args.get("source", "").strip() or None
|
|
resource = request.args.get("resource", "").strip() or None
|
|
only = {"source": source, "resource": resource}
|
|
async with current_app.db_sessionmaker() as db:
|
|
to_review = await anom.views(db, await anom.list_anomalies(db, (anom.TO_REVIEW,), **only))
|
|
past = await anom.views(db, await anom.list_anomalies(
|
|
db, (anom.NORMAL, anom.REAL, anom.LAPSED), limit=PAGE_SIZE * 4, **only))
|
|
days = int(await get_setting(db, "anomalies.review_days") or 14)
|
|
here = "/alerts/anomalies"
|
|
if source or resource:
|
|
here += "?" + urlencode({k: v for k, v in only.items() if v})
|
|
tz = fmt.zone(session.get("tz"))
|
|
to_review = [{"v": v} for v in to_review]
|
|
past = [{"v": v} for v in past]
|
|
at = lambda item: item["v"].anomaly.last_started_at # noqa: E731
|
|
mark_day_groups(to_review, tz, at=at)
|
|
mark_day_groups(past, tz, at=at)
|
|
return await render_template("alerts/anomalies.html", to_review=to_review, past=past,
|
|
review_days=days, source=source, resource=resource, here=here)
|
|
|
|
|
|
@alerts_bp.post("/anomalies/<anomaly_id>/review")
|
|
@require_role(UserRole.operator)
|
|
async def review_anomaly(anomaly_id: str):
|
|
"""Mark an anomaly normal (its hours are learned) or real (they are not),
|
|
with an optional note. Audited."""
|
|
from steward.core import anomalies as anom
|
|
form = await request.form
|
|
review = form.get("review", "")
|
|
username = session.get("username", "")
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
try:
|
|
a = await anom.mark(db, anomaly_id, review, username, form.get("note"))
|
|
except LookupError:
|
|
return "Not found", 404
|
|
except ValueError as exc:
|
|
return f"Bad request: {exc}", 400
|
|
detail = {"source": a.source_module, "resource": a.resource_name,
|
|
"metric": a.metric_name, "note": a.note}
|
|
await log_audit(current_app, session.get("user_id"), username,
|
|
f"metric_anomaly.marked_{review}", entity_type="metric_anomaly",
|
|
entity_id=anomaly_id, detail=detail)
|
|
back = form.get("next", "")
|
|
# Only back to a page of this site.
|
|
if not back.startswith("/") or back.startswith("//"):
|
|
back = url_for("alerts.anomalies")
|
|
return redirect(back)
|
|
|
|
|
|
# How many of one rule's transitions the history verdict reads ("fired N
|
|
# times, X in all"); a rule that flapped more often than this in the range
|
|
# is summed over its newest ones.
|
|
HISTORY_SUMMARY_ROWS = 2000
|
|
|
|
|
|
@alerts_bp.get("/api/rule_fields")
|
|
@require_role(UserRole.viewer)
|
|
async def rule_fields_api():
|
|
"""Everything the rule watches, for a newly picked source."""
|
|
source = request.args.get("source_module", "")
|
|
async with current_app.db_sessionmaker() as db:
|
|
ctx = {**await _cover_choices(db, source), **await _target(db, source)}
|
|
return await render_template("alerts/_rule_fields.html", **ctx)
|
|
|
|
|
|
@alerts_bp.get("/api/rule_target")
|
|
@require_role(UserRole.viewer)
|
|
async def rule_target_api():
|
|
"""The host, part and metric pickers and the preview, after a host or a
|
|
part is picked (#5475)."""
|
|
async with current_app.db_sessionmaker() as db:
|
|
ctx = await _target_from_args(db)
|
|
return await render_template("alerts/_rule_target.html", **ctx)
|
|
|
|
|
|
@alerts_bp.get("/api/metric_preview")
|
|
@require_role(UserRole.viewer)
|
|
async def metric_preview_api():
|
|
"""What the picked metric reads now and over the last day (#5475). Its own
|
|
request, after the page: a cold day of one series can take many seconds
|
|
to read (#5485), and the page doesn't wait for it."""
|
|
async with current_app.db_sessionmaker() as db:
|
|
ctx = await _target_from_args(db, with_preview=True)
|
|
return await render_template("alerts/_metric_preview.html", **ctx)
|
|
|
|
|
|
@alerts_bp.get("/api/cover_groups")
|
|
@require_role(UserRole.viewer)
|
|
async def cover_groups_api():
|
|
"""The inventory groups, re-read when the reader comes back from making one."""
|
|
from steward.hosts.service import group_names
|
|
async with current_app.db_sessionmaker() as db:
|
|
groups = await group_names(db)
|
|
return await render_template("alerts/_cover_groups.html", groups=groups,
|
|
current_group=request.args.get("cover_group", ""),
|
|
current_covers=request.args.get("covers", "resource"))
|
|
|
|
|
|
async def _target(db, source: str, *, host: str = "", resource: str = "", metric: str = "",
|
|
covers: str = "resource", keep: bool = False, with_preview: bool = False) -> dict:
|
|
"""The rule form's target pickers (#5475), from the latest-value table;
|
|
`keep` on a rule's own page, so its resource and metric stay on offer.
|
|
The chosen metric's preview only `with_preview`: it reads a day of raw
|
|
samples, so pages and picker swaps leave it to its own request (#5485)."""
|
|
from steward.core.status import sparkline_svg
|
|
target = await alert_svc.rule_target(db, source, host=host, resource=resource, metric=metric,
|
|
keep=keep)
|
|
preview = None
|
|
if with_preview and target["resource"] and target["metric"]:
|
|
preview = await alert_svc.metric_preview(db, source, target["resource"], target["metric"])
|
|
if preview:
|
|
# From zero to the series' own peak (UI & Design ruling 8), so a
|
|
# disk at 61-62% reads flat rather than as a cliff.
|
|
preview["spark"] = sparkline_svg(preview["series"], width=120, height=20,
|
|
stroke="currentColor", floor=1.0)
|
|
return {**target, "preview": preview, "preview_read": with_preview, "current_covers": covers}
|
|
|
|
|
|
async def _target_from_args(db, *, with_preview: bool = False) -> dict:
|
|
a = request.args
|
|
return await _target(db, a.get("source_module", ""), host=a.get("pick_host", ""),
|
|
resource=a.get("resource_name", ""), metric=a.get("metric_name", ""),
|
|
covers=a.get("covers", "resource"), with_preview=with_preview)
|
|
|
|
|
|
async def _cover_choices(db, source: str, rule: AlertRule | None = None) -> dict:
|
|
"""What the rule form offers for covering hosts (#5464): whether this
|
|
source's resources are hosts, the inventory groups, and the rule's own
|
|
choice, fixed once it exists."""
|
|
from steward.hosts.service import group_names
|
|
host_source = source in alert_svc.HOST_SOURCES
|
|
return {"host_source": host_source, "groups": await group_names(db) if host_source else [],
|
|
"current_covers": rule.covers if rule else "resource",
|
|
"current_group": (rule.cover_group or "") if rule else "",
|
|
"covers_locked": rule is not None}
|
|
|
|
|
|
@alerts_bp.get("/rules/new")
|
|
@require_role(UserRole.operator)
|
|
async def new_rule():
|
|
"""The rule form, empty, or aimed at a series when a page links here with
|
|
source_module, resource_name and metric_name (a syslog pattern's count)."""
|
|
sources = alert_svc.source_modules()
|
|
asked = request.args.get("source_module", "")
|
|
first_source = asked if asked in sources else sources[0]
|
|
async with current_app.db_sessionmaker() as db:
|
|
ctx = {**await _cover_choices(db, first_source),
|
|
**(await _target_from_args(db) if asked == first_source else await _target(db, first_source))}
|
|
return await render_template(
|
|
"alerts/rules_form.html",
|
|
rule=None, **ctx,
|
|
operators=_OPERATORS,
|
|
source_modules=alert_svc.source_modules(),
|
|
initial_source=first_source,
|
|
catalog=alert_svc.metric_catalog(),
|
|
ansible_sources=_ansible_source_names(),
|
|
notify=await _notify(),
|
|
)
|
|
|
|
|
|
@alerts_bp.get("/rules/<rule_id>/edit")
|
|
@require_role(UserRole.operator)
|
|
async def edit_rule(rule_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule is None:
|
|
return "Not found", 404
|
|
if rule.parent_id:
|
|
# A host's rule under a covering rule is changed through it.
|
|
return redirect(f"/alerts/rules/{rule.parent_id}/edit")
|
|
ctx = {**await _cover_choices(db, rule.source_module, rule),
|
|
**await _target(db, rule.source_module, resource=rule.resource_name,
|
|
metric=rule.metric_name, covers=rule.covers, keep=True)}
|
|
hosts = sorted((c.resource_name for c in (await db.execute(
|
|
select(AlertRule).where(AlertRule.parent_id == rule.id))).scalars()), key=str.lower)
|
|
history_count = (await db.execute(
|
|
select(func.count()).select_from(AlertEvent).where(AlertEvent.rule_id == rule.id))).scalar_one()
|
|
fired = (await alert_svc.last_fired(db, rule.id)).get(rule.id)
|
|
return await render_template(
|
|
"alerts/rules_form.html",
|
|
rule=rule, history_count=history_count, fired=fired, notify=await _notify(),
|
|
covered=hosts, **ctx,
|
|
operators=_OPERATORS,
|
|
source_modules=alert_svc.source_modules(),
|
|
initial_source=rule.source_module,
|
|
catalog=alert_svc.metric_catalog(),
|
|
ansible_sources=_ansible_source_names(),
|
|
)
|
|
|
|
|
|
@alerts_bp.post("/rules")
|
|
@require_role(UserRole.operator)
|
|
async def create_rule():
|
|
form = await request.form
|
|
user_id = session.get("user_id")
|
|
# Ansible action is admin-only; non-admins simply can't set one.
|
|
ansible_action = None
|
|
if _is_admin():
|
|
ansible_action, err = _parse_ansible_action(form)
|
|
if err:
|
|
return err, 400
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
try:
|
|
fields = await alert_svc.rule_fields(db, **_rule_form(form))
|
|
except ValueError as exc:
|
|
return str(exc), 400
|
|
rule = await alert_svc.add_rule(db, fields, created_by=user_id,
|
|
ansible_action=ansible_action)
|
|
await log_audit(current_app, user_id, session.get("username", ""), "alert_rule.created",
|
|
entity_type="alert_rule", entity_id=rule.name,
|
|
detail={"source": rule.source_module, "resource": rule.resource_name,
|
|
"metric": rule.metric_name})
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>")
|
|
@require_role(UserRole.operator)
|
|
async def update_rule(rule_id: str):
|
|
form = await request.form
|
|
# Validate the admin-only action before opening the transaction. Non-admins
|
|
# leave the existing action untouched (they can still edit everything else).
|
|
apply_action = _is_admin()
|
|
if apply_action:
|
|
ansible_action, err = _parse_ansible_action(form)
|
|
if err:
|
|
return err, 400
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule is None:
|
|
return "Not found", 404
|
|
if (conflict := await _child_conflict(db, rule)):
|
|
return conflict
|
|
try:
|
|
# What a rule covers is fixed once it exists; its group may change.
|
|
fields = await alert_svc.rule_fields(db, **{**_rule_form(form),
|
|
"covers": rule.covers})
|
|
except ValueError as exc:
|
|
return str(exc), 400
|
|
await alert_svc.apply_rule_change(db, rule, fields, actor_id=session.get("user_id"))
|
|
if apply_action:
|
|
rule.ansible_action = ansible_action
|
|
if rule.covers != "resource":
|
|
await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id"))
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"alert_rule.updated", entity_type="alert_rule", entity_id=rule.name)
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>/toggle")
|
|
@require_role(UserRole.operator)
|
|
async def toggle_rule(rule_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule is None:
|
|
return "Not found", 404
|
|
if (conflict := await _child_conflict(db, rule)):
|
|
return conflict
|
|
await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled,
|
|
actor_id=session.get("user_id"))
|
|
if rule.covers != "resource":
|
|
await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id"))
|
|
action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
action, entity_type="alert_rule", entity_id=rule.name)
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>/test")
|
|
@require_role(UserRole.operator)
|
|
async def test_rule(rule_id: str):
|
|
"""Send test (#5466): a TEST notification for this rule, now, by every
|
|
channel its alerts use; the result shows under the button."""
|
|
try:
|
|
sent = await alert_svc.send_test(current_app, rule_id)
|
|
except LookupError:
|
|
return "Not found", 404
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"alert_rule.tested", entity_type="alert_rule", entity_id=sent["rule"],
|
|
detail={"rule_id": rule_id, "ok": sent["ok"], "lines": sent["lines"]})
|
|
return await render_template("alerts/_test_result.html", ok=sent["ok"], lines=sent["lines"])
|
|
|
|
|
|
@alerts_bp.post("/rules/<rule_id>/delete")
|
|
@require_role(UserRole.admin)
|
|
async def delete_rule(rule_id: str):
|
|
rule_name = None
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
|
rule = result.scalar_one_or_none()
|
|
if rule:
|
|
if (conflict := await _child_conflict(db, rule)):
|
|
return conflict
|
|
rule_name = rule.name
|
|
await db.delete(rule)
|
|
if rule_name:
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"alert_rule.deleted", entity_type="alert_rule", entity_id=rule_name)
|
|
return redirect(url_for("alerts.list_alerts"))
|
|
|
|
|
|
@alerts_bp.get("/maintenance")
|
|
@require_role(UserRole.viewer)
|
|
async def list_maintenance():
|
|
now = datetime.now(timezone.utc)
|
|
async with current_app.db_sessionmaker() as db:
|
|
windows = await alert_svc.list_maintenance_windows(db)
|
|
# Active first (ending soonest first), then upcoming (starting soonest
|
|
# first); expired ones are history, newest first, folded away.
|
|
active = sorted((w for w in windows if w.start_at <= now <= w.end_at), key=lambda w: w.end_at)
|
|
upcoming = sorted((w for w in windows if w.start_at > now), key=lambda w: w.start_at)
|
|
expired = sorted((w for w in windows if w.end_at < now), key=lambda w: w.end_at, reverse=True)
|
|
return await render_template(
|
|
"alerts/maintenance.html",
|
|
windows=windows, active=active, upcoming=upcoming, expired=expired,
|
|
now=now,
|
|
source_modules=alert_svc.source_modules(),
|
|
)
|
|
|
|
|
|
@alerts_bp.get("/maintenance/new")
|
|
@require_role(UserRole.operator)
|
|
async def new_maintenance():
|
|
return await render_template(
|
|
"alerts/maintenance_form.html",
|
|
window=None,
|
|
source_modules=alert_svc.source_modules(),
|
|
)
|
|
|
|
|
|
@alerts_bp.post("/maintenance")
|
|
@require_role(UserRole.operator)
|
|
async def create_maintenance():
|
|
form = await request.form
|
|
user_id = session.get("user_id")
|
|
try:
|
|
fields = alert_svc.window_fields(
|
|
name=form.get("name", ""),
|
|
start_at=_form_time(form.get("start_at")), end_at=_form_time(form.get("end_at")),
|
|
scope_source=form.get("scope_source"), scope_resource=form.get("scope_resource"))
|
|
except ValueError as exc:
|
|
return str(exc), 400
|
|
window = MaintenanceWindow(created_by=user_id, **fields)
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
db.add(window)
|
|
scope_desc = alert_svc.window_scope(window)
|
|
await log_audit(current_app, user_id, session.get("username", ""),
|
|
"maintenance_window.created", entity_type="maintenance_window",
|
|
entity_id=window.name, detail={"scope": scope_desc})
|
|
return redirect(url_for("alerts.list_maintenance"))
|
|
|
|
|
|
@alerts_bp.post("/maintenance/<window_id>/delete")
|
|
@require_role(UserRole.operator)
|
|
async def delete_maintenance(window_id: str):
|
|
window_name = None
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
result = await db.execute(
|
|
select(MaintenanceWindow).where(MaintenanceWindow.id == window_id)
|
|
)
|
|
window = result.scalar_one_or_none()
|
|
if window:
|
|
window_name = window.name
|
|
await db.delete(window)
|
|
if window_name:
|
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
|
"maintenance_window.deleted", entity_type="maintenance_window",
|
|
entity_id=window_name)
|
|
return redirect(url_for("alerts.list_maintenance"))
|
|
|
|
|
|
@alerts_bp.post("/<rule_id>/acknowledge")
|
|
@require_role(UserRole.operator)
|
|
async def acknowledge(rule_id: str):
|
|
user_id = session.get("user_id")
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
try:
|
|
await alert_svc.acknowledge(db, rule_id, user_id)
|
|
except LookupError:
|
|
return "Not found", 404
|
|
except ValueError as exc:
|
|
return f"Conflict: {exc}", 409
|
|
return redirect(url_for("alerts.list_alerts"))
|