bvandeusenandClaude Opus 5.5
22090b02d8
Milestone 495 "Email goes to each account, in its own time zone", on the operator's ruling (System 156): notifications go to each user, times in their zone, defaults on, toggled per user or per role. - users.notify_alerts / notify_report (migration 0036): None follows the role's default, notify.<kind>.<role> in settings, all on to start. - mail.pick_recipients: active accounts that want the email and have an address mail can reach (an SSO placeholder like name@oidc is skipped and flagged), each in its own zone with no UTC beside it, then the Also send to addresses in their chosen zone with UTC beside. One copy per address, all over one SMTP connection. - Alerts, the weekly report and the test email (to you, in your zone, plus the typed addresses) send that way. The save-time smtp.tz capture is gone. - Notifications: a role × email table of defaults, who each email reaches and who can't be reached, and Also send to with its own zone, which starts as the first admin's to open the page. - Account page: your address and your two switches. Users: an Emailed column, and an admin sets an account's switches on its page. - The Alerts page counts the accounts it emails. zone_select macro and formatting.zones_by_region shared by Account and Notifications. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
508 lines
23 KiB
Python
508 lines
23 KiB
Python
"""Alert queries and validation shared by the alerts pages and the MCP tools.
|
|
|
|
Each function takes an open session (or plain values), with no Quart request
|
|
context, so the routes and the /mcp tools (which run outside Quart's
|
|
pipeline) call the same code and refuse the same input for the same reasons.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import func, or_, select
|
|
|
|
from steward.models.alerts import (
|
|
AlertEvent, AlertOperator, AlertRule, AlertState, AlertStateEnum, MaintenanceWindow,
|
|
)
|
|
from steward.models.metrics import PluginMetric, PluginMetricLatest
|
|
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# What core itself records. Unified monitors emit metrics under their TYPE
|
|
# (icmp/tcp/dns/http), each with is_up (1.0 up / 0.0 down) and response_ms
|
|
# (where the probe times it); "steward" is the app watching itself.
|
|
CORE_METRICS: dict[str, list[str]] = {
|
|
"icmp": ["is_up", "response_ms"],
|
|
"tcp": ["is_up", "response_ms"],
|
|
"dns": ["is_up", "response_ms"],
|
|
"http": ["is_up", "response_ms"],
|
|
"steward": ["open_fds", "open_fds_pct"],
|
|
}
|
|
|
|
|
|
def metric_catalog() -> dict[str, list[str]]:
|
|
"""Every source an alert rule can watch, and its metric names.
|
|
|
|
Core's own sources, then each loaded plugin's get_alert_metrics() export
|
|
({source: [metric names]}), so the catalog is written by whatever emits
|
|
the metrics and a disabled plugin's sources drop out. An empty list means
|
|
the names are dynamic (snmp: the OID labels recorded so far).
|
|
"""
|
|
from steward.core.plugin_manager import get_loaded_plugin_modules
|
|
catalog = {src: list(names) for src, names in CORE_METRICS.items()}
|
|
for name, module in get_loaded_plugin_modules().items():
|
|
hook = getattr(module, "get_alert_metrics", None)
|
|
if hook is None:
|
|
continue
|
|
try:
|
|
contributed = hook() or {}
|
|
except Exception:
|
|
logger.exception("Plugin %r: get_alert_metrics() raised; its metrics are skipped", name)
|
|
continue
|
|
for src, names in contributed.items():
|
|
catalog[str(src)] = [str(n) for n in names]
|
|
return catalog
|
|
|
|
|
|
def source_modules() -> list[str]:
|
|
"""The sources in picker order: core's first, then plugins'."""
|
|
return list(metric_catalog())
|
|
|
|
|
|
# States that need someone's attention: pending (breaching, not yet for long
|
|
# enough), firing, and acknowledged-but-still-breaching.
|
|
ACTIVE_STATES = (AlertStateEnum.firing, AlertStateEnum.acknowledged, AlertStateEnum.pending)
|
|
|
|
|
|
async def active_alerts(db) -> list:
|
|
"""(AlertState, AlertRule) rows that need attention, latest transition
|
|
first: not yet resolved, or with a series that has gone silent (no_data)."""
|
|
return list((await db.execute(
|
|
select(AlertState, AlertRule)
|
|
.join(AlertRule, AlertState.rule_id == AlertRule.id)
|
|
.where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
|
|
.order_by(AlertState.last_transition_at.desc())
|
|
)).all())
|
|
|
|
|
|
def alert_label(state: AlertState) -> str:
|
|
"""What an alert is, in one word: its state, or "no_data" for a quiet rule
|
|
whose series has gone silent (a firing alert that goes silent stays
|
|
"firing"; check no_data_since for the mark)."""
|
|
if state.no_data_since is not None and state.state == AlertStateEnum.inactive:
|
|
return "no_data"
|
|
return state.state.value
|
|
|
|
|
|
async def latest_value(db, rule: AlertRule) -> float | None:
|
|
"""The newest sample of the series a rule watches, or None before the first
|
|
(a primary-key read of the latest-value table, #5208)."""
|
|
return (await db.execute(
|
|
select(PluginMetricLatest.value)
|
|
.where(PluginMetricLatest.source_module == rule.source_module,
|
|
PluginMetricLatest.resource_name == rule.resource_name,
|
|
PluginMetricLatest.metric_name == rule.metric_name)
|
|
)).scalar_one_or_none()
|
|
|
|
|
|
# Sources whose resources have no page of their own: the plugin's page lists them.
|
|
PLUGIN_PAGES = {"traefik": "/plugins/traefik/", "unifi": "/plugins/unifi/",
|
|
"snmp": "/plugins/snmp/"}
|
|
|
|
|
|
async def resource_url(db, rule: AlertRule) -> str | None:
|
|
"""Where the thing a rule watches is shown, or None when nothing shows it.
|
|
|
|
host_agent resources are the host's name, or "<host>:<mount|nic|core>";
|
|
docker's are "<host>/<container>"; a monitor's is its name under its type.
|
|
"""
|
|
from steward.models.hosts import Host
|
|
source, resource = rule.source_module, rule.resource_name
|
|
if source == "host_agent":
|
|
name = resource.split(":", 1)[0]
|
|
host_id = (await db.execute(
|
|
select(Host.id).where(Host.name == name).limit(1))).scalar_one_or_none()
|
|
return f"/hosts/{host_id}" if host_id else None
|
|
if source in MONITOR_TYPE_VALUES:
|
|
monitor_id = (await db.execute(
|
|
select(Monitor.id).where(Monitor.type == source, Monitor.name == resource).limit(1)
|
|
)).scalar_one_or_none()
|
|
return f"/monitors/{monitor_id}" if monitor_id else None
|
|
if source == "docker" and "/" in resource:
|
|
host_name, container = resource.split("/", 1)
|
|
host_id = (await db.execute(
|
|
select(Host.id).where(Host.name == host_name).limit(1))).scalar_one_or_none()
|
|
return f"/plugins/docker/container/{host_id}/{container}" if host_id else None
|
|
return PLUGIN_PAGES.get(source)
|
|
|
|
|
|
# Metrics whose value is a word: the sentence says the word, not the condition.
|
|
_BINARY = ("is_up", "is_healthy")
|
|
|
|
|
|
def rule_condition(rule: AlertRule) -> str:
|
|
"""What a rule fires on, in words: "CPU > 90%", "Fullest disk >= 85%",
|
|
or "is down" for an up/down metric rather than "is_up < 0.5"."""
|
|
return condition_text(rule.metric_name, rule.operator, rule.threshold)
|
|
|
|
|
|
def condition_text(metric: str, operator: AlertOperator | str, threshold: float) -> str:
|
|
"""rule_condition from a rule's parts, for an alert carried as a dict (an
|
|
email); the operator may be its value, ">=", rather than the enum."""
|
|
from steward.core.alerts import _is_breached
|
|
from steward.core.formatting import metric_label, metric_value
|
|
operator = AlertOperator(operator)
|
|
if metric in _BINARY:
|
|
fires_on = [metric_value(metric, v) for v in (1.0, 0.0)
|
|
if _is_breached(v, operator, threshold)]
|
|
if len(fires_on) == 1:
|
|
return f"is {fires_on[0]}"
|
|
return f"{metric_label(metric)} {operator.value} {metric_value(metric, threshold)}"
|
|
|
|
|
|
def alert_sentence(rule: AlertRule, label: str, value: float | None,
|
|
since: datetime | None, now: datetime) -> dict:
|
|
"""An alert as one sentence around its resource: {"lead", "resource",
|
|
"tail"}, read lead + resource + tail, e.g. "CPU on " + "web-1" +
|
|
" is 94% (> 90%), firing for 12m". label is alert_label(state); since is
|
|
when that label began (no_data_since for no_data)."""
|
|
from steward.core.formatting import human_duration, metric_label, metric_value
|
|
metric = rule.metric_name
|
|
said = metric_label(metric)
|
|
held = human_duration((now - since).total_seconds()) if since else None
|
|
if label == "no_data":
|
|
return {"lead": f"No {said} reading from ", "resource": rule.resource_name,
|
|
"tail": f" for {held}" if held else ""}
|
|
# An acknowledged alert is still breaching: say for how long it has been,
|
|
# not how long ago someone acknowledged it (the page says that apart).
|
|
word = "breaching" if label == "acknowledged" else label
|
|
for_how_long = f", {word} for {held}" if held else f", {word}"
|
|
if value is None:
|
|
reading = " has no reading yet"
|
|
elif metric in _BINARY:
|
|
return {"lead": "", "resource": rule.resource_name,
|
|
"tail": f" is {metric_value(metric, value)}{for_how_long}"}
|
|
else:
|
|
reading = f" is {metric_value(metric, value)}"
|
|
condition = f" ({rule.operator.value} {metric_value(metric, rule.threshold)})"
|
|
return {"lead": f"{said} on ", "resource": rule.resource_name,
|
|
"tail": f"{reading}{condition}{for_how_long}"}
|
|
|
|
|
|
# The order the active list reads in: what is breaching and unhandled, then
|
|
# what has gone silent, then what someone has taken, then what may yet clear.
|
|
ACTIVE_ORDER = {"firing": 0, "no_data": 1, "acknowledged": 2, "pending": 3}
|
|
|
|
_BREACHING = (AlertStateEnum.pending.value, AlertStateEnum.firing.value,
|
|
AlertStateEnum.acknowledged.value)
|
|
|
|
|
|
async def breach_started(db, rule_ids) -> dict[str, datetime]:
|
|
"""{rule_id: when its current breach began}: the newest transition into
|
|
pending or firing from a quiet state. Acknowledging moves an alert's
|
|
last_transition_at, so that column cannot say this."""
|
|
if not rule_ids:
|
|
return {}
|
|
return dict((await db.execute(
|
|
select(AlertEvent.rule_id, func.max(AlertEvent.transitioned_at))
|
|
.where(AlertEvent.rule_id.in_(list(rule_ids)),
|
|
AlertEvent.from_state.not_in(_BREACHING),
|
|
AlertEvent.to_state.in_((AlertStateEnum.pending.value, AlertStateEnum.firing.value)))
|
|
.group_by(AlertEvent.rule_id)
|
|
)).all())
|
|
|
|
|
|
async def active_rows(db, now: datetime) -> list[dict]:
|
|
"""The alerts page's active list, firing first, then no data, then
|
|
acknowledged, then pending; within each the longest-running first. Each
|
|
row: its state, rule, label, sentence, the link to what it watches, when
|
|
the breach began, and who acknowledged it."""
|
|
from steward.models.users import User
|
|
alerts = await active_alerts(db)
|
|
started = await breach_started(db, {rule.id for state, rule in alerts
|
|
if state.state.value in _BREACHING})
|
|
ackers = {s.acknowledged_by for s, _ in alerts if s.acknowledged_by}
|
|
names = dict((await db.execute(
|
|
select(User.id, User.username).where(User.id.in_(ackers)))).all()) if ackers else {}
|
|
rows = []
|
|
for state, rule in alerts:
|
|
label = alert_label(state)
|
|
if label == "no_data":
|
|
since = state.no_data_since
|
|
elif label == "acknowledged":
|
|
since = started.get(rule.id, state.last_transition_at)
|
|
else:
|
|
since = state.last_transition_at
|
|
rows.append({
|
|
"state": state, "rule": rule, "label": label, "since": since,
|
|
"breach_since": started.get(rule.id) if label != "no_data" else None,
|
|
"acked_by": names.get(state.acknowledged_by, state.acknowledged_by),
|
|
"says": alert_sentence(rule, label, await latest_value(db, rule), since, now),
|
|
"url": await resource_url(db, rule),
|
|
})
|
|
epoch = datetime.max.replace(tzinfo=timezone.utc)
|
|
rows.sort(key=lambda r: (ACTIVE_ORDER.get(r["label"], 9), r["since"] or epoch))
|
|
return rows
|
|
|
|
|
|
def watches_host(rule: AlertRule, host_name: str, monitor_keys: set) -> bool:
|
|
"""Whether a rule watches something on this host: its agent series
|
|
("<host>" or "<host>:<part>"), one of its containers ("<host>/<name>"),
|
|
or one of its monitors ((type, name) in monitor_keys)."""
|
|
source, resource = rule.source_module, rule.resource_name
|
|
if source == "host_agent":
|
|
return resource == host_name or resource.startswith(host_name + ":")
|
|
if source == "docker":
|
|
return resource.startswith(host_name + "/")
|
|
return (source, resource) in monitor_keys
|
|
|
|
|
|
async def active_count(db) -> int:
|
|
"""How many alerts need attention: the sidebar's number."""
|
|
from sqlalchemy import func
|
|
return (await db.execute(
|
|
select(func.count()).select_from(AlertState)
|
|
.where(or_(AlertState.state.in_(ACTIVE_STATES), AlertState.no_data_since.is_not(None)))
|
|
)).scalar_one()
|
|
|
|
|
|
def notify_targets(smtp: dict, webhook: dict, accounts: int = 0) -> list[str]:
|
|
"""Where an alert's notifications go, in words, by the same tests
|
|
core.notifications.dispatch_notifications sends by: email when a server
|
|
is set and `accounts` (that get alerts) or Also send to addresses exist,
|
|
a webhook when its URL is. Empty: nobody (#5315, milestone 495)."""
|
|
from urllib.parse import urlsplit
|
|
out = []
|
|
recipients = smtp.get("recipients") or []
|
|
if isinstance(recipients, str):
|
|
recipients = [r.strip() for r in recipients.split(",") if r.strip()]
|
|
who = ([f"{accounts} account{'s' if accounts != 1 else ''}"] if accounts else []) + recipients
|
|
if smtp.get("host") and who:
|
|
out.append("email to " + ", ".join(who))
|
|
if url := webhook.get("url"):
|
|
out.append("a webhook to " + (urlsplit(url).hostname or url))
|
|
return out
|
|
|
|
|
|
async def last_fired(db, rule_id: str | None = None) -> dict[str, datetime]:
|
|
"""{rule id: when it last went to firing}, for the rules that ever have;
|
|
one rule's when `rule_id` is given."""
|
|
stmt = (select(AlertEvent.rule_id, func.max(AlertEvent.transitioned_at))
|
|
.where(AlertEvent.to_state == AlertStateEnum.firing.value)
|
|
.group_by(AlertEvent.rule_id))
|
|
if rule_id:
|
|
stmt = stmt.where(AlertEvent.rule_id == rule_id)
|
|
rows = await db.execute(stmt)
|
|
return {rid: at for rid, at in rows.all()}
|
|
|
|
|
|
async def list_rules(db) -> list[AlertRule]:
|
|
return list((await db.execute(select(AlertRule).order_by(AlertRule.name))).scalars())
|
|
|
|
|
|
async def history(db, since: datetime, rule_id: str | None = None,
|
|
limit: int = 200, before: str | None = None) -> list[tuple[AlertEvent, AlertRule]]:
|
|
"""Alert transitions since `since`, newest first, with their rule; one
|
|
rule's when `rule_id` is given; only those older than the `before` cursor
|
|
when given (core.paging). Returns up to limit+1 rows so a caller can tell
|
|
the list was cut."""
|
|
from steward.core.paging import keyset
|
|
stmt = (select(AlertEvent, AlertRule)
|
|
.join(AlertRule, AlertEvent.rule_id == AlertRule.id)
|
|
.where(AlertEvent.transitioned_at >= since))
|
|
if rule_id:
|
|
stmt = stmt.where(AlertEvent.rule_id == rule_id)
|
|
stmt = keyset(stmt, AlertEvent.transitioned_at, AlertEvent.id, before, limit)
|
|
return [tuple(row) for row in (await db.execute(stmt)).all()]
|
|
|
|
|
|
async def history_columns(db, since: datetime, rule_id: str | None = None) -> tuple[bool, bool]:
|
|
"""(whether any transition in the range notified someone or was muted,
|
|
whether any started an Ansible run): the history page shows its Notified
|
|
and Action columns only then (#5316)."""
|
|
from steward.core.alerts import MUTED
|
|
base = select(AlertEvent.id).where(AlertEvent.transitioned_at >= since)
|
|
if rule_id:
|
|
base = base.where(AlertEvent.rule_id == rule_id)
|
|
notified = await db.scalar(select(base.where(or_(
|
|
AlertEvent.email_sent.is_not(None), AlertEvent.webhook_sent.is_not(None),
|
|
AlertEvent.note.like(f"%{MUTED}%"))).exists()))
|
|
acted = await db.scalar(select(base.where(AlertEvent.ansible_run_id.is_not(None)).exists()))
|
|
return bool(notified), bool(acted)
|
|
|
|
|
|
async def list_maintenance_windows(db) -> list[MaintenanceWindow]:
|
|
"""Every window, latest start first."""
|
|
return list((await db.execute(
|
|
select(MaintenanceWindow).order_by(MaintenanceWindow.start_at.desc())
|
|
)).scalars())
|
|
|
|
|
|
async def resources_for(db, source_module: str) -> list[str]:
|
|
"""Sorted distinct resource names for a source module.
|
|
|
|
For monitor types (icmp/tcp/dns/http), supplements with monitor names so
|
|
the list is populated even before any metrics have been recorded.
|
|
"""
|
|
result = await db.execute(
|
|
select(PluginMetric.resource_name)
|
|
.where(PluginMetric.source_module == source_module)
|
|
.distinct()
|
|
.order_by(PluginMetric.resource_name)
|
|
)
|
|
resources: set[str] = {r for (r,) in result}
|
|
if source_module in MONITOR_TYPE_VALUES:
|
|
mon_result = await db.execute(
|
|
select(Monitor.name).where(Monitor.type == source_module).order_by(Monitor.name))
|
|
resources.update(name for (name,) in mon_result)
|
|
return sorted(resources)
|
|
|
|
|
|
async def metrics_for(db, source_module: str) -> list[str]:
|
|
"""The metric names a rule on this source can watch: the static catalog, or
|
|
for dynamic sources (snmp) the names recorded so far."""
|
|
metrics = metric_catalog().get(source_module, [])
|
|
if source_module and not metrics:
|
|
result = await db.execute(
|
|
select(PluginMetric.metric_name)
|
|
.where(PluginMetric.source_module == source_module)
|
|
.distinct()
|
|
.order_by(PluginMetric.metric_name))
|
|
metrics = [r for (r,) in result]
|
|
return metrics
|
|
|
|
|
|
def _choices(values: list[str], limit: int = 20) -> str:
|
|
if not values:
|
|
return "none yet"
|
|
shown = ", ".join(values[:limit])
|
|
return shown + (f" (and {len(values) - limit} more)" if len(values) > limit else "")
|
|
|
|
|
|
async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
|
|
operator: str, threshold, consecutive_failures_required=1,
|
|
consecutive_ok_required=1, notify_no_data=True) -> dict:
|
|
"""Validated column values for a new or edited alert rule.
|
|
|
|
The same choices the rule form offers: a known source, a resource that
|
|
source has reported (or, for monitors, a monitor of that type), and a
|
|
metric it emits. ValueError says what was wrong and what is valid.
|
|
"""
|
|
name = (name or "").strip()
|
|
if not name:
|
|
raise ValueError("an alert rule needs a name.")
|
|
source_module = (source_module or "").strip()
|
|
catalog = metric_catalog()
|
|
if source_module not in catalog:
|
|
raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.")
|
|
resource_name = (resource_name or "").strip()
|
|
resources = await resources_for(db, source_module)
|
|
if resource_name not in resources:
|
|
raise ValueError(f"{source_module} has no resource {resource_name!r}. "
|
|
f"Known: {_choices(resources)}.")
|
|
metric_name = (metric_name or "").strip()
|
|
metrics = await metrics_for(db, source_module)
|
|
if metric_name not in metrics:
|
|
raise ValueError(f"{source_module} does not report {metric_name!r}. "
|
|
f"Its metrics: {_choices(metrics)}.")
|
|
try:
|
|
op = AlertOperator((operator or "").strip())
|
|
except ValueError:
|
|
raise ValueError(f"operator {operator!r} is not one of "
|
|
f"{', '.join(o.value for o in AlertOperator)}.") from None
|
|
try:
|
|
threshold = float(threshold)
|
|
except (TypeError, ValueError):
|
|
raise ValueError(f"threshold must be a number, got {threshold!r}.") from None
|
|
try:
|
|
consecutive = (1 if consecutive_failures_required in (None, "")
|
|
else int(consecutive_failures_required))
|
|
except (TypeError, ValueError):
|
|
raise ValueError("consecutive checks must be a whole number.") from None
|
|
if consecutive < 1:
|
|
raise ValueError("consecutive checks must be at least 1.")
|
|
try:
|
|
ok_checks = (1 if consecutive_ok_required in (None, "")
|
|
else int(consecutive_ok_required))
|
|
except (TypeError, ValueError):
|
|
raise ValueError("checks to clear must be a whole number.") from None
|
|
if ok_checks < 1:
|
|
raise ValueError("checks to clear must be at least 1.")
|
|
return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
|
|
"metric_name": metric_name, "operator": op, "threshold": threshold,
|
|
"consecutive_failures_required": consecutive,
|
|
"consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data)}
|
|
|
|
|
|
# Changing any of these changes what the rule is watching, so whatever state
|
|
# it had built up no longer means anything.
|
|
WATCHED_FIELDS = ("source_module", "resource_name", "metric_name", "operator", "threshold")
|
|
|
|
|
|
async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool | None = None) -> None:
|
|
"""Apply validated fields (and optionally enabled) to a rule, in the
|
|
caller's transaction. A change to what it watches, or disabling or
|
|
re-enabling it, puts its alert back to inactive and starts its silence
|
|
watch over (core.alerts.reset_rule_state)."""
|
|
from steward.core.alerts import reset_rule_state
|
|
changed = [k for k in WATCHED_FIELDS if k in fields and getattr(rule, k) != fields[k]]
|
|
for k, v in fields.items():
|
|
setattr(rule, k, v)
|
|
disabling = enabled is False and rule.enabled
|
|
enabling = enabled is True and not rule.enabled
|
|
if enabled is not None:
|
|
rule.enabled = bool(enabled)
|
|
if disabling:
|
|
await reset_rule_state(db, rule, "rule disabled")
|
|
elif enabling:
|
|
await reset_rule_state(db, rule, "rule enabled")
|
|
elif changed:
|
|
await reset_rule_state(db, rule, "rule changed: " + ", ".join(changed))
|
|
|
|
|
|
async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
|
|
ansible_action: dict | None = None) -> AlertRule:
|
|
"""Insert a rule and its inactive state row, in the caller's transaction."""
|
|
rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
|
|
db.add(rule)
|
|
await db.flush()
|
|
db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
|
|
last_transition_at=datetime.now(timezone.utc)))
|
|
return rule
|
|
|
|
|
|
async def acknowledge(db, rule_id: str, user_id: str) -> AlertState:
|
|
"""Acknowledge a firing alert, in the caller's transaction.
|
|
|
|
LookupError when the rule has no state; ValueError when it is not firing.
|
|
"""
|
|
state = (await db.execute(
|
|
select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one_or_none()
|
|
if state is None:
|
|
raise LookupError(rule_id)
|
|
if state.state != AlertStateEnum.firing:
|
|
raise ValueError(f"the alert is {state.state.value}, not firing; only a firing alert "
|
|
"can be acknowledged.")
|
|
now = datetime.now(timezone.utc)
|
|
state.state = AlertStateEnum.acknowledged
|
|
state.acknowledged_by = user_id
|
|
state.acknowledged_at = now
|
|
state.last_transition_at = now
|
|
return state
|
|
|
|
|
|
def window_fields(*, name: str, start_at: datetime, end_at: datetime,
|
|
scope_source: str | None, scope_resource: str | None) -> dict:
|
|
"""Validated columns for a maintenance window. A resource needs a source."""
|
|
name = (name or "").strip()
|
|
if not name:
|
|
raise ValueError("a maintenance window needs a name.")
|
|
if end_at <= start_at:
|
|
raise ValueError("the window must end after it starts.")
|
|
scope_source = (scope_source or "").strip() or None
|
|
scope_resource = (scope_resource or "").strip() or None
|
|
catalog = metric_catalog()
|
|
if scope_source is not None and scope_source not in catalog:
|
|
raise ValueError(f"source {scope_source!r} is not one of {', '.join(catalog)}.")
|
|
if scope_source is None:
|
|
scope_resource = None # can't have resource without source
|
|
return {"name": name[:128], "start_at": start_at, "end_at": end_at,
|
|
"scope_source": scope_source, "scope_resource": scope_resource}
|
|
|
|
|
|
def window_scope(w) -> str:
|
|
if not w.scope_source:
|
|
return "all"
|
|
return f"{w.scope_source}/{w.scope_resource}" if w.scope_resource else f"{w.scope_source}/*"
|