feat(alerts): renames carry their alerts — monitor and host renames move rules, windows and history (#5065)
#5065 "Renames carry their alerts". Alert rules, maintenance-window scopes and metric history key a thing by (source, name), so a rename used to leave the rules watching a name nothing reported under, and split the history. - core/renames.carry_rename moves AlertRule.resource_name, MaintenanceWindow.scope_resource, plugin_metrics and plugin_metrics_hourly to the new name in the caller's transaction. A host rename also carries its parts: host_agent "<host>:…" and docker "<host>/…" (HOST_SOURCES). Matching is exact (a prefix compare, not LIKE or a collation range). History left under the new name by something deleted is dropped first rather than merged in. - Web monitor edit, MCP update_monitor and web host edit call it; MCP answers with `carried` (it used to return a warning naming stranded rules). - A name another monitor of the same type, or another host, already has is refused on create and rename: two would share one history. - The rename is audited (monitor.updated / host.renamed) with what moved. Tests: an integration test renames a firing monitor through the web route and finds its rule still firing and cleared by a sample on the new name, its window and raw+hourly history moved and an orphan's history dropped; a host rename moves agent and container parts but not a longer-named host's or another source's; a guard that every host_agent resource is "<host>" or "<host>:…". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
"""steward/core/renames.py
|
||||
|
||||
Alert rules, maintenance-window scopes and metric history all key a thing
|
||||
by (source, name). When a monitor or host is renamed, carry_rename moves
|
||||
every one of them to the new name, in the caller's transaction, so the
|
||||
thing's rules keep watching it and its history stays in one piece (#5065).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import and_, delete, func, or_, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from steward.models.alerts import AlertRule, MaintenanceWindow
|
||||
from steward.models.metrics import PluginMetric, PluginMetricHourly
|
||||
|
||||
# What a host's name keys, and how each source names a part of the host:
|
||||
# host_agent "web01:/var", "web01:core3"; docker "web01/nginx".
|
||||
HOST_SOURCES: tuple[tuple[str, str | None], ...] = (("host_agent", ":"), ("docker", "/"))
|
||||
|
||||
|
||||
def _matches(source_col, name_col, sources, name: str):
|
||||
"""(source, name) itself, or a sub-resource "<name><sep>…" of it."""
|
||||
clauses = []
|
||||
for source, sep in sources:
|
||||
under = [name_col == name]
|
||||
if sep:
|
||||
# An exact prefix: LIKE would read % or _ in a name as wildcards, and
|
||||
# a >=/< range follows the collation, which may ignore punctuation.
|
||||
under.append(func.left(name_col, len(name) + len(sep)) == name + sep)
|
||||
clauses.append(and_(source_col == source, or_(*under)))
|
||||
return or_(*clauses)
|
||||
|
||||
|
||||
def _renamed(name_col, old: str, new: str):
|
||||
return func.concat(new, func.substr(name_col, len(old) + 1))
|
||||
|
||||
|
||||
async def carry_rename(session: AsyncSession, sources, old: str, new: str) -> dict[str, int]:
|
||||
"""Point everything keyed by `old` under each (source, separator) in
|
||||
`sources` at `new`. A separator also carries the sub-resources named
|
||||
"<old><sep>…"; None carries the exact name only.
|
||||
|
||||
The caller has already refused a `new` that a live thing holds, so any
|
||||
history under `new` belongs to something deleted: it is dropped first,
|
||||
rather than merged into this thing's history. Returns rows moved, by kind.
|
||||
"""
|
||||
if not old or old == new:
|
||||
return {}
|
||||
sources = tuple(sources)
|
||||
for model in (PluginMetric, PluginMetricHourly):
|
||||
await session.execute(delete(model).where(
|
||||
_matches(model.source_module, model.resource_name, sources, new)))
|
||||
moved = {}
|
||||
for kind, model, src, res in (
|
||||
("rules", AlertRule, AlertRule.source_module, AlertRule.resource_name),
|
||||
("maintenance_windows", MaintenanceWindow, MaintenanceWindow.scope_source,
|
||||
MaintenanceWindow.scope_resource),
|
||||
("metric_rows", PluginMetric, PluginMetric.source_module, PluginMetric.resource_name),
|
||||
("hourly_rows", PluginMetricHourly, PluginMetricHourly.source_module,
|
||||
PluginMetricHourly.resource_name),
|
||||
):
|
||||
result = await session.execute(
|
||||
update(model).where(_matches(src, res, sources, old))
|
||||
.values({res: _renamed(res, old, new)})
|
||||
.execution_options(synchronize_session=False))
|
||||
moved[kind] = result.rowcount
|
||||
return moved
|
||||
|
||||
|
||||
def describe(moved: dict[str, int]) -> str:
|
||||
"""'2 alert rules, 1 maintenance window and 18,204 metric samples', or ''."""
|
||||
parts = []
|
||||
for kind, one, many in (("rules", "alert rule", "alert rules"),
|
||||
("maintenance_windows", "maintenance window", "maintenance windows")):
|
||||
if moved.get(kind):
|
||||
parts.append(f"{moved[kind]} {one if moved[kind] == 1 else many}")
|
||||
samples = moved.get("metric_rows", 0) + moved.get("hourly_rows", 0)
|
||||
if samples:
|
||||
parts.append(f"{samples:,} metric sample{'' if samples == 1 else 's'}")
|
||||
if len(parts) > 1:
|
||||
return ", ".join(parts[:-1]) + " and " + parts[-1]
|
||||
return "".join(parts)
|
||||
+19
-2
@@ -5,7 +5,8 @@ from quart import Blueprint, render_template, request, redirect, url_for, curren
|
||||
from sqlalchemy import select
|
||||
from steward.ansible import executor, sources as ansible_src
|
||||
from steward.auth.middleware import require_role
|
||||
from steward.core.audit import log_audit
|
||||
from steward.core import renames
|
||||
from steward.core.audit import audit_request, log_audit
|
||||
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.users import UserRole
|
||||
@@ -107,6 +108,10 @@ async def create_host():
|
||||
)
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
async with db.begin():
|
||||
try:
|
||||
await host_svc.check_name_free(db, host.name)
|
||||
except ValueError as exc:
|
||||
return str(exc), 400
|
||||
db.add(host)
|
||||
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
||||
"host.created", entity_type="host", entity_id=host.name,
|
||||
@@ -139,8 +144,20 @@ async def update_host(host_id: str):
|
||||
host = result.scalar_one_or_none()
|
||||
if host is None:
|
||||
return "Not found", 404
|
||||
host.name = form["name"].strip()
|
||||
old_name, name = host.name, form["name"].strip()
|
||||
if name != old_name:
|
||||
try:
|
||||
await host_svc.check_name_free(db, name, exclude_id=host.id)
|
||||
except ValueError as exc:
|
||||
return str(exc), 400
|
||||
host.name = name
|
||||
host.address = form["address"].strip()
|
||||
# Agent and container metrics, and the rules and windows on them,
|
||||
# go by the host's name: move them with it.
|
||||
moved = await renames.carry_rename(db, renames.HOST_SOURCES, old_name, name)
|
||||
if moved:
|
||||
await audit_request("host.renamed", "host", name,
|
||||
{"host_id": host_id, "renamed_from": old_name, "carried": moved})
|
||||
return redirect(url_for("hosts.host_detail", host_id=host_id))
|
||||
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ from datetime import datetime, timedelta, timezone
|
||||
from sqlalchemy import and_, case, func, select
|
||||
|
||||
from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.monitors import Monitor, MonitorResult
|
||||
|
||||
|
||||
@@ -18,6 +19,21 @@ from steward.models.monitors import Monitor, MonitorResult
|
||||
AGENT_GLANCE_METRICS = ("cpu_pct", "mem_used_pct", "disk_used_pct", "disk_used_pct_worst")
|
||||
|
||||
|
||||
async def check_name_free(db, name: str, exclude_id: str | None = None) -> None:
|
||||
"""ValueError when another host already has this name, or it is blank.
|
||||
|
||||
Agent metrics, container metrics and the alert rules on them go by the
|
||||
host's name, so two hosts sharing one would share one history.
|
||||
"""
|
||||
if not name:
|
||||
raise ValueError("a host needs a name.")
|
||||
q = select(Host.id).where(Host.name == name)
|
||||
if exclude_id:
|
||||
q = q.where(Host.id != exclude_id)
|
||||
if (await db.execute(q.limit(1))).first():
|
||||
raise ValueError(f"another host is already named {name!r}; pick a different name.")
|
||||
|
||||
|
||||
def agent_stale_after_seconds(config) -> int:
|
||||
"""The host_agent plugin's freshness window from app config (default 180s)."""
|
||||
return int(((config.get("PLUGINS") or {}).get("host_agent") or {})
|
||||
|
||||
@@ -5,7 +5,7 @@ import json
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from steward.models.alerts import AlertRule
|
||||
from steward.core import renames
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
|
||||
from steward.models.users import UserRole
|
||||
@@ -188,6 +188,7 @@ async def create_monitor(name: str, type: str, target: str, host: str = "",
|
||||
fields = monitor_svc.monitor_fields(type, target=target, name=name,
|
||||
check_interval_seconds=check_interval_s, opts=config or {})
|
||||
async with db_session() as db, db.begin():
|
||||
await monitor_svc.check_name_free(db, fields["type"], fields["name"])
|
||||
m = Monitor(host_id=await _host_id(db, host), enabled=enabled, **fields)
|
||||
db.add(m)
|
||||
_monitors, host_names = await _load(db)
|
||||
@@ -207,8 +208,9 @@ async def update_monitor(monitor: str, name: str | None = None, target: str | No
|
||||
create_monitor for the keys of each type). A monitor's type cannot
|
||||
change; delete it and create another.
|
||||
|
||||
Alert rules watch a monitor by its name, so a rename returns `warning`
|
||||
naming the rules still on the old name.
|
||||
A rename carries the monitor's alert rules, maintenance windows and
|
||||
check history to the new name; `carried` says what moved. A name another
|
||||
monitor of the same type has is refused.
|
||||
|
||||
Recorded in the audit log.
|
||||
"""
|
||||
@@ -223,23 +225,23 @@ async def update_monitor(monitor: str, name: str | None = None, target: str | No
|
||||
else check_interval_s),
|
||||
opts={**current, **(config or {})})
|
||||
old_name = m.name
|
||||
if fields["name"] != old_name:
|
||||
await monitor_svc.check_name_free(db, m.type, fields["name"], exclude_id=m.id)
|
||||
for k, v in fields.items():
|
||||
setattr(m, k, v)
|
||||
if host is not None:
|
||||
m.host_id = await _host_id(db, host)
|
||||
stranded = []
|
||||
if m.name != old_name:
|
||||
stranded = list((await db.execute(
|
||||
select(AlertRule.name).where(AlertRule.source_module == m.type,
|
||||
AlertRule.resource_name == old_name))).scalars())
|
||||
moved = await renames.carry_rename(db, ((m.type, None),), old_name, m.name)
|
||||
changed = sorted(k for k, v in {"name": name, "target": target, "host": host, "config": config,
|
||||
"check_interval_s": check_interval_s}.items() if v is not None)
|
||||
await audit_mcp("monitor.updated", "monitor", m.name, {"monitor_id": m.id, "changed": changed})
|
||||
detail = {"monitor_id": m.id, "changed": changed}
|
||||
if moved:
|
||||
detail["renamed_from"] = old_name
|
||||
detail["carried"] = moved
|
||||
await audit_mcp("monitor.updated", "monitor", m.name, detail)
|
||||
out = _saved(m, host_names)
|
||||
if stranded:
|
||||
out["warning"] = (f"Alert rules watch this monitor by name and still watch {old_name!r}: "
|
||||
f"{', '.join(stranded)}. update_alert_rule(resource=\"{m.name}\") "
|
||||
"points each at the new name.")
|
||||
if moved:
|
||||
out["carried"] = {"from": old_name, **moved}
|
||||
return out
|
||||
|
||||
|
||||
|
||||
@@ -11,6 +11,8 @@ from datetime import datetime, timezone
|
||||
from quart import Blueprint, current_app, render_template, request, redirect, url_for
|
||||
|
||||
from steward.auth.middleware import require_role
|
||||
from steward.core import renames
|
||||
from steward.core.audit import audit_request
|
||||
from steward.models.users import UserRole
|
||||
from steward.models.monitors import Monitor, MonitorType, MONITOR_TYPE_VALUES
|
||||
from steward.core.time_range import DEFAULT_RANGE
|
||||
@@ -81,6 +83,10 @@ async def add_monitor():
|
||||
created_at=datetime.now(timezone.utc), **fields)
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
async with db.begin():
|
||||
try:
|
||||
await monitor_svc.check_name_free(db, monitor.type, monitor.name)
|
||||
except ValueError as exc:
|
||||
return str(exc), 400
|
||||
db.add(monitor)
|
||||
|
||||
# A host-scoped add returns to that host's hub; otherwise the monitors page.
|
||||
@@ -117,10 +123,18 @@ async def edit_monitor(monitor_id: str):
|
||||
m.type, target=form.get("target", m.target), name=form.get("name", ""),
|
||||
check_interval_seconds=form.get("check_interval_seconds"),
|
||||
opts=_form_opts(form, m.type))
|
||||
if fields["name"] != m.name:
|
||||
await monitor_svc.check_name_free(db, m.type, fields["name"], exclude_id=m.id)
|
||||
except ValueError as exc:
|
||||
return str(exc), 400
|
||||
old_name = m.name
|
||||
for k, v in fields.items():
|
||||
setattr(m, k, v)
|
||||
# Its rules, windows and history go by (type, name): move them with it.
|
||||
moved = await renames.carry_rename(db, ((m.type, None),), old_name, m.name)
|
||||
if moved:
|
||||
await audit_request("monitor.updated", "monitor", m.name,
|
||||
{"monitor_id": m.id, "renamed_from": old_name, "carried": moved})
|
||||
return redirect(url_for("monitors.index"))
|
||||
|
||||
|
||||
|
||||
@@ -114,6 +114,20 @@ def monitor_fields(mtype: str, *, target: str, name: str = "", check_interval_se
|
||||
}
|
||||
|
||||
|
||||
async def check_name_free(db, mtype: str, name: str, exclude_id: str | None = None) -> None:
|
||||
"""ValueError when another monitor of this type already has this name.
|
||||
|
||||
A monitor's checks are recorded, and its alert rules watch it, by
|
||||
(type, name), so two monitors sharing both would share one history.
|
||||
"""
|
||||
q = select(Monitor.id).where(Monitor.type == mtype, Monitor.name == name)
|
||||
if exclude_id:
|
||||
q = q.where(Monitor.id != exclude_id)
|
||||
if (await db.execute(q.limit(1))).first():
|
||||
raise ValueError(f"another {mtype} monitor is already named {name!r}; its checks and "
|
||||
"alert rules go by that name, so pick a different one.")
|
||||
|
||||
|
||||
async def monitor_rows(db, range_key: str):
|
||||
"""Per-monitor state: latest result, heartbeat strip, uptime and TLS days.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user