feat(alerts): renames carry their alerts — monitor and host renames move rules, windows and history (#5065)
CI / lint (push) Successful in 3s
CI / integration (push) Successful in 50s
CI / unit (push) Successful in 1m6s
CI / publish (push) Successful in 1m2s

#5065 "Renames carry their alerts". Alert rules, maintenance-window scopes
and metric history key a thing by (source, name), so a rename used to leave
the rules watching a name nothing reported under, and split the history.

- core/renames.carry_rename moves AlertRule.resource_name,
  MaintenanceWindow.scope_resource, plugin_metrics and plugin_metrics_hourly
  to the new name in the caller's transaction. A host rename also carries
  its parts: host_agent "<host>:…" and docker "<host>/…" (HOST_SOURCES).
  Matching is exact (a prefix compare, not LIKE or a collation range).
  History left under the new name by something deleted is dropped first
  rather than merged in.
- Web monitor edit, MCP update_monitor and web host edit call it; MCP
  answers with `carried` (it used to return a warning naming stranded rules).
- A name another monitor of the same type, or another host, already has is
  refused on create and rename: two would share one history.
- The rename is audited (monitor.updated / host.renamed) with what moved.

Tests: an integration test renames a firing monitor through the web route
and finds its rule still firing and cleared by a sample on the new name,
its window and raw+hourly history moved and an orphan's history dropped; a
host rename moves agent and container parts but not a longer-named host's
or another source's; a guard that every host_agent resource is "<host>" or
"<host>:…".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-06 14:02:43 -04:00
co-authored by Claude Opus 5.5
parent 88d34a3827
commit c9f5c049db
9 changed files with 355 additions and 21 deletions
+82
View File
@@ -0,0 +1,82 @@
"""steward/core/renames.py
Alert rules, maintenance-window scopes and metric history all key a thing
by (source, name). When a monitor or host is renamed, carry_rename moves
every one of them to the new name, in the caller's transaction, so the
thing's rules keep watching it and its history stays in one piece (#5065).
"""
from __future__ import annotations
from sqlalchemy import and_, delete, func, or_, update
from sqlalchemy.ext.asyncio import AsyncSession
from steward.models.alerts import AlertRule, MaintenanceWindow
from steward.models.metrics import PluginMetric, PluginMetricHourly
# What a host's name keys, and how each source names a part of the host:
# host_agent "web01:/var", "web01:core3"; docker "web01/nginx".
HOST_SOURCES: tuple[tuple[str, str | None], ...] = (("host_agent", ":"), ("docker", "/"))
def _matches(source_col, name_col, sources, name: str):
"""(source, name) itself, or a sub-resource "<name><sep>…" of it."""
clauses = []
for source, sep in sources:
under = [name_col == name]
if sep:
# An exact prefix: LIKE would read % or _ in a name as wildcards, and
# a >=/< range follows the collation, which may ignore punctuation.
under.append(func.left(name_col, len(name) + len(sep)) == name + sep)
clauses.append(and_(source_col == source, or_(*under)))
return or_(*clauses)
def _renamed(name_col, old: str, new: str):
return func.concat(new, func.substr(name_col, len(old) + 1))
async def carry_rename(session: AsyncSession, sources, old: str, new: str) -> dict[str, int]:
"""Point everything keyed by `old` under each (source, separator) in
`sources` at `new`. A separator also carries the sub-resources named
"<old><sep>…"; None carries the exact name only.
The caller has already refused a `new` that a live thing holds, so any
history under `new` belongs to something deleted: it is dropped first,
rather than merged into this thing's history. Returns rows moved, by kind.
"""
if not old or old == new:
return {}
sources = tuple(sources)
for model in (PluginMetric, PluginMetricHourly):
await session.execute(delete(model).where(
_matches(model.source_module, model.resource_name, sources, new)))
moved = {}
for kind, model, src, res in (
("rules", AlertRule, AlertRule.source_module, AlertRule.resource_name),
("maintenance_windows", MaintenanceWindow, MaintenanceWindow.scope_source,
MaintenanceWindow.scope_resource),
("metric_rows", PluginMetric, PluginMetric.source_module, PluginMetric.resource_name),
("hourly_rows", PluginMetricHourly, PluginMetricHourly.source_module,
PluginMetricHourly.resource_name),
):
result = await session.execute(
update(model).where(_matches(src, res, sources, old))
.values({res: _renamed(res, old, new)})
.execution_options(synchronize_session=False))
moved[kind] = result.rowcount
return moved
def describe(moved: dict[str, int]) -> str:
"""'2 alert rules, 1 maintenance window and 18,204 metric samples', or ''."""
parts = []
for kind, one, many in (("rules", "alert rule", "alert rules"),
("maintenance_windows", "maintenance window", "maintenance windows")):
if moved.get(kind):
parts.append(f"{moved[kind]} {one if moved[kind] == 1 else many}")
samples = moved.get("metric_rows", 0) + moved.get("hourly_rows", 0)
if samples:
parts.append(f"{samples:,} metric sample{'' if samples == 1 else 's'}")
if len(parts) > 1:
return ", ".join(parts[:-1]) + " and " + parts[-1]
return "".join(parts)
+19 -2
View File
@@ -5,7 +5,8 @@ from quart import Blueprint, render_template, request, redirect, url_for, curren
from sqlalchemy import select
from steward.ansible import executor, sources as ansible_src
from steward.auth.middleware import require_role
from steward.core.audit import log_audit
from steward.core import renames
from steward.core.audit import audit_request, log_audit
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
from steward.models.hosts import Host
from steward.models.users import UserRole
@@ -107,6 +108,10 @@ async def create_host():
)
async with current_app.db_sessionmaker() as db:
async with db.begin():
try:
await host_svc.check_name_free(db, host.name)
except ValueError as exc:
return str(exc), 400
db.add(host)
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
"host.created", entity_type="host", entity_id=host.name,
@@ -139,8 +144,20 @@ async def update_host(host_id: str):
host = result.scalar_one_or_none()
if host is None:
return "Not found", 404
host.name = form["name"].strip()
old_name, name = host.name, form["name"].strip()
if name != old_name:
try:
await host_svc.check_name_free(db, name, exclude_id=host.id)
except ValueError as exc:
return str(exc), 400
host.name = name
host.address = form["address"].strip()
# Agent and container metrics, and the rules and windows on them,
# go by the host's name: move them with it.
moved = await renames.carry_rename(db, renames.HOST_SOURCES, old_name, name)
if moved:
await audit_request("host.renamed", "host", name,
{"host_id": host_id, "renamed_from": old_name, "carried": moved})
return redirect(url_for("hosts.host_detail", host_id=host_id))
+16
View File
@@ -10,6 +10,7 @@ from datetime import datetime, timedelta, timezone
from sqlalchemy import and_, case, func, select
from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor
from steward.models.hosts import Host
from steward.models.monitors import Monitor, MonitorResult
@@ -18,6 +19,21 @@ from steward.models.monitors import Monitor, MonitorResult
AGENT_GLANCE_METRICS = ("cpu_pct", "mem_used_pct", "disk_used_pct", "disk_used_pct_worst")
async def check_name_free(db, name: str, exclude_id: str | None = None) -> None:
"""ValueError when another host already has this name, or it is blank.
Agent metrics, container metrics and the alert rules on them go by the
host's name, so two hosts sharing one would share one history.
"""
if not name:
raise ValueError("a host needs a name.")
q = select(Host.id).where(Host.name == name)
if exclude_id:
q = q.where(Host.id != exclude_id)
if (await db.execute(q.limit(1))).first():
raise ValueError(f"another host is already named {name!r}; pick a different name.")
def agent_stale_after_seconds(config) -> int:
"""The host_agent plugin's freshness window from app config (default 180s)."""
return int(((config.get("PLUGINS") or {}).get("host_agent") or {})
+15 -13
View File
@@ -5,7 +5,7 @@ import json
from sqlalchemy import select
from steward.models.alerts import AlertRule
from steward.core import renames
from steward.models.hosts import Host
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
from steward.models.users import UserRole
@@ -188,6 +188,7 @@ async def create_monitor(name: str, type: str, target: str, host: str = "",
fields = monitor_svc.monitor_fields(type, target=target, name=name,
check_interval_seconds=check_interval_s, opts=config or {})
async with db_session() as db, db.begin():
await monitor_svc.check_name_free(db, fields["type"], fields["name"])
m = Monitor(host_id=await _host_id(db, host), enabled=enabled, **fields)
db.add(m)
_monitors, host_names = await _load(db)
@@ -207,8 +208,9 @@ async def update_monitor(monitor: str, name: str | None = None, target: str | No
create_monitor for the keys of each type). A monitor's type cannot
change; delete it and create another.
Alert rules watch a monitor by its name, so a rename returns `warning`
naming the rules still on the old name.
A rename carries the monitor's alert rules, maintenance windows and
check history to the new name; `carried` says what moved. A name another
monitor of the same type has is refused.
Recorded in the audit log.
"""
@@ -223,23 +225,23 @@ async def update_monitor(monitor: str, name: str | None = None, target: str | No
else check_interval_s),
opts={**current, **(config or {})})
old_name = m.name
if fields["name"] != old_name:
await monitor_svc.check_name_free(db, m.type, fields["name"], exclude_id=m.id)
for k, v in fields.items():
setattr(m, k, v)
if host is not None:
m.host_id = await _host_id(db, host)
stranded = []
if m.name != old_name:
stranded = list((await db.execute(
select(AlertRule.name).where(AlertRule.source_module == m.type,
AlertRule.resource_name == old_name))).scalars())
moved = await renames.carry_rename(db, ((m.type, None),), old_name, m.name)
changed = sorted(k for k, v in {"name": name, "target": target, "host": host, "config": config,
"check_interval_s": check_interval_s}.items() if v is not None)
await audit_mcp("monitor.updated", "monitor", m.name, {"monitor_id": m.id, "changed": changed})
detail = {"monitor_id": m.id, "changed": changed}
if moved:
detail["renamed_from"] = old_name
detail["carried"] = moved
await audit_mcp("monitor.updated", "monitor", m.name, detail)
out = _saved(m, host_names)
if stranded:
out["warning"] = (f"Alert rules watch this monitor by name and still watch {old_name!r}: "
f"{', '.join(stranded)}. update_alert_rule(resource=\"{m.name}\") "
"points each at the new name.")
if moved:
out["carried"] = {"from": old_name, **moved}
return out
+14
View File
@@ -11,6 +11,8 @@ from datetime import datetime, timezone
from quart import Blueprint, current_app, render_template, request, redirect, url_for
from steward.auth.middleware import require_role
from steward.core import renames
from steward.core.audit import audit_request
from steward.models.users import UserRole
from steward.models.monitors import Monitor, MonitorType, MONITOR_TYPE_VALUES
from steward.core.time_range import DEFAULT_RANGE
@@ -81,6 +83,10 @@ async def add_monitor():
created_at=datetime.now(timezone.utc), **fields)
async with current_app.db_sessionmaker() as db:
async with db.begin():
try:
await monitor_svc.check_name_free(db, monitor.type, monitor.name)
except ValueError as exc:
return str(exc), 400
db.add(monitor)
# A host-scoped add returns to that host's hub; otherwise the monitors page.
@@ -117,10 +123,18 @@ async def edit_monitor(monitor_id: str):
m.type, target=form.get("target", m.target), name=form.get("name", ""),
check_interval_seconds=form.get("check_interval_seconds"),
opts=_form_opts(form, m.type))
if fields["name"] != m.name:
await monitor_svc.check_name_free(db, m.type, fields["name"], exclude_id=m.id)
except ValueError as exc:
return str(exc), 400
old_name = m.name
for k, v in fields.items():
setattr(m, k, v)
# Its rules, windows and history go by (type, name): move them with it.
moved = await renames.carry_rename(db, ((m.type, None),), old_name, m.name)
if moved:
await audit_request("monitor.updated", "monitor", m.name,
{"monitor_id": m.id, "renamed_from": old_name, "carried": moved})
return redirect(url_for("monitors.index"))
+14
View File
@@ -114,6 +114,20 @@ def monitor_fields(mtype: str, *, target: str, name: str = "", check_interval_se
}
async def check_name_free(db, mtype: str, name: str, exclude_id: str | None = None) -> None:
"""ValueError when another monitor of this type already has this name.
A monitor's checks are recorded, and its alert rules watch it, by
(type, name), so two monitors sharing both would share one history.
"""
q = select(Monitor.id).where(Monitor.type == mtype, Monitor.name == name)
if exclude_id:
q = q.where(Monitor.id != exclude_id)
if (await db.execute(q.limit(1))).first():
raise ValueError(f"another {mtype} monitor is already named {name!r}; its checks and "
"alert rules go by that name, so pick a different one.")
async def monitor_rows(db, range_key: str):
"""Per-monitor state: latest result, heartbeat strip, uptime and TLS days.
@@ -1,7 +1,7 @@
"""Integration: the monitoring-config write tools against real Postgres.
A monitor is created on a host, edited (config merged, a rename warning for
the rule that watches it by name), paused and deleted; a rule is created on
A monitor is created on a host, edited (config merged, a rename that carries
the rule watching it by name), paused and deleted; a rule is created on
that monitor, edited, acknowledged once firing, and deleted; a maintenance
window in progress is ended and an upcoming one cancelled. Every write leaves
an audit row as "<user> via MCP".
@@ -95,11 +95,15 @@ def test_monitor_rule_ack_and_maintenance_writes(app):
row = await s.get(Monitor, made["monitor_id"])
assert row.config["headers"] == {"Authorization": "Bearer s3cret"}
# A rename leaves the rule on the old name, and says so.
# A rename takes the rule with it, and says what moved.
renamed = await m.update_monitor(mon_name, name=f"{mon_name}-2")
assert rule_name in renamed["warning"]
await a.update_alert_rule(rule_name, resource=f"{mon_name}-2")
assert "warning" not in await m.update_monitor(f"{mon_name}-2", check_interval_s=30)
assert renamed["carried"]["from"] == mon_name and renamed["carried"]["rules"] == 1
async with app.db_sessionmaker() as s:
assert (await s.get(AlertRule, rule["rule_id"])).resource_name == f"{mon_name}-2"
assert "carried" not in await m.update_monitor(f"{mon_name}-2", check_interval_s=30)
# Another http monitor may not take the same name: they would share one history.
with pytest.raises(ValueError, match="already named"):
await m.create_monitor(f"{mon_name}-2", "http", "other.lan")
assert (await m.set_monitor_enabled(f"{mon_name}-2", False))["enabled"] is False
# ── acknowledge ────────────────────────────────────────────────────
+165
View File
@@ -0,0 +1,165 @@
"""Integration: a renamed monitor or host takes its alert rules, maintenance
windows and metric history with it (#5065), through the web edit routes,
against real Postgres."""
from __future__ import annotations
import asyncio
import os
import time
import uuid
from datetime import datetime, timedelta, timezone
import pytest
pytestmark = pytest.mark.integration
@pytest.fixture
def app():
if not os.environ.get("STEWARD_DATABASE_URL"):
pytest.skip("needs Postgres")
from steward.app import create_app
return create_app(testing=False)
async def _operator(app, tag):
from steward.models.users import User, UserRole
uid = str(uuid.uuid4())
async with app.db_sessionmaker() as s:
async with s.begin():
s.add(User(id=uid, username=f"ren-{tag}", email=f"ren-{tag}@test",
password_hash="x", role=UserRole.operator, is_active=True))
client = app.test_client()
async with client.session_transaction() as sess:
sess.update(user_id=uid, user_role="operator", username=f"ren-{tag}",
_csrf="test-token", login_at=time.time())
return client, uid
CSRF = {"X-CSRF-Token": "test-token"}
def test_a_renamed_monitor_keeps_its_firing_rule_window_and_history(app):
from sqlalchemy import func, select
from steward.alerts import service as alert_svc
from steward.core.alerts import record_metric
from steward.models.alerts import AlertRule, AlertState, AlertStateEnum, MaintenanceWindow
from steward.models.metrics import PluginMetric, PluginMetricHourly
from steward.models.monitors import Monitor
async def _go():
tag = uuid.uuid4().hex[:8]
old, new = f"svc-{tag}", f"svc-{tag}-renamed"
client, uid = await _operator(app, tag)
mid = str(uuid.uuid4())
now = datetime.now(timezone.utc)
async with app.db_sessionmaker() as s:
async with s.begin():
s.add(Monitor(id=mid, name=old, type="tcp", target="10.9.9.9", config_json='{"port": 22}'))
s.add(Monitor(id=str(uuid.uuid4()), name=f"other-{tag}", type="tcp", target="10.9.9.8",
config_json='{"port": 22}'))
# History under the old name, and an orphan's under the new one.
s.add(PluginMetric(source_module="tcp", resource_name=old, metric_name="is_up", value=1))
s.add(PluginMetricHourly(source_module="tcp", resource_name=old, metric_name="is_up",
bucket=now.replace(minute=0, second=0, microsecond=0) - timedelta(days=3),
value_avg=1, value_max=1, sample_count=60))
s.add(PluginMetric(source_module="tcp", resource_name=new, metric_name="is_up", value=0,
recorded_at=now - timedelta(days=30)))
await s.flush()
fields = await alert_svc.rule_fields(
s, name=f"down {tag}", source_module="tcp", resource_name=old,
metric_name="is_up", operator="<", threshold=1,
consecutive_failures_required=1, consecutive_ok_required=1)
rule = await alert_svc.add_rule(s, fields, created_by=uid)
win = MaintenanceWindow(name=f"later {tag}", scope_source="tcp", scope_resource=old,
created_by=uid, start_at=now + timedelta(days=1), end_at=now + timedelta(days=1, hours=1))
s.add(win)
rule_id, win_id = rule.id, win.id
async def sample(name, v):
async with app.db_sessionmaker() as s:
async with s.begin():
await record_metric(s, "tcp", name, "is_up", v)
async def state():
async with app.db_sessionmaker() as s:
return (await s.execute(select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one()
await sample(old, 0)
assert (await state()).state == AlertStateEnum.firing
# Renaming to a name another tcp monitor has is refused, and moves nothing.
resp = await client.post(f"/monitors/{mid}/edit", headers=CSRF, form={
"name": f"other-{tag}", "target": "10.9.9.9", "port": "22"})
assert resp.status_code == 400 and "already named" in await resp.get_data(as_text=True)
resp = await client.post(f"/monitors/{mid}/edit", headers=CSRF, form={
"name": new, "target": "10.9.9.9", "port": "22"})
assert resp.status_code == 302
async with app.db_sessionmaker() as s:
assert (await s.get(AlertRule, rule_id)).resource_name == new
assert (await s.get(MaintenanceWindow, win_id)).scope_resource == new
raw = (await s.execute(select(PluginMetric.value, PluginMetric.resource_name).where(
PluginMetric.source_module == "tcp",
PluginMetric.resource_name.in_([old, new])))).all()
hourly = (await s.execute(select(func.count()).select_from(PluginMetricHourly).where(
PluginMetricHourly.source_module == "tcp",
PluginMetricHourly.resource_name == new))).scalar_one()
# Old samples (1 and the 0 that fired it) moved; the orphan's 30-day-old 0 was dropped.
assert {r for _, r in raw} == {new} and sorted(v for v, _ in raw) == [0, 1]
assert hourly == 1
# Still firing, and now watching the new name: one OK sample clears it.
assert (await state()).state == AlertStateEnum.firing
await sample(old, 1) # the old name is no one's now
assert (await state()).state == AlertStateEnum.firing
await sample(new, 1)
assert (await state()).state == AlertStateEnum.inactive
asyncio.run(_go())
def test_a_renamed_host_carries_agent_and_container_metrics_and_their_rules(app):
from sqlalchemy import select
from steward.alerts import service as alert_svc
from steward.models.alerts import AlertRule
from steward.models.hosts import Host
from steward.models.metrics import PluginMetric
async def _go():
tag = uuid.uuid4().hex[:8]
old, new = f"h{tag}", f"h{tag}-new"
client, uid = await _operator(app, tag)
hid = str(uuid.uuid4())
rows = [("host_agent", old), ("host_agent", f"{old}:/"), ("host_agent", f"{old}:core0"),
("docker", f"{old}/nginx"),
# Not this host's: a longer name sharing the prefix, and a different source.
("host_agent", f"{old}x:/"), ("snmp", old)]
async with app.db_sessionmaker() as s:
async with s.begin():
s.add(Host(id=hid, name=old, address="10.8.8.8"))
s.add(Host(id=str(uuid.uuid4()), name=f"{old}x", address="10.8.8.9"))
for src, res in rows:
s.add(PluginMetric(source_module=src, resource_name=res, metric_name="disk_used_pct", value=1))
await s.flush()
fields = await alert_svc.rule_fields(
s, name=f"disk {tag}", source_module="host_agent", resource_name=f"{old}:/",
metric_name="disk_used_pct", operator=">", threshold=90,
consecutive_failures_required=1, consecutive_ok_required=1)
rule = await alert_svc.add_rule(s, fields, created_by=uid)
rule_id = rule.id
resp = await client.post(f"/hosts/{hid}", headers=CSRF, form={"name": f"{old}x", "address": "10.8.8.8"})
assert resp.status_code == 400
resp = await client.post(f"/hosts/{hid}", headers=CSRF, form={"name": new, "address": "10.8.8.8"})
assert resp.status_code == 302
async with app.db_sessionmaker() as s:
got = set((await s.execute(select(PluginMetric.source_module, PluginMetric.resource_name).where(
PluginMetric.resource_name.like(f"h{tag}%")))).all())
assert (await s.get(AlertRule, rule_id)).resource_name == f"{new}:/"
assert got == {("host_agent", new), ("host_agent", f"{new}:/"), ("host_agent", f"{new}:core0"),
("docker", f"{new}/nginx"), ("host_agent", f"{old}x:/"), ("snmp", old)}
asyncio.run(_go())
@@ -58,3 +58,23 @@ def test_expand_is_backcompat_with_old_agent_sample():
n.startswith(("net_", "disk_read", "disk_write", "psi_", "temp_", "mem_cached"))
for n in names
)
def test_every_resource_is_the_host_or_a_host_colon_part():
"""A host rename carries "<host>" and "<host>:…" (steward/core/renames.py
HOST_SOURCES); a resource named any other way would stay behind."""
from steward.core.renames import HOST_SOURCES
sample = {
"ts": "x", "cpu_pct": 1.0, "cpu_cores": [1.0],
"mem": {"total_bytes": 10, "available_bytes": 5, "cached_bytes": 1, "buffers_bytes": 1},
"load": {"1m": 0.1}, "uptime_secs": 1,
"storage": [{"mount": "/", "total_bytes": 10, "used_bytes": 5}],
"net": [{"iface": "eth0", "rx_bps": 1.0, "tx_bps": 1.0}],
"diskio": [{"device": "sda", "read_bps": 1.0, "write_bps": 1.0}],
"temps": [{"label": "Package", "celsius": 40.0}],
"psi": {"cpu_some_avg10": 0.0},
}
sep = dict(HOST_SOURCES)["host_agent"]
resources = {r.resource_name for r in _expand_sample_to_metrics(sample, "web01", datetime.now(timezone.utc))}
assert len(resources) > 5
assert {r for r in resources if r != "web01" and not r.startswith("web01" + sep)} == set()