feat(alerts): one rule can cover every host, or every host in a group (#5464)
#5464 "Group rules: one rule covers every host, or every host in a group", step 2 of milestone 499 "Alerts you can trust". Each rule named one host, so the live instance carried six rules for three hosts, and a new host had no agent-offline or disk alert until someone copied them. - alert_rules.covers / cover_group / parent_id (migration 0041). A rule on host_agent may cover "all_hosts" or a "group" (an inventory group: the Hosts ruling, #5294). A covering rule has no state; Steward keeps one ordinary child rule per covered host, named "<rule>: <host>", so each host's alert keeps its own state, history and acknowledgement. - alerts.service.reconcile_children: a covered host that has reported in the last 7 days gets a child; a host no longer covered loses its child; children take the parent's condition, checks, notify setting, action and on/off. Run on save and each minute (alert_reconcile). - A child is changed through its parent: edit, toggle and delete of a child are refused (409 / PartOfCoveringRule); its edit page opens the parent. - Rule form: a Covers choice for host sources, with the group picker; fixed once the rule exists. Rules list: a covering rule reads "every host in swarm · 3 hosts" with its hosts' rules nested under it, each opening its history. The history picker and the rule count leave covering rules out. - MCP: create_alert_rule takes covers/group; update_alert_rule takes group and passes changes down; list_alert_rules shows covers, hosts and part_of. - docs/core/alerting.md: "Rules that cover hosts". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -75,6 +75,29 @@ Alert rules are created in the UI at `/alerts/`. Each rule requires:
|
||||
- **Threshold** — numeric value
|
||||
- **Consecutive failures required** — how many consecutive breaches before FIRING (default 1)
|
||||
|
||||
### Rules that cover hosts
|
||||
|
||||
A rule on `host_agent` can cover **every host**, or **every host in an
|
||||
inventory group**, instead of naming one resource (#5464). Inventory groups
|
||||
are Steward's grouping system: a host is in a group through its linked
|
||||
Ansible target.
|
||||
|
||||
A covering rule has no state of its own. Steward keeps one ordinary rule per
|
||||
covered host under it, named `<rule>: <host>`, so each host's alert keeps its
|
||||
own state, history and acknowledgement. Each minute (the `alert_reconcile`
|
||||
task), and whenever the rule is saved:
|
||||
|
||||
- a host that has reported in the last 7 days, and is covered, gets a rule;
|
||||
- a host that is no longer covered loses its rule, and that rule's history;
|
||||
- the hosts' rules take the covering rule's condition, checks, notification
|
||||
setting, action and on/off.
|
||||
|
||||
A host's rule is changed through its covering rule; editing, disabling or
|
||||
deleting it alone is refused. What a rule covers is fixed once it exists, but
|
||||
a group rule can move to another group. Over MCP, `create_alert_rule` takes
|
||||
`covers` ("all_hosts" or "group") and `group`, and `list_alert_rules` shows
|
||||
each covering rule's `hosts` and each host's rule's `part_of`.
|
||||
|
||||
### Available Metrics by Module
|
||||
|
||||
| `source_module` | `metric_name` | Description |
|
||||
|
||||
@@ -36,6 +36,8 @@ def _rule_form(form) -> dict:
|
||||
"consecutive_failures_required": form.get("consecutive_failures_required"),
|
||||
"consecutive_ok_required": form.get("consecutive_ok_required"),
|
||||
"notify_no_data": "notify_no_data" in form,
|
||||
"covers": form.get("covers") or "resource",
|
||||
"cover_group": form.get("cover_group", ""),
|
||||
}
|
||||
|
||||
|
||||
@@ -47,6 +49,15 @@ def _form_time(value) -> datetime:
|
||||
raise ValueError(f"{value!r} is not a date and time.") from None
|
||||
|
||||
|
||||
async def _child_conflict(db, rule: AlertRule):
|
||||
"""A 409 for a host's rule under a covering rule, which is changed
|
||||
through its parent (#5464); None for any other rule."""
|
||||
if not rule.parent_id:
|
||||
return None
|
||||
parent = await db.get(AlertRule, rule.parent_id)
|
||||
return str(alert_svc.PartOfCoveringRule(rule, parent.name if parent else "?")), 409
|
||||
|
||||
|
||||
def _is_admin() -> bool:
|
||||
return session.get("user_role") == UserRole.admin.value
|
||||
|
||||
@@ -123,11 +134,27 @@ async def list_alerts():
|
||||
states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
|
||||
fired = await alert_svc.last_fired(db)
|
||||
|
||||
# Every rule with what it is doing now; disabled rules last.
|
||||
rules.sort(key=lambda r: (not r.enabled, r.name.lower()))
|
||||
# Every rule with what it is doing now; disabled rules last. A covering
|
||||
# rule's per-host children sit under it (#5464), and it reads as the most
|
||||
# pressing of their states.
|
||||
rule_state = {r.id: alert_svc.alert_label(states[r.id]) for r in rules if r.id in states}
|
||||
children: dict[str, list] = {}
|
||||
for r in rules:
|
||||
if r.parent_id:
|
||||
children.setdefault(r.parent_id, []).append(r)
|
||||
for kids in children.values():
|
||||
kids.sort(key=lambda r: r.resource_name.lower())
|
||||
parent_id = kids[0].parent_id
|
||||
labels = [rule_state.get(k.id) for k in kids]
|
||||
rule_state[parent_id] = min(
|
||||
(lb for lb in labels if lb and lb not in ("inactive", "resolved")),
|
||||
key=lambda lb: alert_svc.ACTIVE_ORDER.get(lb, 9), default="inactive")
|
||||
fired[parent_id] = max((fired[k.id] for k in kids if k.id in fired), default=None)
|
||||
rules = sorted((r for r in rules if not r.parent_id),
|
||||
key=lambda r: (not r.enabled, r.name.lower()))
|
||||
return await render_template(
|
||||
"alerts/list.html", active=active, rules=rules, rule_state=rule_state, last_fired=fired,
|
||||
"alerts/list.html", active=active, rules=rules, children=children,
|
||||
rule_state=rule_state, last_fired={k: v for k, v in fired.items() if v},
|
||||
notify=await _notify(), operators=_OPERATORS,
|
||||
poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60))
|
||||
|
||||
@@ -138,7 +165,8 @@ async def active_fragment():
|
||||
"""The verdict and Active section alone, which the alerts page polls."""
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
|
||||
rules = await alert_svc.list_rules(db)
|
||||
# Rules as people made them: a covering rule counts once, not per host.
|
||||
rules = [r for r in await alert_svc.list_rules(db) if not r.parent_id]
|
||||
return await render_template(
|
||||
"alerts/_active.html", active=active, notify=await _notify(), rules_total=len(rules),
|
||||
rules_off=sum(1 for r in rules if not r.enabled))
|
||||
@@ -195,7 +223,8 @@ async def alert_history():
|
||||
since, range_key = parse_range(request.args.get("range") or "7d")
|
||||
before = request.args.get("before")
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
rules = await alert_svc.list_rules(db)
|
||||
# A covering rule has no history of its own; its hosts' rules do.
|
||||
rules = [r for r in await alert_svc.list_rules(db) if r.covers == "resource"]
|
||||
chosen = next((r for r in rules if r.id == request.args.get("rule")), None)
|
||||
rule_id = chosen.id if chosen else None
|
||||
rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before)
|
||||
@@ -232,26 +261,40 @@ async def rule_fields_api():
|
||||
current_metric = request.args.get("current_metric", "")
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
resources = await alert_svc.resources_for(db, source) if source else []
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
metrics = await alert_svc.metrics_for(db, source)
|
||||
cover = await _cover_choices(db, source)
|
||||
return await render_template(
|
||||
"alerts/_rule_fields.html",
|
||||
resources=resources,
|
||||
metrics=metrics,
|
||||
current_resource=current_resource,
|
||||
current_metric=current_metric,
|
||||
**cover,
|
||||
)
|
||||
|
||||
|
||||
async def _cover_choices(db, source: str, rule: AlertRule | None = None) -> dict:
|
||||
"""What the rule form offers for covering hosts (#5464): whether this
|
||||
source's resources are hosts, the inventory groups, and the rule's own
|
||||
choice, fixed once it exists."""
|
||||
from steward.hosts.service import group_names
|
||||
host_source = source in alert_svc.HOST_SOURCES
|
||||
return {"host_source": host_source, "groups": await group_names(db) if host_source else [],
|
||||
"current_covers": rule.covers if rule else "resource",
|
||||
"current_group": (rule.cover_group or "") if rule else "",
|
||||
"covers_locked": rule is not None}
|
||||
|
||||
|
||||
@alerts_bp.get("/rules/new")
|
||||
@require_role(UserRole.operator)
|
||||
async def new_rule():
|
||||
first_source = alert_svc.source_modules()[0]
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
resources = await alert_svc.resources_for(db, first_source)
|
||||
cover = await _cover_choices(db, first_source)
|
||||
return await render_template(
|
||||
"alerts/rules_form.html",
|
||||
rule=None,
|
||||
rule=None, **cover,
|
||||
operators=_OPERATORS,
|
||||
source_modules=alert_svc.source_modules(),
|
||||
initial_source=first_source,
|
||||
@@ -271,13 +314,20 @@ async def edit_rule(rule_id: str):
|
||||
rule = result.scalar_one_or_none()
|
||||
if rule is None:
|
||||
return "Not found", 404
|
||||
if rule.parent_id:
|
||||
# A host's rule under a covering rule is changed through it.
|
||||
return redirect(f"/alerts/rules/{rule.parent_id}/edit")
|
||||
resources = await alert_svc.resources_for(db, rule.source_module)
|
||||
cover = await _cover_choices(db, rule.source_module, rule)
|
||||
hosts = sorted((c.resource_name for c in (await db.execute(
|
||||
select(AlertRule).where(AlertRule.parent_id == rule.id))).scalars()), key=str.lower)
|
||||
history_count = (await db.execute(
|
||||
select(func.count()).select_from(AlertEvent).where(AlertEvent.rule_id == rule.id))).scalar_one()
|
||||
fired = (await alert_svc.last_fired(db, rule.id)).get(rule.id)
|
||||
return await render_template(
|
||||
"alerts/rules_form.html",
|
||||
rule=rule, history_count=history_count, fired=fired, notify=await _notify(),
|
||||
covered=hosts, **cover,
|
||||
operators=_OPERATORS,
|
||||
source_modules=alert_svc.source_modules(),
|
||||
initial_source=rule.source_module,
|
||||
@@ -331,13 +381,19 @@ async def update_rule(rule_id: str):
|
||||
rule = result.scalar_one_or_none()
|
||||
if rule is None:
|
||||
return "Not found", 404
|
||||
if (conflict := await _child_conflict(db, rule)):
|
||||
return conflict
|
||||
try:
|
||||
fields = await alert_svc.rule_fields(db, **_rule_form(form))
|
||||
# What a rule covers is fixed once it exists; its group may change.
|
||||
fields = await alert_svc.rule_fields(db, **{**_rule_form(form),
|
||||
"covers": rule.covers})
|
||||
except ValueError as exc:
|
||||
return str(exc), 400
|
||||
await alert_svc.apply_rule_change(db, rule, fields, actor_id=session.get("user_id"))
|
||||
if apply_action:
|
||||
rule.ansible_action = ansible_action
|
||||
if rule.covers != "resource":
|
||||
await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id"))
|
||||
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
||||
"alert_rule.updated", entity_type="alert_rule", entity_id=rule.name)
|
||||
return redirect(url_for("alerts.list_alerts"))
|
||||
@@ -352,8 +408,12 @@ async def toggle_rule(rule_id: str):
|
||||
rule = result.scalar_one_or_none()
|
||||
if rule is None:
|
||||
return "Not found", 404
|
||||
if (conflict := await _child_conflict(db, rule)):
|
||||
return conflict
|
||||
await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled,
|
||||
actor_id=session.get("user_id"))
|
||||
if rule.covers != "resource":
|
||||
await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id"))
|
||||
action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
|
||||
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
||||
action, entity_type="alert_rule", entity_id=rule.name)
|
||||
@@ -369,6 +429,8 @@ async def delete_rule(rule_id: str):
|
||||
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
||||
rule = result.scalar_one_or_none()
|
||||
if rule:
|
||||
if (conflict := await _child_conflict(db, rule)):
|
||||
return conflict
|
||||
rule_name = rule.name
|
||||
await db.delete(rule)
|
||||
if rule_name:
|
||||
|
||||
+150
-9
@@ -378,12 +378,16 @@ def _choices(values: list[str], limit: int = 20) -> str:
|
||||
|
||||
async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
|
||||
operator: str, threshold, consecutive_failures_required=1,
|
||||
consecutive_ok_required=1, notify_no_data=True) -> dict:
|
||||
consecutive_ok_required=1, notify_no_data=True,
|
||||
covers: str = "resource", cover_group: str = "") -> dict:
|
||||
"""Validated column values for a new or edited alert rule.
|
||||
|
||||
The same choices the rule form offers: a known source, a resource that
|
||||
source has reported (or, for monitors, a monitor of that type), and a
|
||||
metric it emits. ValueError says what was wrong and what is valid.
|
||||
metric it emits. A covering rule (`covers` "all_hosts", or "group" with
|
||||
an inventory group's name in `cover_group`, #5464) names no resource: it
|
||||
is for a source whose resources are hosts. ValueError says what was wrong
|
||||
and what is valid.
|
||||
"""
|
||||
name = (name or "").strip()
|
||||
if not name:
|
||||
@@ -392,11 +396,29 @@ async def rule_fields(db, *, name: str, source_module: str, resource_name: str,
|
||||
catalog = metric_catalog()
|
||||
if source_module not in catalog:
|
||||
raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.")
|
||||
resource_name = (resource_name or "").strip()
|
||||
resources = await resources_for(db, source_module)
|
||||
if resource_name not in resources:
|
||||
raise ValueError(f"{source_module} has no resource {resource_name!r}. "
|
||||
f"Known: {_choices(resources)}.")
|
||||
covers = (covers or "resource").strip()
|
||||
if covers not in COVERS:
|
||||
raise ValueError(f"covers {covers!r} is not one of {', '.join(COVERS)}.")
|
||||
cover_group = (cover_group or "").strip()
|
||||
if covers == "resource":
|
||||
resource_name, cover_group = (resource_name or "").strip(), None
|
||||
resources = await resources_for(db, source_module)
|
||||
if resource_name not in resources:
|
||||
raise ValueError(f"{source_module} has no resource {resource_name!r}. "
|
||||
f"Known: {_choices(resources)}.")
|
||||
else:
|
||||
if source_module not in HOST_SOURCES:
|
||||
raise ValueError(f"only a rule on {', '.join(HOST_SOURCES)} can cover hosts; "
|
||||
f"{source_module}'s resources are not hosts.")
|
||||
resource_name = ""
|
||||
if covers == "group":
|
||||
from steward.hosts.service import group_names
|
||||
groups = await group_names(db)
|
||||
if cover_group not in groups:
|
||||
raise ValueError(f"there is no inventory group {cover_group!r}. "
|
||||
f"Groups: {_choices(groups)}.")
|
||||
else:
|
||||
cover_group = None
|
||||
metric_name = (metric_name or "").strip()
|
||||
metrics = await metrics_for(db, source_module)
|
||||
if metric_name not in metrics:
|
||||
@@ -428,7 +450,8 @@ async def rule_fields(db, *, name: str, source_module: str, resource_name: str,
|
||||
return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
|
||||
"metric_name": metric_name, "operator": op, "threshold": threshold,
|
||||
"consecutive_failures_required": consecutive,
|
||||
"consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data)}
|
||||
"consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data),
|
||||
"covers": covers, "cover_group": cover_group}
|
||||
|
||||
|
||||
# Changing any of these changes what the rule is watching, so whatever state
|
||||
@@ -460,15 +483,133 @@ async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool
|
||||
|
||||
async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
|
||||
ansible_action: dict | None = None) -> AlertRule:
|
||||
"""Insert a rule and its inactive state row, in the caller's transaction."""
|
||||
"""Insert a rule and its inactive state row, in the caller's transaction.
|
||||
A covering rule gets no state; its per-host children are made here."""
|
||||
rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
|
||||
db.add(rule)
|
||||
await db.flush()
|
||||
if rule.covers != "resource":
|
||||
await reconcile_children(db, rule)
|
||||
return rule
|
||||
db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
|
||||
last_transition_at=datetime.now(timezone.utc)))
|
||||
return rule
|
||||
|
||||
|
||||
# ── Covering rules (#5464) ───────────────────────────────────────────────────
|
||||
# A rule can cover every host, or every host in an inventory group (inventory
|
||||
# groups are Steward's grouping system: the Hosts ruling, #5294). Steward keeps
|
||||
# one ordinary child rule per covered host, so each host's alert has its own
|
||||
# state, history and acknowledgement, and a host that appears later is covered
|
||||
# without anyone copying the rule.
|
||||
|
||||
COVERS = ("resource", "all_hosts", "group")
|
||||
# Sources whose bare resource names are hosts ("<host>", parts are "<host>:…").
|
||||
HOST_SOURCES = ("host_agent",)
|
||||
# A host counts as present while it has reported within this long: the raw
|
||||
# metrics window, so a decommissioned host's children go once its data does.
|
||||
COVER_SEEN_DAYS = 7
|
||||
|
||||
|
||||
def covers_text(rule: AlertRule) -> str:
|
||||
"""What a covering rule spans, in words: "every host", "every host in web"."""
|
||||
if rule.covers == "group":
|
||||
return f"every host in {rule.cover_group}"
|
||||
return "every host" if rule.covers == "all_hosts" else rule.resource_name
|
||||
|
||||
|
||||
async def covered_hosts(db, rule: AlertRule, now: datetime | None = None) -> list[str]:
|
||||
"""The host names a covering rule spans now: hosts that have reported
|
||||
under its source within COVER_SEEN_DAYS, all of them or those whose
|
||||
linked Ansible target is in its group."""
|
||||
from datetime import timedelta
|
||||
from steward.hosts.service import host_groups
|
||||
from steward.models.hosts import Host
|
||||
now = now or datetime.now(timezone.utc)
|
||||
reported = set((await db.execute(
|
||||
select(PluginMetricLatest.resource_name)
|
||||
.where(PluginMetricLatest.source_module == rule.source_module,
|
||||
PluginMetricLatest.recorded_at >= now - timedelta(days=COVER_SEEN_DAYS),
|
||||
~PluginMetricLatest.resource_name.contains(":"))
|
||||
.distinct())).scalars())
|
||||
if rule.covers != "group":
|
||||
return sorted(reported, key=str.lower)
|
||||
hosts = (await db.execute(select(Host.id, Host.name).where(Host.name.in_(reported)))).all()
|
||||
groups = await host_groups(db, [hid for hid, _ in hosts])
|
||||
return sorted((name for hid, name in hosts if rule.cover_group in groups.get(hid, [])),
|
||||
key=str.lower)
|
||||
|
||||
|
||||
def child_fields(parent: AlertRule, host: str) -> dict:
|
||||
"""A covering rule's child for one host: the parent's condition on that
|
||||
host, named "<parent>: <host>"."""
|
||||
return {"name": f"{parent.name}: {host}"[:128], "source_module": parent.source_module,
|
||||
"resource_name": host, "metric_name": parent.metric_name,
|
||||
"operator": parent.operator, "threshold": parent.threshold,
|
||||
"consecutive_failures_required": parent.consecutive_failures_required,
|
||||
"consecutive_ok_required": parent.consecutive_ok_required,
|
||||
"notify_no_data": parent.notify_no_data}
|
||||
|
||||
|
||||
async def reconcile_children(db, parent: AlertRule, *, actor_id: str | None = None,
|
||||
now: datetime | None = None) -> tuple[list[str], list[str]]:
|
||||
"""Bring a covering rule's children in line with it, in the caller's
|
||||
transaction: one per covered host, each carrying the parent's condition,
|
||||
action and on/off. A host newly covered gets a child; a child whose host
|
||||
is no longer covered is deleted, its history with it. Returns (hosts
|
||||
added, hosts removed)."""
|
||||
hosts = set(await covered_hosts(db, parent, now))
|
||||
children = {c.resource_name: c for c in (await db.execute(
|
||||
select(AlertRule).where(AlertRule.parent_id == parent.id))).scalars()}
|
||||
added = sorted(hosts - children.keys(), key=str.lower)
|
||||
removed = sorted(children.keys() - hosts, key=str.lower)
|
||||
for host in added:
|
||||
await add_rule(db, {**child_fields(parent, host), "parent_id": parent.id},
|
||||
created_by=parent.created_by, enabled=parent.enabled,
|
||||
ansible_action=parent.ansible_action)
|
||||
for host in removed:
|
||||
await db.delete(children[host])
|
||||
for host, child in children.items():
|
||||
if host in hosts:
|
||||
fields = child_fields(parent, host)
|
||||
if (any(getattr(child, k) != v for k, v in fields.items())
|
||||
or child.enabled != parent.enabled):
|
||||
await apply_rule_change(db, child, fields, enabled=parent.enabled,
|
||||
actor_id=actor_id)
|
||||
if child.ansible_action != parent.ansible_action:
|
||||
child.ansible_action = parent.ansible_action
|
||||
return added, removed
|
||||
|
||||
|
||||
async def reconcile_covering_rules(app) -> None:
|
||||
"""Every covering rule's children brought in line with the hosts there are
|
||||
now. Run each minute by the core scheduler (alert_reconcile), so a host
|
||||
that joins a group, or starts reporting, is covered within a minute."""
|
||||
async with app.db_sessionmaker() as db:
|
||||
async with db.begin():
|
||||
parents = (await db.execute(
|
||||
select(AlertRule).where(AlertRule.covers != "resource"))).scalars().all()
|
||||
for parent in parents:
|
||||
added, removed = await reconcile_children(db, parent)
|
||||
if added or removed:
|
||||
logger.info("Covering rule %r: added %s, removed %s", parent.name,
|
||||
", ".join(added) or "none", ", ".join(removed) or "none")
|
||||
|
||||
|
||||
def part_of(rule: AlertRule, rules) -> AlertRule | None:
|
||||
"""The covering rule a child belongs to, from a list of rules; None for a
|
||||
rule that stands alone."""
|
||||
return next((r for r in rules if r.id == rule.parent_id), None) if rule.parent_id else None
|
||||
|
||||
|
||||
class PartOfCoveringRule(ValueError):
|
||||
"""A child of a covering rule is changed through its parent, never alone."""
|
||||
|
||||
def __init__(self, child: AlertRule, parent_name: str):
|
||||
super().__init__(f"{child.name} is part of the rule {parent_name!r}, which keeps one "
|
||||
f"rule per host; change or disable it there.")
|
||||
|
||||
|
||||
async def acknowledge(db, rule_id: str, user_id: str | None) -> AlertState:
|
||||
"""Acknowledge a firing alert, in the caller's transaction, and record it
|
||||
in the alert's history as a firing → acknowledged event by `user_id`, so
|
||||
|
||||
+4
-2
@@ -408,8 +408,9 @@ def _register_core_tasks(app: Quart) -> None:
|
||||
await _cleanup(app)
|
||||
|
||||
async def run_alert_reconcile():
|
||||
from .alerts.service import reconcile_covering_rules
|
||||
from .core.alerts import check_silence, notify_after_maintenance
|
||||
for step in (check_silence, notify_after_maintenance):
|
||||
for step in (reconcile_covering_rules, check_silence, notify_after_maintenance):
|
||||
try:
|
||||
await step(app)
|
||||
except Exception:
|
||||
@@ -439,7 +440,8 @@ def _register_core_tasks(app: Quart) -> None:
|
||||
interval_seconds=60,
|
||||
run_on_startup=True,
|
||||
),
|
||||
# Rules whose series has gone quiet get a visible no_data mark; alerts
|
||||
# Covering rules gain a child for each newly covered host (#5464);
|
||||
# rules whose series has gone quiet get a visible no_data mark; alerts
|
||||
# a maintenance window muted notify once it ends.
|
||||
ScheduledTask(
|
||||
name="alert_reconcile",
|
||||
|
||||
+69
-12
@@ -134,6 +134,11 @@ async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict
|
||||
if it has no state row. `no_data_since` is set while its metric is silent,
|
||||
whatever the state. `ansible_action` is the
|
||||
playbook the rule runs when it fires, if any (`check: true` = dry run).
|
||||
|
||||
`covers` is "resource" for a rule on one resource, or "all_hosts" /
|
||||
"group" (with `group`) for a rule covering hosts: it has no state of its
|
||||
own, `hosts` names the hosts it covers, and each host's rule is listed
|
||||
with `part_of` naming it.
|
||||
"""
|
||||
limit = clamp_limit(limit, 200, 500)
|
||||
async with db_session() as db:
|
||||
@@ -141,13 +146,19 @@ async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict
|
||||
states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
|
||||
if enabled_only:
|
||||
rules = [r for r in rules if r.enabled]
|
||||
names = {r.id: r.name for r in rules}
|
||||
hosts: dict[str, list[str]] = {}
|
||||
for r in rules:
|
||||
if r.parent_id:
|
||||
hosts.setdefault(r.parent_id, []).append(r.resource_name)
|
||||
rows = [
|
||||
{
|
||||
"rule_id": r.id,
|
||||
"rule_name": r.name,
|
||||
"enabled": r.enabled,
|
||||
"source_module": r.source_module,
|
||||
"resource": r.resource_name,
|
||||
"resource": r.resource_name or None,
|
||||
**_covers(r, hosts.get(r.id, []), names.get(r.parent_id)),
|
||||
"metric": r.metric_name,
|
||||
"condition": _condition(r),
|
||||
"state": alert_svc.alert_label(states[r.id]) if r.id in states else None,
|
||||
@@ -219,21 +230,51 @@ async def _rule(db, rule: str) -> AlertRule:
|
||||
return pick(rule, await alert_svc.list_rules(db), kind="alert rule", list_tool="list_alert_rules")
|
||||
|
||||
|
||||
def _rule_row(r: AlertRule) -> dict:
|
||||
def _covers(r: AlertRule, hosts: list[str], parent_name: str | None) -> dict:
|
||||
"""What a rule covers (#5464): the hosts under a covering rule, or the
|
||||
covering rule a host's rule is part of."""
|
||||
out: dict = {"covers": r.covers}
|
||||
if r.covers == "group":
|
||||
out["group"] = r.cover_group
|
||||
if r.covers != "resource":
|
||||
out["hosts"] = sorted(hosts, key=str.lower)
|
||||
if r.parent_id:
|
||||
out["part_of"] = parent_name
|
||||
return out
|
||||
|
||||
|
||||
async def _rule_row(db, r: AlertRule) -> dict:
|
||||
hosts = [c.resource_name for c in (await db.execute(
|
||||
select(AlertRule).where(AlertRule.parent_id == r.id))).scalars()] if r.covers != "resource" else []
|
||||
parent = await db.get(AlertRule, r.parent_id) if r.parent_id else None
|
||||
return {"rule_id": r.id, "rule_name": r.name, "enabled": r.enabled,
|
||||
"source_module": r.source_module, "resource": r.resource_name,
|
||||
"source_module": r.source_module, "resource": r.resource_name or None,
|
||||
**_covers(r, hosts, parent.name if parent else None),
|
||||
"metric": r.metric_name, "condition": _condition(r),
|
||||
"notify_no_data": r.notify_no_data, "ansible_action": _action(r)}
|
||||
|
||||
|
||||
async def _not_a_child(db, r: AlertRule) -> None:
|
||||
"""A host's rule under a covering rule is changed through its parent."""
|
||||
if r.parent_id:
|
||||
parent = await db.get(AlertRule, r.parent_id)
|
||||
raise alert_svc.PartOfCoveringRule(r, parent.name if parent else "?")
|
||||
|
||||
|
||||
async def create_alert_rule(name: str, source_module: str, resource: str, metric: str,
|
||||
operator: str, threshold: float, consecutive_checks: int = 1,
|
||||
clear_after_checks: int = 1, notify_no_data: bool = True,
|
||||
enabled: bool = True) -> dict:
|
||||
enabled: bool = True, covers: str = "resource",
|
||||
group: str = "") -> dict:
|
||||
"""Add an alert rule: notify when `metric` of `resource` crosses `threshold`.
|
||||
|
||||
source_module, resource, metric: get_alert_rule_options lists the valid
|
||||
choices; a resource must have reported (or, for a monitor, exist).
|
||||
covers: "resource" (default) watches `resource`. "all_hosts" covers every
|
||||
host, "group" every host in the inventory group `group`; pass
|
||||
`resource` as "". Steward then keeps one rule per host, named
|
||||
"<name>: <host>", and adds one within a minute for a host that starts
|
||||
reporting or joins the group. Only for host_agent.
|
||||
operator: ">", "<", ">=", "<=", "==" or "!=".
|
||||
consecutive_checks: how many checks in a row must breach before it fires
|
||||
(default 1); higher values ride out a single blip.
|
||||
@@ -254,12 +295,15 @@ async def create_alert_rule(name: str, source_module: str, resource: str, metric
|
||||
db, name=name, source_module=source_module, resource_name=resource,
|
||||
metric_name=metric, operator=operator, threshold=threshold,
|
||||
consecutive_failures_required=consecutive_checks,
|
||||
consecutive_ok_required=clear_after_checks, notify_no_data=notify_no_data)
|
||||
consecutive_ok_required=clear_after_checks, notify_no_data=notify_no_data,
|
||||
covers=covers, cover_group=group)
|
||||
r = await alert_svc.add_rule(db, fields, created_by=caller.user_id, enabled=enabled)
|
||||
row = await _rule_row(db, r)
|
||||
await audit_mcp("alert_rule.created", "alert_rule", r.name,
|
||||
{"rule_id": r.id, "source": r.source_module, "resource": r.resource_name,
|
||||
{"rule_id": r.id, "source": r.source_module,
|
||||
"resource": r.resource_name or alert_svc.covers_text(r),
|
||||
"metric": r.metric_name})
|
||||
return _rule_row(r)
|
||||
return row
|
||||
|
||||
|
||||
async def update_alert_rule(rule: str, name: str | None = None, source_module: str | None = None,
|
||||
@@ -268,7 +312,7 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s
|
||||
consecutive_checks: int | None = None,
|
||||
clear_after_checks: int | None = None,
|
||||
notify_no_data: bool | None = None,
|
||||
enabled: bool | None = None) -> dict:
|
||||
enabled: bool | None = None, group: str | None = None) -> dict:
|
||||
"""Change an alert rule. Only the arguments you pass change; the rest stay.
|
||||
|
||||
rule: its name or id (list_alert_rules). Pass enabled=false to silence it
|
||||
@@ -277,9 +321,15 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s
|
||||
operator, threshold), or disabling or re-enabling it, puts its alert back
|
||||
to inactive; a firing alert records a resolved event saying why. Recorded
|
||||
in the audit log.
|
||||
|
||||
A rule covering hosts passes every change on to its per-host rules;
|
||||
`group` moves a group-covering rule to another inventory group. What a
|
||||
rule covers is fixed once it exists. A host's rule under a covering rule
|
||||
is changed through that rule, not alone.
|
||||
"""
|
||||
async with db_session() as db, db.begin():
|
||||
r = await _rule(db, rule)
|
||||
await _not_a_child(db, r)
|
||||
fields = await alert_svc.rule_fields(
|
||||
db,
|
||||
name=r.name if name is None else name,
|
||||
@@ -292,24 +342,31 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s
|
||||
if consecutive_checks is None else consecutive_checks),
|
||||
consecutive_ok_required=(r.consecutive_ok_required
|
||||
if clear_after_checks is None else clear_after_checks),
|
||||
notify_no_data=r.notify_no_data if notify_no_data is None else notify_no_data)
|
||||
notify_no_data=r.notify_no_data if notify_no_data is None else notify_no_data,
|
||||
covers=r.covers, cover_group=(r.cover_group or "") if group is None else group)
|
||||
await alert_svc.apply_rule_change(db, r, fields, enabled=enabled,
|
||||
actor_id=current_caller().user_id)
|
||||
if r.covers != "resource":
|
||||
await alert_svc.reconcile_children(db, r, actor_id=current_caller().user_id)
|
||||
row = await _rule_row(db, r)
|
||||
changed = sorted(k for k, v in {"name": name, "source_module": source_module,
|
||||
"resource": resource, "metric": metric, "operator": operator,
|
||||
"threshold": threshold, "consecutive_checks": consecutive_checks,
|
||||
"clear_after_checks": clear_after_checks,
|
||||
"notify_no_data": notify_no_data,
|
||||
"enabled": enabled}.items() if v is not None)
|
||||
"enabled": enabled, "group": group}.items() if v is not None)
|
||||
await audit_mcp("alert_rule.updated", "alert_rule", r.name, {"rule_id": r.id, "changed": changed})
|
||||
return _rule_row(r)
|
||||
return row
|
||||
|
||||
|
||||
async def delete_alert_rule(rule: str) -> dict:
|
||||
"""Delete an alert rule and its history. To silence it but keep it,
|
||||
update_alert_rule(enabled=false). Admin only; recorded in the audit log."""
|
||||
update_alert_rule(enabled=false). Deleting a rule that covers hosts
|
||||
deletes its per-host rules and their history too. Admin only; recorded
|
||||
in the audit log."""
|
||||
async with db_session() as db, db.begin():
|
||||
r = await _rule(db, rule)
|
||||
await _not_a_child(db, r)
|
||||
await db.delete(r)
|
||||
await audit_mcp("alert_rule.deleted", "alert_rule", r.name, {"rule_id": r.id})
|
||||
return {"deleted": r.name, "rule_id": r.id}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
"""A rule can cover every host, or every host in an inventory group (#5464):
|
||||
the covering rule keeps one child rule per host under it.
|
||||
|
||||
Revision ID: 0041_alert_rule_covers
|
||||
Revises: 0040_alert_event_actor
|
||||
Create Date: 2026-10-09
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0041_alert_rule_covers"
|
||||
down_revision: Union[str, None] = "0040_alert_event_actor"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("alert_rules", sa.Column("covers", sa.String(16), nullable=False,
|
||||
server_default="resource"))
|
||||
op.add_column("alert_rules", sa.Column("cover_group", sa.String(128), nullable=True))
|
||||
op.add_column("alert_rules", sa.Column(
|
||||
"parent_id", sa.String(36), sa.ForeignKey("alert_rules.id", ondelete="CASCADE"),
|
||||
nullable=True))
|
||||
op.create_index("ix_alert_rules_parent_id", "alert_rules", ["parent_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_alert_rules_parent_id", table_name="alert_rules")
|
||||
op.drop_column("alert_rules", "parent_id")
|
||||
op.drop_column("alert_rules", "cover_group")
|
||||
op.drop_column("alert_rules", "covers")
|
||||
@@ -55,6 +55,18 @@ class AlertRule(Base):
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
# What the rule covers (#5464): "resource" watches resource_name itself;
|
||||
# "all_hosts" and "group" make it a covering rule over hosts (every host,
|
||||
# or every host in the inventory group cover_group). A covering rule has
|
||||
# no state and is never evaluated: Steward keeps one ordinary child rule
|
||||
# per covered host (parent_id), so each host's alert keeps its own state,
|
||||
# history and acknowledgement. Fixed when the rule is created.
|
||||
covers: Mapped[str] = mapped_column(String(16), nullable=False, default="resource",
|
||||
server_default="resource")
|
||||
cover_group: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
parent_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("alert_rules.id", ondelete="CASCADE"), nullable=True, index=True
|
||||
)
|
||||
|
||||
|
||||
class AlertState(Base):
|
||||
|
||||
@@ -216,6 +216,9 @@ strong, b, th { font-weight: 500; }
|
||||
/* The Docker page's one-line answer, above its tiles (#5298). */
|
||||
.page-verdict { margin: 0 0 0.75rem; font-size: 0.95rem; }
|
||||
.table tr.row-off > td { opacity: 0.55; }
|
||||
/* A covering alert rule's per-host rules (#5464), indented under it. */
|
||||
.table tr.row-child > td { border-top-style: dashed; }
|
||||
.table tr.row-child .cell-name { padding-left: 2rem; }
|
||||
/* A row that opens as a whole: its one .row-target link stretches over the
|
||||
row, and the actions cell sits above it so its buttons keep their own
|
||||
clicks. One real link per row, so keyboard and screen readers meet it once. */
|
||||
|
||||
@@ -1,16 +1,49 @@
|
||||
{# alerts/_rule_fields.html — the resource and metric pickers for one source;
|
||||
the rule form includes it and swaps it in again when the source changes. #}
|
||||
the rule form includes it and swaps it in again when the source changes.
|
||||
For a source whose resources are hosts, the rule can instead cover every
|
||||
host, or every host in an inventory group (#5464): Steward then keeps one
|
||||
rule per host under it. What a rule covers is fixed once it exists. #}
|
||||
{% set covers = current_covers|default("resource") %}
|
||||
{% set one = covers == "resource" %}
|
||||
{% if host_source|default(false) %}
|
||||
<div class="form-group">
|
||||
<label for="rule-covers">Covers</label>
|
||||
{% if covers_locked|default(false) %}
|
||||
<input type="hidden" name="covers" value="{{ covers }}">
|
||||
<p style="margin:0;">{% if one %}One host, or one part of a host{% elif covers == "all_hosts" %}Every host{% else %}Every host in a group{% endif %}
|
||||
<span class="tone-dim">· fixed once the rule exists</span></p>
|
||||
{% else %}
|
||||
<select id="rule-covers" name="covers" data-covers>
|
||||
<option value="resource" {% if one %}selected{% endif %}>One host, or one part of a host</option>
|
||||
<option value="all_hosts" {% if covers == "all_hosts" %}selected{% endif %}>Every host</option>
|
||||
<option value="group" {% if covers == "group" %}selected{% endif %} {% if not groups %}disabled{% endif %}>Every host in a group{% if not groups %} (no inventory groups yet){% endif %}</option>
|
||||
</select>
|
||||
{% endif %}
|
||||
<p class="field-hint">Covering hosts keeps one alert per host under this rule, each named
|
||||
"rule: host". A host that starts reporting, or joins the group, is covered within a minute;
|
||||
one that leaves loses its alert and that alert's history.</p>
|
||||
</div>
|
||||
<div class="form-group" data-cover-group {% if covers != "group" %}hidden{% endif %}>
|
||||
<label for="rule-group">Group</label>
|
||||
<select id="rule-group" name="cover_group" {% if covers != "group" %}disabled{% endif %}>
|
||||
{% for g in groups %}
|
||||
<option value="{{ g }}" {% if g == current_group|default("") %}selected{% endif %}>{{ g }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<p class="field-hint">A host is in a group through its linked Ansible target (Ansible → Inventory).</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
<div class="form-group" data-cover-resource {% if not one %}hidden{% endif %}>
|
||||
<label>Resource</label>
|
||||
{% if resources %}
|
||||
<select name="resource_name" required>
|
||||
<select name="resource_name" required {% if not one %}disabled{% endif %}>
|
||||
{% if not current_resource %}<option value="" disabled selected>— pick a resource —</option>{% endif %}
|
||||
{% for r in resources %}
|
||||
<option value="{{ r }}" {% if r == current_resource %}selected{% endif %}>{{ r }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
{% else %}
|
||||
<select name="resource_name" required disabled>
|
||||
<select name="resource_name" required disabled data-none="1">
|
||||
<option value="">Nothing has reported from this source yet</option>
|
||||
</select>
|
||||
<p class="field-hint">
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
(alerts/_active.html, polled), then every rule: what it is doing now and
|
||||
when it last fired, what it fires on, and its History and Disable. The
|
||||
whole row opens the rule (.row-link, its name is the link); deleting one is
|
||||
on its own page, behind the confirm dialog. #}
|
||||
on its own page, behind the confirm dialog. A rule covering hosts (#5464)
|
||||
lists its per-host rules under it; each opens its history. #}
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Alerts — Steward{% endblock %}
|
||||
{% from "_macros.html" import page_header, status_badge, empty_state %}
|
||||
@@ -36,11 +37,13 @@
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for rule in rules %}
|
||||
{% set kids = (children or {}).get(rule.id, []) %}
|
||||
{% set fired = (last_fired or {}).get(rule.id) %}
|
||||
<tr class="row-link{% if not rule.enabled %} row-off{% endif %}">
|
||||
<td class="cell-name">
|
||||
<a class="row-target" href="{{ ('/alerts/rules/' ~ rule.id ~ '/edit') if can_edit else ('/alerts/history?rule=' ~ rule.id) }}" title="{{ rule.name }}">{{ rule.name }}</a>
|
||||
<div class="cell-sub">{{ source_label(rule.source_module) }}</div>
|
||||
<div class="cell-sub">{{ source_label(rule.source_module) }}
|
||||
{%- if rule.covers != "resource" %} · {{ kids|length }} host{{ "s" if kids|length != 1 }}{% endif %}</div>
|
||||
</td>
|
||||
<td>
|
||||
{%- set now_label = (rule_state or {}).get(rule.id) %}
|
||||
@@ -50,7 +53,7 @@
|
||||
<div class="cell-sub">{% if fired %}last fired {{ fired|ago }}{% else %}never fired{% endif %}</div>
|
||||
</td>
|
||||
<td>
|
||||
{{ rule.resource_name }} · {{ rule_condition(rule) }}
|
||||
{% if rule.covers == "group" %}every host in {{ rule.cover_group }}{% elif rule.covers == "all_hosts" %}every host{% else %}{{ rule.resource_name }}{% endif %} · {{ rule_condition(rule) }}
|
||||
<div class="cell-sub">
|
||||
{%- set n_fire, n_ok = rule.consecutive_failures_required, rule.consecutive_ok_required %}
|
||||
{%- if n_fire > 1 %}after {{ n_fire }} checks in a row{% else %}on the first check{% endif %}
|
||||
@@ -58,7 +61,7 @@
|
||||
</div>
|
||||
</td>
|
||||
<td class="td-actions">
|
||||
<a href="/alerts/history?rule={{ rule.id }}" class="btn btn-sm btn-ghost">History</a>
|
||||
{% if rule.covers == "resource" %}<a href="/alerts/history?rule={{ rule.id }}" class="btn btn-sm btn-ghost">History</a>{% endif %}
|
||||
{% if can_edit %}
|
||||
<form method="post" action="/alerts/rules/{{ rule.id }}/toggle" style="display:inline;">{{ csrf_field() }}
|
||||
<button type="submit" class="btn btn-sm btn-ghost">
|
||||
@@ -68,6 +71,23 @@
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% for kid in kids %}
|
||||
{% set kid_label = (rule_state or {}).get(kid.id) %}
|
||||
{% set kid_fired = (last_fired or {}).get(kid.id) %}
|
||||
<tr class="row-link row-child{% if not kid.enabled %} row-off{% endif %}">
|
||||
<td class="cell-name">
|
||||
<a class="row-target" href="/alerts/history?rule={{ kid.id }}" title="{{ kid.name }}">{{ kid.resource_name }}</a>
|
||||
</td>
|
||||
<td>
|
||||
{%- if not kid.enabled %}<span class="tone-dim">off</span>
|
||||
{%- elif kid_label and kid_label not in ('inactive', 'resolved') %}{{ status_badge(kid_label) }}
|
||||
{%- else %}<span class="tone-ok" title="Within its threshold">quiet</span>{% endif -%}
|
||||
<div class="cell-sub">{% if kid_fired %}last fired {{ kid_fired|ago }}{% else %}never fired{% endif %}</div>
|
||||
</td>
|
||||
<td class="tone-dim">as above</td>
|
||||
<td class="td-actions"><a href="/alerts/history?rule={{ kid.id }}" class="btn btn-sm btn-ghost">History</a></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
@@ -17,6 +17,13 @@
|
||||
{%- if fired %}Last fired {{ fired|ago }}{% else %}Never fired{% endif -%}
|
||||
{%- if history_count is defined and history_count is not none %} · {{ history_count }} change{{ "s" if history_count != 1 }} of state on record{% endif -%}
|
||||
</p>
|
||||
{% if rule.covers != "resource" %}
|
||||
<p class="field-hint" style="margin-top:-0.5rem;">
|
||||
Covers {% if rule.covers == "group" %}every host in <strong>{{ rule.cover_group }}</strong>{% else %}every host{% endif %}:
|
||||
{% if covered %}{{ covered|length }} now ({{ covered|join(", ") }}), each with its own alert and history.
|
||||
{%- else %}no host yet; one is covered within a minute of reporting.{% endif %}
|
||||
Changes here apply to all of them.</p>
|
||||
{% endif %}
|
||||
{% else %}
|
||||
{{ page_header("New alert rule") }}
|
||||
{% endif %}
|
||||
@@ -212,6 +219,16 @@
|
||||
document.addEventListener("input", function (e) {
|
||||
if (e.target.name === "threshold" && !e.target.form.querySelector("[data-threshold-echo]").hidden) showDuration(e.target.form);
|
||||
});
|
||||
// Covering hosts swaps the resource picker for the group picker (#5464).
|
||||
document.addEventListener("change", function (e) {
|
||||
if (!e.target.hasAttribute("data-covers")) return;
|
||||
var form = e.target.form, v = e.target.value;
|
||||
var res = form.querySelector("[data-cover-resource]"), grp = form.querySelector("[data-cover-group]");
|
||||
res.hidden = v !== "resource";
|
||||
res.querySelectorAll("select").forEach(function (s) { s.disabled = v !== "resource" || s.dataset.none === "1"; });
|
||||
grp.hidden = v !== "group";
|
||||
grp.querySelector("select").disabled = v !== "group";
|
||||
});
|
||||
// The playbook's fields show only while "run a playbook" is ticked.
|
||||
document.addEventListener("change", function (e) {
|
||||
if (!e.target.hasAttribute("data-ansible-toggle")) return;
|
||||
|
||||
@@ -17,7 +17,8 @@ def _rule(metric="cpu_pct", op=AlertOperator.gt, threshold=90.0, resource="web-1
|
||||
return SimpleNamespace(id=rid, name=f"{resource} {metric}", source_module="host_agent",
|
||||
resource_name=resource, metric_name=metric, operator=op,
|
||||
threshold=threshold, enabled=True,
|
||||
consecutive_failures_required=1, consecutive_ok_required=1)
|
||||
consecutive_failures_required=1, consecutive_ok_required=1,
|
||||
covers="resource", cover_group=None, parent_id=None)
|
||||
|
||||
|
||||
def _said(says):
|
||||
@@ -272,3 +273,35 @@ async def test_the_rule_form_speaks_in_names_and_units(app):
|
||||
assert '<option value="down" selected>' in page # the binary choice, shown
|
||||
assert 'name="threshold" step="any" required disabled' in page # the raw threshold, off
|
||||
assert ">is_up<" not in page and "SNMP" not in page # no raw key; empty source skipped
|
||||
|
||||
|
||||
async def test_a_rule_covering_hosts_lists_its_hosts_under_it(app):
|
||||
"""#5464: a covering rule reads as what it covers, with each host's rule
|
||||
nested under it opening that host's history; its page names the hosts."""
|
||||
parent = _rule("seconds_since_seen", AlertOperator.gt, 180.0, "", rid="p1")
|
||||
parent.name, parent.covers, parent.cover_group = "Agent offline", "group", "swarm"
|
||||
parent.ansible_action, parent.notify_no_data = None, True
|
||||
kids = []
|
||||
for i, host in enumerate(("node-a", "node-b")):
|
||||
k = _rule("seconds_since_seen", AlertOperator.gt, 180.0, host, rid=f"k{i}")
|
||||
k.name, k.parent_id = f"Agent offline: {host}", "p1"
|
||||
kids.append(k)
|
||||
async with app.test_request_context("/alerts/"):
|
||||
session["user_role"] = "operator"
|
||||
listing = await render_template(
|
||||
"alerts/list.html", active=[], rules=[parent], children={"p1": kids},
|
||||
rule_state={"p1": "firing", "k0": "firing", "k1": "inactive"}, operators=[])
|
||||
page = await render_template(
|
||||
"alerts/rules_form.html", rule=parent, operators=[], source_modules=["host_agent"],
|
||||
initial_source="host_agent", resources=[], metrics=["seconds_since_seen"],
|
||||
ansible_sources=[], catalog={}, history_count=0, fired=None, notify=[],
|
||||
covered=["node-a", "node-b"], host_source=True, groups=["swarm"],
|
||||
current_covers="group", current_group="swarm", covers_locked=True)
|
||||
listing, page = " ".join(listing.split()), " ".join(page.split())
|
||||
assert "every host in swarm" in listing and "2 hosts" in listing
|
||||
assert listing.index("Agent offline</a>") < listing.index('href="/alerts/history?rule=k0"') \
|
||||
< listing.index('href="/alerts/history?rule=k1"')
|
||||
assert 'class="row-link row-child"' in listing
|
||||
assert "Covers every host in <strong>swarm</strong>: 2 now (node-a, node-b)" in page
|
||||
assert 'name="covers" value="group"' in page and "fixed once the rule exists" in page
|
||||
assert 'data-cover-resource hidden' in page
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Integration: a rule covering hosts (#5464) keeps one ordinary rule per
|
||||
covered host, adds one for a host that joins its group, removes one for a host
|
||||
that leaves, passes edits and on/off down to them, and refuses a change made
|
||||
to a host's rule alone. Each host's rule fires on its own readings."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
if not os.environ.get("STEWARD_DATABASE_URL"):
|
||||
pytest.skip("needs Postgres")
|
||||
from steward.app import create_app
|
||||
return create_app(testing=False)
|
||||
|
||||
|
||||
def test_a_group_rule_keeps_one_rule_per_host_in_the_group(app):
|
||||
from sqlalchemy import select
|
||||
from steward.alerts import service as alert_svc
|
||||
from steward.core.alerts import record_metric
|
||||
from steward.models.alerts import AlertRule, AlertState, AlertStateEnum
|
||||
from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.users import User, UserRole
|
||||
|
||||
async def _go():
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
names = [f"cv-{tag}-{i}" for i in range(3)]
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
user = User(username=f"cv-{tag}", email=f"cv-{tag}@test", password_hash="x",
|
||||
role=UserRole.admin)
|
||||
hosts = [Host(name=n, address=f"10.8.{i}.1") for i, n in enumerate(names)]
|
||||
s.add_all([user, *hosts])
|
||||
await s.flush()
|
||||
group = AnsibleGroup(name=f"web {tag}")
|
||||
targets = [AnsibleTarget(name=f"t-{n}", address=h.address, host_id=h.id)
|
||||
for n, h in zip(names, hosts)]
|
||||
for t in targets[:2]:
|
||||
t.groups = [group]
|
||||
s.add_all([group, *targets])
|
||||
for n in names:
|
||||
await record_metric(s, "host_agent", n, "cpu_pct", 10.0)
|
||||
fields = await alert_svc.rule_fields(
|
||||
s, name=f"Hot {tag}", source_module="host_agent", resource_name="",
|
||||
metric_name="cpu_pct", operator=">", threshold=90,
|
||||
covers="group", cover_group=group.name)
|
||||
parent = await alert_svc.add_rule(s, fields, created_by=user.id)
|
||||
ids = {"parent": parent.id, "user": user.id, "third": targets[2].id,
|
||||
"group": group.id}
|
||||
|
||||
async def kids():
|
||||
async with app.db_sessionmaker() as s:
|
||||
rows = (await s.execute(select(AlertRule).where(
|
||||
AlertRule.parent_id == ids["parent"]))).scalars().all()
|
||||
return {r.resource_name: r for r in rows}
|
||||
|
||||
async def reconcile():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
return await alert_svc.reconcile_children(
|
||||
s, await s.get(AlertRule, ids["parent"]))
|
||||
|
||||
# The two hosts in the group, each its own rule; the parent has no state.
|
||||
got = await kids()
|
||||
assert sorted(got) == names[:2]
|
||||
assert got[names[0]].name == f"Hot {tag}: {names[0]}"
|
||||
assert got[names[0]].threshold == 90 and got[names[0]].covers == "resource"
|
||||
async with app.db_sessionmaker() as s:
|
||||
assert (await s.execute(select(AlertState).where(
|
||||
AlertState.rule_id == ids["parent"]))).scalar_one_or_none() is None
|
||||
|
||||
# A host's own readings fire its own rule only.
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await record_metric(s, "host_agent", names[0], "cpu_pct", 97.0)
|
||||
async with app.db_sessionmaker() as s:
|
||||
states = {st.rule_id: st.state for st in (await s.execute(select(AlertState).where(
|
||||
AlertState.rule_id.in_([r.id for r in got.values()])))).scalars()}
|
||||
assert states[got[names[0]].id] == AlertStateEnum.firing
|
||||
assert states[got[names[1]].id] == AlertStateEnum.inactive
|
||||
|
||||
# The third host joins the group: covered on the next reconcile.
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
from sqlalchemy.orm import selectinload
|
||||
t = (await s.execute(select(AnsibleTarget).where(AnsibleTarget.id == ids["third"])
|
||||
.options(selectinload(AnsibleTarget.groups)))).scalar_one()
|
||||
t.groups = [await s.get(AnsibleGroup, ids["group"])]
|
||||
assert await reconcile() == ([names[2]], [])
|
||||
assert sorted(await kids()) == names
|
||||
|
||||
# Raising the threshold passes down; the firing host's alert resets.
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
p = await s.get(AlertRule, ids["parent"])
|
||||
fields = await alert_svc.rule_fields(
|
||||
s, name=p.name, source_module="host_agent", resource_name="",
|
||||
metric_name="cpu_pct", operator=">", threshold=99,
|
||||
covers="group", cover_group=p.cover_group)
|
||||
await alert_svc.apply_rule_change(s, p, fields, actor_id=ids["user"])
|
||||
await alert_svc.reconcile_children(s, p, actor_id=ids["user"])
|
||||
got = await kids()
|
||||
assert {r.threshold for r in got.values()} == {99}
|
||||
async with app.db_sessionmaker() as s:
|
||||
st = (await s.execute(select(AlertState).where(
|
||||
AlertState.rule_id == got[names[0]].id))).scalar_one()
|
||||
assert st.state == AlertStateEnum.inactive
|
||||
|
||||
# Disabling the parent disables every host's rule.
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
p = await s.get(AlertRule, ids["parent"])
|
||||
await alert_svc.apply_rule_change(s, p, {}, enabled=False)
|
||||
await alert_svc.reconcile_children(s, p)
|
||||
assert not any(r.enabled for r in (await kids()).values())
|
||||
|
||||
# A host leaving the group loses its rule.
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
t = (await s.execute(select(AnsibleTarget).where(AnsibleTarget.id == ids["third"])
|
||||
.options(selectinload(AnsibleTarget.groups)))).scalar_one()
|
||||
t.groups = []
|
||||
assert await reconcile() == ([], [names[2]])
|
||||
assert sorted(await kids()) == names[:2]
|
||||
|
||||
asyncio.run(_go())
|
||||
|
||||
|
||||
def test_a_host_rule_is_changed_through_its_covering_rule(app):
|
||||
"""Over MCP, a host's rule under a covering rule refuses an edit made to it
|
||||
alone, and the covering rule lists the hosts it covers."""
|
||||
from steward.alerts import service as alert_svc
|
||||
from steward.core.alerts import record_metric
|
||||
from steward.core.api_tokens import Caller
|
||||
from steward.mcp import _context
|
||||
from steward.mcp.tools import alerts as a
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.users import User, UserRole
|
||||
|
||||
async def _go():
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
name = f"cvm-{tag}"
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
user = User(username=f"cvm-{tag}", email=f"cvm-{tag}@test", password_hash="x",
|
||||
role=UserRole.admin)
|
||||
s.add_all([user, Host(name=name, address="10.8.9.1")])
|
||||
await s.flush()
|
||||
await record_metric(s, "host_agent", name, "seconds_since_seen", 5.0)
|
||||
uid = user.id
|
||||
_context._app_ctx.set(app)
|
||||
_context._caller_ctx.set(Caller(user_id=uid, username=f"cvm-{tag}",
|
||||
role=UserRole.admin, token_id=7))
|
||||
made = await a.create_alert_rule(f"Offline {tag}", "host_agent", "",
|
||||
"seconds_since_seen", ">", 180, covers="all_hosts")
|
||||
assert made["covers"] == "all_hosts" and name in made["hosts"]
|
||||
with pytest.raises(alert_svc.PartOfCoveringRule, match="change or disable it there"):
|
||||
await a.update_alert_rule(f"Offline {tag}: {name}", threshold=60)
|
||||
listed = {r["rule_name"]: r for r in (await a.list_alert_rules(limit=500))["rules"]}
|
||||
assert listed[f"Offline {tag}: {name}"]["part_of"] == f"Offline {tag}"
|
||||
await a.update_alert_rule(f"Offline {tag}", threshold=300)
|
||||
listed = {r["rule_name"]: r for r in (await a.list_alert_rules(limit=500))["rules"]}
|
||||
assert listed[f"Offline {tag}: {name}"]["condition"].startswith("seconds_since_seen > 300")
|
||||
|
||||
asyncio.run(_go())
|
||||
Reference in New Issue
Block a user