feat(alerts): one rule can cover every host, or every host in a group (#5464)
#5464 "Group rules: one rule covers every host, or every host in a group", step 2 of milestone 499 "Alerts you can trust". Each rule named one host, so the live instance carried six rules for three hosts, and a new host had no agent-offline or disk alert until someone copied them. - alert_rules.covers / cover_group / parent_id (migration 0041). A rule on host_agent may cover "all_hosts" or a "group" (an inventory group: the Hosts ruling, #5294). A covering rule has no state; Steward keeps one ordinary child rule per covered host, named "<rule>: <host>", so each host's alert keeps its own state, history and acknowledgement. - alerts.service.reconcile_children: a covered host that has reported in the last 7 days gets a child; a host no longer covered loses its child; children take the parent's condition, checks, notify setting, action and on/off. Run on save and each minute (alert_reconcile). - A child is changed through its parent: edit, toggle and delete of a child are refused (409 / PartOfCoveringRule); its edit page opens the parent. - Rule form: a Covers choice for host sources, with the group picker; fixed once the rule exists. Rules list: a covering rule reads "every host in swarm · 3 hosts" with its hosts' rules nested under it, each opening its history. The history picker and the rule count leave covering rules out. - MCP: create_alert_rule takes covers/group; update_alert_rule takes group and passes changes down; list_alert_rules shows covers, hosts and part_of. - docs/core/alerting.md: "Rules that cover hosts". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -75,6 +75,29 @@ Alert rules are created in the UI at `/alerts/`. Each rule requires:
|
|||||||
- **Threshold** — numeric value
|
- **Threshold** — numeric value
|
||||||
- **Consecutive failures required** — how many consecutive breaches before FIRING (default 1)
|
- **Consecutive failures required** — how many consecutive breaches before FIRING (default 1)
|
||||||
|
|
||||||
|
### Rules that cover hosts
|
||||||
|
|
||||||
|
A rule on `host_agent` can cover **every host**, or **every host in an
|
||||||
|
inventory group**, instead of naming one resource (#5464). Inventory groups
|
||||||
|
are Steward's grouping system: a host is in a group through its linked
|
||||||
|
Ansible target.
|
||||||
|
|
||||||
|
A covering rule has no state of its own. Steward keeps one ordinary rule per
|
||||||
|
covered host under it, named `<rule>: <host>`, so each host's alert keeps its
|
||||||
|
own state, history and acknowledgement. Each minute (the `alert_reconcile`
|
||||||
|
task), and whenever the rule is saved:
|
||||||
|
|
||||||
|
- a host that has reported in the last 7 days, and is covered, gets a rule;
|
||||||
|
- a host that is no longer covered loses its rule, and that rule's history;
|
||||||
|
- the hosts' rules take the covering rule's condition, checks, notification
|
||||||
|
setting, action and on/off.
|
||||||
|
|
||||||
|
A host's rule is changed through its covering rule; editing, disabling or
|
||||||
|
deleting it alone is refused. What a rule covers is fixed once it exists, but
|
||||||
|
a group rule can move to another group. Over MCP, `create_alert_rule` takes
|
||||||
|
`covers` ("all_hosts" or "group") and `group`, and `list_alert_rules` shows
|
||||||
|
each covering rule's `hosts` and each host's rule's `part_of`.
|
||||||
|
|
||||||
### Available Metrics by Module
|
### Available Metrics by Module
|
||||||
|
|
||||||
| `source_module` | `metric_name` | Description |
|
| `source_module` | `metric_name` | Description |
|
||||||
|
|||||||
@@ -36,6 +36,8 @@ def _rule_form(form) -> dict:
|
|||||||
"consecutive_failures_required": form.get("consecutive_failures_required"),
|
"consecutive_failures_required": form.get("consecutive_failures_required"),
|
||||||
"consecutive_ok_required": form.get("consecutive_ok_required"),
|
"consecutive_ok_required": form.get("consecutive_ok_required"),
|
||||||
"notify_no_data": "notify_no_data" in form,
|
"notify_no_data": "notify_no_data" in form,
|
||||||
|
"covers": form.get("covers") or "resource",
|
||||||
|
"cover_group": form.get("cover_group", ""),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -47,6 +49,15 @@ def _form_time(value) -> datetime:
|
|||||||
raise ValueError(f"{value!r} is not a date and time.") from None
|
raise ValueError(f"{value!r} is not a date and time.") from None
|
||||||
|
|
||||||
|
|
||||||
|
async def _child_conflict(db, rule: AlertRule):
|
||||||
|
"""A 409 for a host's rule under a covering rule, which is changed
|
||||||
|
through its parent (#5464); None for any other rule."""
|
||||||
|
if not rule.parent_id:
|
||||||
|
return None
|
||||||
|
parent = await db.get(AlertRule, rule.parent_id)
|
||||||
|
return str(alert_svc.PartOfCoveringRule(rule, parent.name if parent else "?")), 409
|
||||||
|
|
||||||
|
|
||||||
def _is_admin() -> bool:
|
def _is_admin() -> bool:
|
||||||
return session.get("user_role") == UserRole.admin.value
|
return session.get("user_role") == UserRole.admin.value
|
||||||
|
|
||||||
@@ -123,11 +134,27 @@ async def list_alerts():
|
|||||||
states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
|
states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
|
||||||
fired = await alert_svc.last_fired(db)
|
fired = await alert_svc.last_fired(db)
|
||||||
|
|
||||||
# Every rule with what it is doing now; disabled rules last.
|
# Every rule with what it is doing now; disabled rules last. A covering
|
||||||
rules.sort(key=lambda r: (not r.enabled, r.name.lower()))
|
# rule's per-host children sit under it (#5464), and it reads as the most
|
||||||
|
# pressing of their states.
|
||||||
rule_state = {r.id: alert_svc.alert_label(states[r.id]) for r in rules if r.id in states}
|
rule_state = {r.id: alert_svc.alert_label(states[r.id]) for r in rules if r.id in states}
|
||||||
|
children: dict[str, list] = {}
|
||||||
|
for r in rules:
|
||||||
|
if r.parent_id:
|
||||||
|
children.setdefault(r.parent_id, []).append(r)
|
||||||
|
for kids in children.values():
|
||||||
|
kids.sort(key=lambda r: r.resource_name.lower())
|
||||||
|
parent_id = kids[0].parent_id
|
||||||
|
labels = [rule_state.get(k.id) for k in kids]
|
||||||
|
rule_state[parent_id] = min(
|
||||||
|
(lb for lb in labels if lb and lb not in ("inactive", "resolved")),
|
||||||
|
key=lambda lb: alert_svc.ACTIVE_ORDER.get(lb, 9), default="inactive")
|
||||||
|
fired[parent_id] = max((fired[k.id] for k in kids if k.id in fired), default=None)
|
||||||
|
rules = sorted((r for r in rules if not r.parent_id),
|
||||||
|
key=lambda r: (not r.enabled, r.name.lower()))
|
||||||
return await render_template(
|
return await render_template(
|
||||||
"alerts/list.html", active=active, rules=rules, rule_state=rule_state, last_fired=fired,
|
"alerts/list.html", active=active, rules=rules, children=children,
|
||||||
|
rule_state=rule_state, last_fired={k: v for k, v in fired.items() if v},
|
||||||
notify=await _notify(), operators=_OPERATORS,
|
notify=await _notify(), operators=_OPERATORS,
|
||||||
poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60))
|
poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60))
|
||||||
|
|
||||||
@@ -138,7 +165,8 @@ async def active_fragment():
|
|||||||
"""The verdict and Active section alone, which the alerts page polls."""
|
"""The verdict and Active section alone, which the alerts page polls."""
|
||||||
async with current_app.db_sessionmaker() as db:
|
async with current_app.db_sessionmaker() as db:
|
||||||
active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
|
active = await alert_svc.active_rows(db, datetime.now(timezone.utc))
|
||||||
rules = await alert_svc.list_rules(db)
|
# Rules as people made them: a covering rule counts once, not per host.
|
||||||
|
rules = [r for r in await alert_svc.list_rules(db) if not r.parent_id]
|
||||||
return await render_template(
|
return await render_template(
|
||||||
"alerts/_active.html", active=active, notify=await _notify(), rules_total=len(rules),
|
"alerts/_active.html", active=active, notify=await _notify(), rules_total=len(rules),
|
||||||
rules_off=sum(1 for r in rules if not r.enabled))
|
rules_off=sum(1 for r in rules if not r.enabled))
|
||||||
@@ -195,7 +223,8 @@ async def alert_history():
|
|||||||
since, range_key = parse_range(request.args.get("range") or "7d")
|
since, range_key = parse_range(request.args.get("range") or "7d")
|
||||||
before = request.args.get("before")
|
before = request.args.get("before")
|
||||||
async with current_app.db_sessionmaker() as db:
|
async with current_app.db_sessionmaker() as db:
|
||||||
rules = await alert_svc.list_rules(db)
|
# A covering rule has no history of its own; its hosts' rules do.
|
||||||
|
rules = [r for r in await alert_svc.list_rules(db) if r.covers == "resource"]
|
||||||
chosen = next((r for r in rules if r.id == request.args.get("rule")), None)
|
chosen = next((r for r in rules if r.id == request.args.get("rule")), None)
|
||||||
rule_id = chosen.id if chosen else None
|
rule_id = chosen.id if chosen else None
|
||||||
rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before)
|
rows = await alert_svc.history(db, since, rule_id, PAGE_SIZE, before)
|
||||||
@@ -232,26 +261,40 @@ async def rule_fields_api():
|
|||||||
current_metric = request.args.get("current_metric", "")
|
current_metric = request.args.get("current_metric", "")
|
||||||
async with current_app.db_sessionmaker() as db:
|
async with current_app.db_sessionmaker() as db:
|
||||||
resources = await alert_svc.resources_for(db, source) if source else []
|
resources = await alert_svc.resources_for(db, source) if source else []
|
||||||
async with current_app.db_sessionmaker() as db:
|
|
||||||
metrics = await alert_svc.metrics_for(db, source)
|
metrics = await alert_svc.metrics_for(db, source)
|
||||||
|
cover = await _cover_choices(db, source)
|
||||||
return await render_template(
|
return await render_template(
|
||||||
"alerts/_rule_fields.html",
|
"alerts/_rule_fields.html",
|
||||||
resources=resources,
|
resources=resources,
|
||||||
metrics=metrics,
|
metrics=metrics,
|
||||||
current_resource=current_resource,
|
current_resource=current_resource,
|
||||||
current_metric=current_metric,
|
current_metric=current_metric,
|
||||||
|
**cover,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _cover_choices(db, source: str, rule: AlertRule | None = None) -> dict:
|
||||||
|
"""What the rule form offers for covering hosts (#5464): whether this
|
||||||
|
source's resources are hosts, the inventory groups, and the rule's own
|
||||||
|
choice, fixed once it exists."""
|
||||||
|
from steward.hosts.service import group_names
|
||||||
|
host_source = source in alert_svc.HOST_SOURCES
|
||||||
|
return {"host_source": host_source, "groups": await group_names(db) if host_source else [],
|
||||||
|
"current_covers": rule.covers if rule else "resource",
|
||||||
|
"current_group": (rule.cover_group or "") if rule else "",
|
||||||
|
"covers_locked": rule is not None}
|
||||||
|
|
||||||
|
|
||||||
@alerts_bp.get("/rules/new")
|
@alerts_bp.get("/rules/new")
|
||||||
@require_role(UserRole.operator)
|
@require_role(UserRole.operator)
|
||||||
async def new_rule():
|
async def new_rule():
|
||||||
first_source = alert_svc.source_modules()[0]
|
first_source = alert_svc.source_modules()[0]
|
||||||
async with current_app.db_sessionmaker() as db:
|
async with current_app.db_sessionmaker() as db:
|
||||||
resources = await alert_svc.resources_for(db, first_source)
|
resources = await alert_svc.resources_for(db, first_source)
|
||||||
|
cover = await _cover_choices(db, first_source)
|
||||||
return await render_template(
|
return await render_template(
|
||||||
"alerts/rules_form.html",
|
"alerts/rules_form.html",
|
||||||
rule=None,
|
rule=None, **cover,
|
||||||
operators=_OPERATORS,
|
operators=_OPERATORS,
|
||||||
source_modules=alert_svc.source_modules(),
|
source_modules=alert_svc.source_modules(),
|
||||||
initial_source=first_source,
|
initial_source=first_source,
|
||||||
@@ -271,13 +314,20 @@ async def edit_rule(rule_id: str):
|
|||||||
rule = result.scalar_one_or_none()
|
rule = result.scalar_one_or_none()
|
||||||
if rule is None:
|
if rule is None:
|
||||||
return "Not found", 404
|
return "Not found", 404
|
||||||
|
if rule.parent_id:
|
||||||
|
# A host's rule under a covering rule is changed through it.
|
||||||
|
return redirect(f"/alerts/rules/{rule.parent_id}/edit")
|
||||||
resources = await alert_svc.resources_for(db, rule.source_module)
|
resources = await alert_svc.resources_for(db, rule.source_module)
|
||||||
|
cover = await _cover_choices(db, rule.source_module, rule)
|
||||||
|
hosts = sorted((c.resource_name for c in (await db.execute(
|
||||||
|
select(AlertRule).where(AlertRule.parent_id == rule.id))).scalars()), key=str.lower)
|
||||||
history_count = (await db.execute(
|
history_count = (await db.execute(
|
||||||
select(func.count()).select_from(AlertEvent).where(AlertEvent.rule_id == rule.id))).scalar_one()
|
select(func.count()).select_from(AlertEvent).where(AlertEvent.rule_id == rule.id))).scalar_one()
|
||||||
fired = (await alert_svc.last_fired(db, rule.id)).get(rule.id)
|
fired = (await alert_svc.last_fired(db, rule.id)).get(rule.id)
|
||||||
return await render_template(
|
return await render_template(
|
||||||
"alerts/rules_form.html",
|
"alerts/rules_form.html",
|
||||||
rule=rule, history_count=history_count, fired=fired, notify=await _notify(),
|
rule=rule, history_count=history_count, fired=fired, notify=await _notify(),
|
||||||
|
covered=hosts, **cover,
|
||||||
operators=_OPERATORS,
|
operators=_OPERATORS,
|
||||||
source_modules=alert_svc.source_modules(),
|
source_modules=alert_svc.source_modules(),
|
||||||
initial_source=rule.source_module,
|
initial_source=rule.source_module,
|
||||||
@@ -331,13 +381,19 @@ async def update_rule(rule_id: str):
|
|||||||
rule = result.scalar_one_or_none()
|
rule = result.scalar_one_or_none()
|
||||||
if rule is None:
|
if rule is None:
|
||||||
return "Not found", 404
|
return "Not found", 404
|
||||||
|
if (conflict := await _child_conflict(db, rule)):
|
||||||
|
return conflict
|
||||||
try:
|
try:
|
||||||
fields = await alert_svc.rule_fields(db, **_rule_form(form))
|
# What a rule covers is fixed once it exists; its group may change.
|
||||||
|
fields = await alert_svc.rule_fields(db, **{**_rule_form(form),
|
||||||
|
"covers": rule.covers})
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
return str(exc), 400
|
return str(exc), 400
|
||||||
await alert_svc.apply_rule_change(db, rule, fields, actor_id=session.get("user_id"))
|
await alert_svc.apply_rule_change(db, rule, fields, actor_id=session.get("user_id"))
|
||||||
if apply_action:
|
if apply_action:
|
||||||
rule.ansible_action = ansible_action
|
rule.ansible_action = ansible_action
|
||||||
|
if rule.covers != "resource":
|
||||||
|
await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id"))
|
||||||
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
||||||
"alert_rule.updated", entity_type="alert_rule", entity_id=rule.name)
|
"alert_rule.updated", entity_type="alert_rule", entity_id=rule.name)
|
||||||
return redirect(url_for("alerts.list_alerts"))
|
return redirect(url_for("alerts.list_alerts"))
|
||||||
@@ -352,8 +408,12 @@ async def toggle_rule(rule_id: str):
|
|||||||
rule = result.scalar_one_or_none()
|
rule = result.scalar_one_or_none()
|
||||||
if rule is None:
|
if rule is None:
|
||||||
return "Not found", 404
|
return "Not found", 404
|
||||||
|
if (conflict := await _child_conflict(db, rule)):
|
||||||
|
return conflict
|
||||||
await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled,
|
await alert_svc.apply_rule_change(db, rule, {}, enabled=not rule.enabled,
|
||||||
actor_id=session.get("user_id"))
|
actor_id=session.get("user_id"))
|
||||||
|
if rule.covers != "resource":
|
||||||
|
await alert_svc.reconcile_children(db, rule, actor_id=session.get("user_id"))
|
||||||
action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
|
action = "alert_rule.enabled" if rule.enabled else "alert_rule.disabled"
|
||||||
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
await log_audit(current_app, session.get("user_id"), session.get("username", ""),
|
||||||
action, entity_type="alert_rule", entity_id=rule.name)
|
action, entity_type="alert_rule", entity_id=rule.name)
|
||||||
@@ -369,6 +429,8 @@ async def delete_rule(rule_id: str):
|
|||||||
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
||||||
rule = result.scalar_one_or_none()
|
rule = result.scalar_one_or_none()
|
||||||
if rule:
|
if rule:
|
||||||
|
if (conflict := await _child_conflict(db, rule)):
|
||||||
|
return conflict
|
||||||
rule_name = rule.name
|
rule_name = rule.name
|
||||||
await db.delete(rule)
|
await db.delete(rule)
|
||||||
if rule_name:
|
if rule_name:
|
||||||
|
|||||||
+150
-9
@@ -378,12 +378,16 @@ def _choices(values: list[str], limit: int = 20) -> str:
|
|||||||
|
|
||||||
async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
|
async def rule_fields(db, *, name: str, source_module: str, resource_name: str, metric_name: str,
|
||||||
operator: str, threshold, consecutive_failures_required=1,
|
operator: str, threshold, consecutive_failures_required=1,
|
||||||
consecutive_ok_required=1, notify_no_data=True) -> dict:
|
consecutive_ok_required=1, notify_no_data=True,
|
||||||
|
covers: str = "resource", cover_group: str = "") -> dict:
|
||||||
"""Validated column values for a new or edited alert rule.
|
"""Validated column values for a new or edited alert rule.
|
||||||
|
|
||||||
The same choices the rule form offers: a known source, a resource that
|
The same choices the rule form offers: a known source, a resource that
|
||||||
source has reported (or, for monitors, a monitor of that type), and a
|
source has reported (or, for monitors, a monitor of that type), and a
|
||||||
metric it emits. ValueError says what was wrong and what is valid.
|
metric it emits. A covering rule (`covers` "all_hosts", or "group" with
|
||||||
|
an inventory group's name in `cover_group`, #5464) names no resource: it
|
||||||
|
is for a source whose resources are hosts. ValueError says what was wrong
|
||||||
|
and what is valid.
|
||||||
"""
|
"""
|
||||||
name = (name or "").strip()
|
name = (name or "").strip()
|
||||||
if not name:
|
if not name:
|
||||||
@@ -392,11 +396,29 @@ async def rule_fields(db, *, name: str, source_module: str, resource_name: str,
|
|||||||
catalog = metric_catalog()
|
catalog = metric_catalog()
|
||||||
if source_module not in catalog:
|
if source_module not in catalog:
|
||||||
raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.")
|
raise ValueError(f"source_module {source_module!r} is not one of {', '.join(catalog)}.")
|
||||||
resource_name = (resource_name or "").strip()
|
covers = (covers or "resource").strip()
|
||||||
resources = await resources_for(db, source_module)
|
if covers not in COVERS:
|
||||||
if resource_name not in resources:
|
raise ValueError(f"covers {covers!r} is not one of {', '.join(COVERS)}.")
|
||||||
raise ValueError(f"{source_module} has no resource {resource_name!r}. "
|
cover_group = (cover_group or "").strip()
|
||||||
f"Known: {_choices(resources)}.")
|
if covers == "resource":
|
||||||
|
resource_name, cover_group = (resource_name or "").strip(), None
|
||||||
|
resources = await resources_for(db, source_module)
|
||||||
|
if resource_name not in resources:
|
||||||
|
raise ValueError(f"{source_module} has no resource {resource_name!r}. "
|
||||||
|
f"Known: {_choices(resources)}.")
|
||||||
|
else:
|
||||||
|
if source_module not in HOST_SOURCES:
|
||||||
|
raise ValueError(f"only a rule on {', '.join(HOST_SOURCES)} can cover hosts; "
|
||||||
|
f"{source_module}'s resources are not hosts.")
|
||||||
|
resource_name = ""
|
||||||
|
if covers == "group":
|
||||||
|
from steward.hosts.service import group_names
|
||||||
|
groups = await group_names(db)
|
||||||
|
if cover_group not in groups:
|
||||||
|
raise ValueError(f"there is no inventory group {cover_group!r}. "
|
||||||
|
f"Groups: {_choices(groups)}.")
|
||||||
|
else:
|
||||||
|
cover_group = None
|
||||||
metric_name = (metric_name or "").strip()
|
metric_name = (metric_name or "").strip()
|
||||||
metrics = await metrics_for(db, source_module)
|
metrics = await metrics_for(db, source_module)
|
||||||
if metric_name not in metrics:
|
if metric_name not in metrics:
|
||||||
@@ -428,7 +450,8 @@ async def rule_fields(db, *, name: str, source_module: str, resource_name: str,
|
|||||||
return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
|
return {"name": name[:128], "source_module": source_module, "resource_name": resource_name,
|
||||||
"metric_name": metric_name, "operator": op, "threshold": threshold,
|
"metric_name": metric_name, "operator": op, "threshold": threshold,
|
||||||
"consecutive_failures_required": consecutive,
|
"consecutive_failures_required": consecutive,
|
||||||
"consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data)}
|
"consecutive_ok_required": ok_checks, "notify_no_data": bool(notify_no_data),
|
||||||
|
"covers": covers, "cover_group": cover_group}
|
||||||
|
|
||||||
|
|
||||||
# Changing any of these changes what the rule is watching, so whatever state
|
# Changing any of these changes what the rule is watching, so whatever state
|
||||||
@@ -460,15 +483,133 @@ async def apply_rule_change(db, rule: AlertRule, fields: dict, *, enabled: bool
|
|||||||
|
|
||||||
async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
|
async def add_rule(db, fields: dict, *, created_by: str, enabled: bool = True,
|
||||||
ansible_action: dict | None = None) -> AlertRule:
|
ansible_action: dict | None = None) -> AlertRule:
|
||||||
"""Insert a rule and its inactive state row, in the caller's transaction."""
|
"""Insert a rule and its inactive state row, in the caller's transaction.
|
||||||
|
A covering rule gets no state; its per-host children are made here."""
|
||||||
rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
|
rule = AlertRule(enabled=enabled, ansible_action=ansible_action, created_by=created_by, **fields)
|
||||||
db.add(rule)
|
db.add(rule)
|
||||||
await db.flush()
|
await db.flush()
|
||||||
|
if rule.covers != "resource":
|
||||||
|
await reconcile_children(db, rule)
|
||||||
|
return rule
|
||||||
db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
|
db.add(AlertState(rule_id=rule.id, state=AlertStateEnum.inactive,
|
||||||
last_transition_at=datetime.now(timezone.utc)))
|
last_transition_at=datetime.now(timezone.utc)))
|
||||||
return rule
|
return rule
|
||||||
|
|
||||||
|
|
||||||
|
# ── Covering rules (#5464) ───────────────────────────────────────────────────
|
||||||
|
# A rule can cover every host, or every host in an inventory group (inventory
|
||||||
|
# groups are Steward's grouping system: the Hosts ruling, #5294). Steward keeps
|
||||||
|
# one ordinary child rule per covered host, so each host's alert has its own
|
||||||
|
# state, history and acknowledgement, and a host that appears later is covered
|
||||||
|
# without anyone copying the rule.
|
||||||
|
|
||||||
|
COVERS = ("resource", "all_hosts", "group")
|
||||||
|
# Sources whose bare resource names are hosts ("<host>", parts are "<host>:…").
|
||||||
|
HOST_SOURCES = ("host_agent",)
|
||||||
|
# A host counts as present while it has reported within this long: the raw
|
||||||
|
# metrics window, so a decommissioned host's children go once its data does.
|
||||||
|
COVER_SEEN_DAYS = 7
|
||||||
|
|
||||||
|
|
||||||
|
def covers_text(rule: AlertRule) -> str:
|
||||||
|
"""What a covering rule spans, in words: "every host", "every host in web"."""
|
||||||
|
if rule.covers == "group":
|
||||||
|
return f"every host in {rule.cover_group}"
|
||||||
|
return "every host" if rule.covers == "all_hosts" else rule.resource_name
|
||||||
|
|
||||||
|
|
||||||
|
async def covered_hosts(db, rule: AlertRule, now: datetime | None = None) -> list[str]:
|
||||||
|
"""The host names a covering rule spans now: hosts that have reported
|
||||||
|
under its source within COVER_SEEN_DAYS, all of them or those whose
|
||||||
|
linked Ansible target is in its group."""
|
||||||
|
from datetime import timedelta
|
||||||
|
from steward.hosts.service import host_groups
|
||||||
|
from steward.models.hosts import Host
|
||||||
|
now = now or datetime.now(timezone.utc)
|
||||||
|
reported = set((await db.execute(
|
||||||
|
select(PluginMetricLatest.resource_name)
|
||||||
|
.where(PluginMetricLatest.source_module == rule.source_module,
|
||||||
|
PluginMetricLatest.recorded_at >= now - timedelta(days=COVER_SEEN_DAYS),
|
||||||
|
~PluginMetricLatest.resource_name.contains(":"))
|
||||||
|
.distinct())).scalars())
|
||||||
|
if rule.covers != "group":
|
||||||
|
return sorted(reported, key=str.lower)
|
||||||
|
hosts = (await db.execute(select(Host.id, Host.name).where(Host.name.in_(reported)))).all()
|
||||||
|
groups = await host_groups(db, [hid for hid, _ in hosts])
|
||||||
|
return sorted((name for hid, name in hosts if rule.cover_group in groups.get(hid, [])),
|
||||||
|
key=str.lower)
|
||||||
|
|
||||||
|
|
||||||
|
def child_fields(parent: AlertRule, host: str) -> dict:
|
||||||
|
"""A covering rule's child for one host: the parent's condition on that
|
||||||
|
host, named "<parent>: <host>"."""
|
||||||
|
return {"name": f"{parent.name}: {host}"[:128], "source_module": parent.source_module,
|
||||||
|
"resource_name": host, "metric_name": parent.metric_name,
|
||||||
|
"operator": parent.operator, "threshold": parent.threshold,
|
||||||
|
"consecutive_failures_required": parent.consecutive_failures_required,
|
||||||
|
"consecutive_ok_required": parent.consecutive_ok_required,
|
||||||
|
"notify_no_data": parent.notify_no_data}
|
||||||
|
|
||||||
|
|
||||||
|
async def reconcile_children(db, parent: AlertRule, *, actor_id: str | None = None,
|
||||||
|
now: datetime | None = None) -> tuple[list[str], list[str]]:
|
||||||
|
"""Bring a covering rule's children in line with it, in the caller's
|
||||||
|
transaction: one per covered host, each carrying the parent's condition,
|
||||||
|
action and on/off. A host newly covered gets a child; a child whose host
|
||||||
|
is no longer covered is deleted, its history with it. Returns (hosts
|
||||||
|
added, hosts removed)."""
|
||||||
|
hosts = set(await covered_hosts(db, parent, now))
|
||||||
|
children = {c.resource_name: c for c in (await db.execute(
|
||||||
|
select(AlertRule).where(AlertRule.parent_id == parent.id))).scalars()}
|
||||||
|
added = sorted(hosts - children.keys(), key=str.lower)
|
||||||
|
removed = sorted(children.keys() - hosts, key=str.lower)
|
||||||
|
for host in added:
|
||||||
|
await add_rule(db, {**child_fields(parent, host), "parent_id": parent.id},
|
||||||
|
created_by=parent.created_by, enabled=parent.enabled,
|
||||||
|
ansible_action=parent.ansible_action)
|
||||||
|
for host in removed:
|
||||||
|
await db.delete(children[host])
|
||||||
|
for host, child in children.items():
|
||||||
|
if host in hosts:
|
||||||
|
fields = child_fields(parent, host)
|
||||||
|
if (any(getattr(child, k) != v for k, v in fields.items())
|
||||||
|
or child.enabled != parent.enabled):
|
||||||
|
await apply_rule_change(db, child, fields, enabled=parent.enabled,
|
||||||
|
actor_id=actor_id)
|
||||||
|
if child.ansible_action != parent.ansible_action:
|
||||||
|
child.ansible_action = parent.ansible_action
|
||||||
|
return added, removed
|
||||||
|
|
||||||
|
|
||||||
|
async def reconcile_covering_rules(app) -> None:
|
||||||
|
"""Every covering rule's children brought in line with the hosts there are
|
||||||
|
now. Run each minute by the core scheduler (alert_reconcile), so a host
|
||||||
|
that joins a group, or starts reporting, is covered within a minute."""
|
||||||
|
async with app.db_sessionmaker() as db:
|
||||||
|
async with db.begin():
|
||||||
|
parents = (await db.execute(
|
||||||
|
select(AlertRule).where(AlertRule.covers != "resource"))).scalars().all()
|
||||||
|
for parent in parents:
|
||||||
|
added, removed = await reconcile_children(db, parent)
|
||||||
|
if added or removed:
|
||||||
|
logger.info("Covering rule %r: added %s, removed %s", parent.name,
|
||||||
|
", ".join(added) or "none", ", ".join(removed) or "none")
|
||||||
|
|
||||||
|
|
||||||
|
def part_of(rule: AlertRule, rules) -> AlertRule | None:
|
||||||
|
"""The covering rule a child belongs to, from a list of rules; None for a
|
||||||
|
rule that stands alone."""
|
||||||
|
return next((r for r in rules if r.id == rule.parent_id), None) if rule.parent_id else None
|
||||||
|
|
||||||
|
|
||||||
|
class PartOfCoveringRule(ValueError):
|
||||||
|
"""A child of a covering rule is changed through its parent, never alone."""
|
||||||
|
|
||||||
|
def __init__(self, child: AlertRule, parent_name: str):
|
||||||
|
super().__init__(f"{child.name} is part of the rule {parent_name!r}, which keeps one "
|
||||||
|
f"rule per host; change or disable it there.")
|
||||||
|
|
||||||
|
|
||||||
async def acknowledge(db, rule_id: str, user_id: str | None) -> AlertState:
|
async def acknowledge(db, rule_id: str, user_id: str | None) -> AlertState:
|
||||||
"""Acknowledge a firing alert, in the caller's transaction, and record it
|
"""Acknowledge a firing alert, in the caller's transaction, and record it
|
||||||
in the alert's history as a firing → acknowledged event by `user_id`, so
|
in the alert's history as a firing → acknowledged event by `user_id`, so
|
||||||
|
|||||||
+4
-2
@@ -408,8 +408,9 @@ def _register_core_tasks(app: Quart) -> None:
|
|||||||
await _cleanup(app)
|
await _cleanup(app)
|
||||||
|
|
||||||
async def run_alert_reconcile():
|
async def run_alert_reconcile():
|
||||||
|
from .alerts.service import reconcile_covering_rules
|
||||||
from .core.alerts import check_silence, notify_after_maintenance
|
from .core.alerts import check_silence, notify_after_maintenance
|
||||||
for step in (check_silence, notify_after_maintenance):
|
for step in (reconcile_covering_rules, check_silence, notify_after_maintenance):
|
||||||
try:
|
try:
|
||||||
await step(app)
|
await step(app)
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -439,7 +440,8 @@ def _register_core_tasks(app: Quart) -> None:
|
|||||||
interval_seconds=60,
|
interval_seconds=60,
|
||||||
run_on_startup=True,
|
run_on_startup=True,
|
||||||
),
|
),
|
||||||
# Rules whose series has gone quiet get a visible no_data mark; alerts
|
# Covering rules gain a child for each newly covered host (#5464);
|
||||||
|
# rules whose series has gone quiet get a visible no_data mark; alerts
|
||||||
# a maintenance window muted notify once it ends.
|
# a maintenance window muted notify once it ends.
|
||||||
ScheduledTask(
|
ScheduledTask(
|
||||||
name="alert_reconcile",
|
name="alert_reconcile",
|
||||||
|
|||||||
+69
-12
@@ -134,6 +134,11 @@ async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict
|
|||||||
if it has no state row. `no_data_since` is set while its metric is silent,
|
if it has no state row. `no_data_since` is set while its metric is silent,
|
||||||
whatever the state. `ansible_action` is the
|
whatever the state. `ansible_action` is the
|
||||||
playbook the rule runs when it fires, if any (`check: true` = dry run).
|
playbook the rule runs when it fires, if any (`check: true` = dry run).
|
||||||
|
|
||||||
|
`covers` is "resource" for a rule on one resource, or "all_hosts" /
|
||||||
|
"group" (with `group`) for a rule covering hosts: it has no state of its
|
||||||
|
own, `hosts` names the hosts it covers, and each host's rule is listed
|
||||||
|
with `part_of` naming it.
|
||||||
"""
|
"""
|
||||||
limit = clamp_limit(limit, 200, 500)
|
limit = clamp_limit(limit, 200, 500)
|
||||||
async with db_session() as db:
|
async with db_session() as db:
|
||||||
@@ -141,13 +146,19 @@ async def list_alert_rules(enabled_only: bool = False, limit: int = 200) -> dict
|
|||||||
states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
|
states = {s.rule_id: s for s in (await db.execute(select(AlertState))).scalars()}
|
||||||
if enabled_only:
|
if enabled_only:
|
||||||
rules = [r for r in rules if r.enabled]
|
rules = [r for r in rules if r.enabled]
|
||||||
|
names = {r.id: r.name for r in rules}
|
||||||
|
hosts: dict[str, list[str]] = {}
|
||||||
|
for r in rules:
|
||||||
|
if r.parent_id:
|
||||||
|
hosts.setdefault(r.parent_id, []).append(r.resource_name)
|
||||||
rows = [
|
rows = [
|
||||||
{
|
{
|
||||||
"rule_id": r.id,
|
"rule_id": r.id,
|
||||||
"rule_name": r.name,
|
"rule_name": r.name,
|
||||||
"enabled": r.enabled,
|
"enabled": r.enabled,
|
||||||
"source_module": r.source_module,
|
"source_module": r.source_module,
|
||||||
"resource": r.resource_name,
|
"resource": r.resource_name or None,
|
||||||
|
**_covers(r, hosts.get(r.id, []), names.get(r.parent_id)),
|
||||||
"metric": r.metric_name,
|
"metric": r.metric_name,
|
||||||
"condition": _condition(r),
|
"condition": _condition(r),
|
||||||
"state": alert_svc.alert_label(states[r.id]) if r.id in states else None,
|
"state": alert_svc.alert_label(states[r.id]) if r.id in states else None,
|
||||||
@@ -219,21 +230,51 @@ async def _rule(db, rule: str) -> AlertRule:
|
|||||||
return pick(rule, await alert_svc.list_rules(db), kind="alert rule", list_tool="list_alert_rules")
|
return pick(rule, await alert_svc.list_rules(db), kind="alert rule", list_tool="list_alert_rules")
|
||||||
|
|
||||||
|
|
||||||
def _rule_row(r: AlertRule) -> dict:
|
def _covers(r: AlertRule, hosts: list[str], parent_name: str | None) -> dict:
|
||||||
|
"""What a rule covers (#5464): the hosts under a covering rule, or the
|
||||||
|
covering rule a host's rule is part of."""
|
||||||
|
out: dict = {"covers": r.covers}
|
||||||
|
if r.covers == "group":
|
||||||
|
out["group"] = r.cover_group
|
||||||
|
if r.covers != "resource":
|
||||||
|
out["hosts"] = sorted(hosts, key=str.lower)
|
||||||
|
if r.parent_id:
|
||||||
|
out["part_of"] = parent_name
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
async def _rule_row(db, r: AlertRule) -> dict:
|
||||||
|
hosts = [c.resource_name for c in (await db.execute(
|
||||||
|
select(AlertRule).where(AlertRule.parent_id == r.id))).scalars()] if r.covers != "resource" else []
|
||||||
|
parent = await db.get(AlertRule, r.parent_id) if r.parent_id else None
|
||||||
return {"rule_id": r.id, "rule_name": r.name, "enabled": r.enabled,
|
return {"rule_id": r.id, "rule_name": r.name, "enabled": r.enabled,
|
||||||
"source_module": r.source_module, "resource": r.resource_name,
|
"source_module": r.source_module, "resource": r.resource_name or None,
|
||||||
|
**_covers(r, hosts, parent.name if parent else None),
|
||||||
"metric": r.metric_name, "condition": _condition(r),
|
"metric": r.metric_name, "condition": _condition(r),
|
||||||
"notify_no_data": r.notify_no_data, "ansible_action": _action(r)}
|
"notify_no_data": r.notify_no_data, "ansible_action": _action(r)}
|
||||||
|
|
||||||
|
|
||||||
|
async def _not_a_child(db, r: AlertRule) -> None:
|
||||||
|
"""A host's rule under a covering rule is changed through its parent."""
|
||||||
|
if r.parent_id:
|
||||||
|
parent = await db.get(AlertRule, r.parent_id)
|
||||||
|
raise alert_svc.PartOfCoveringRule(r, parent.name if parent else "?")
|
||||||
|
|
||||||
|
|
||||||
async def create_alert_rule(name: str, source_module: str, resource: str, metric: str,
|
async def create_alert_rule(name: str, source_module: str, resource: str, metric: str,
|
||||||
operator: str, threshold: float, consecutive_checks: int = 1,
|
operator: str, threshold: float, consecutive_checks: int = 1,
|
||||||
clear_after_checks: int = 1, notify_no_data: bool = True,
|
clear_after_checks: int = 1, notify_no_data: bool = True,
|
||||||
enabled: bool = True) -> dict:
|
enabled: bool = True, covers: str = "resource",
|
||||||
|
group: str = "") -> dict:
|
||||||
"""Add an alert rule: notify when `metric` of `resource` crosses `threshold`.
|
"""Add an alert rule: notify when `metric` of `resource` crosses `threshold`.
|
||||||
|
|
||||||
source_module, resource, metric: get_alert_rule_options lists the valid
|
source_module, resource, metric: get_alert_rule_options lists the valid
|
||||||
choices; a resource must have reported (or, for a monitor, exist).
|
choices; a resource must have reported (or, for a monitor, exist).
|
||||||
|
covers: "resource" (default) watches `resource`. "all_hosts" covers every
|
||||||
|
host, "group" every host in the inventory group `group`; pass
|
||||||
|
`resource` as "". Steward then keeps one rule per host, named
|
||||||
|
"<name>: <host>", and adds one within a minute for a host that starts
|
||||||
|
reporting or joins the group. Only for host_agent.
|
||||||
operator: ">", "<", ">=", "<=", "==" or "!=".
|
operator: ">", "<", ">=", "<=", "==" or "!=".
|
||||||
consecutive_checks: how many checks in a row must breach before it fires
|
consecutive_checks: how many checks in a row must breach before it fires
|
||||||
(default 1); higher values ride out a single blip.
|
(default 1); higher values ride out a single blip.
|
||||||
@@ -254,12 +295,15 @@ async def create_alert_rule(name: str, source_module: str, resource: str, metric
|
|||||||
db, name=name, source_module=source_module, resource_name=resource,
|
db, name=name, source_module=source_module, resource_name=resource,
|
||||||
metric_name=metric, operator=operator, threshold=threshold,
|
metric_name=metric, operator=operator, threshold=threshold,
|
||||||
consecutive_failures_required=consecutive_checks,
|
consecutive_failures_required=consecutive_checks,
|
||||||
consecutive_ok_required=clear_after_checks, notify_no_data=notify_no_data)
|
consecutive_ok_required=clear_after_checks, notify_no_data=notify_no_data,
|
||||||
|
covers=covers, cover_group=group)
|
||||||
r = await alert_svc.add_rule(db, fields, created_by=caller.user_id, enabled=enabled)
|
r = await alert_svc.add_rule(db, fields, created_by=caller.user_id, enabled=enabled)
|
||||||
|
row = await _rule_row(db, r)
|
||||||
await audit_mcp("alert_rule.created", "alert_rule", r.name,
|
await audit_mcp("alert_rule.created", "alert_rule", r.name,
|
||||||
{"rule_id": r.id, "source": r.source_module, "resource": r.resource_name,
|
{"rule_id": r.id, "source": r.source_module,
|
||||||
|
"resource": r.resource_name or alert_svc.covers_text(r),
|
||||||
"metric": r.metric_name})
|
"metric": r.metric_name})
|
||||||
return _rule_row(r)
|
return row
|
||||||
|
|
||||||
|
|
||||||
async def update_alert_rule(rule: str, name: str | None = None, source_module: str | None = None,
|
async def update_alert_rule(rule: str, name: str | None = None, source_module: str | None = None,
|
||||||
@@ -268,7 +312,7 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s
|
|||||||
consecutive_checks: int | None = None,
|
consecutive_checks: int | None = None,
|
||||||
clear_after_checks: int | None = None,
|
clear_after_checks: int | None = None,
|
||||||
notify_no_data: bool | None = None,
|
notify_no_data: bool | None = None,
|
||||||
enabled: bool | None = None) -> dict:
|
enabled: bool | None = None, group: str | None = None) -> dict:
|
||||||
"""Change an alert rule. Only the arguments you pass change; the rest stay.
|
"""Change an alert rule. Only the arguments you pass change; the rest stay.
|
||||||
|
|
||||||
rule: its name or id (list_alert_rules). Pass enabled=false to silence it
|
rule: its name or id (list_alert_rules). Pass enabled=false to silence it
|
||||||
@@ -277,9 +321,15 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s
|
|||||||
operator, threshold), or disabling or re-enabling it, puts its alert back
|
operator, threshold), or disabling or re-enabling it, puts its alert back
|
||||||
to inactive; a firing alert records a resolved event saying why. Recorded
|
to inactive; a firing alert records a resolved event saying why. Recorded
|
||||||
in the audit log.
|
in the audit log.
|
||||||
|
|
||||||
|
A rule covering hosts passes every change on to its per-host rules;
|
||||||
|
`group` moves a group-covering rule to another inventory group. What a
|
||||||
|
rule covers is fixed once it exists. A host's rule under a covering rule
|
||||||
|
is changed through that rule, not alone.
|
||||||
"""
|
"""
|
||||||
async with db_session() as db, db.begin():
|
async with db_session() as db, db.begin():
|
||||||
r = await _rule(db, rule)
|
r = await _rule(db, rule)
|
||||||
|
await _not_a_child(db, r)
|
||||||
fields = await alert_svc.rule_fields(
|
fields = await alert_svc.rule_fields(
|
||||||
db,
|
db,
|
||||||
name=r.name if name is None else name,
|
name=r.name if name is None else name,
|
||||||
@@ -292,24 +342,31 @@ async def update_alert_rule(rule: str, name: str | None = None, source_module: s
|
|||||||
if consecutive_checks is None else consecutive_checks),
|
if consecutive_checks is None else consecutive_checks),
|
||||||
consecutive_ok_required=(r.consecutive_ok_required
|
consecutive_ok_required=(r.consecutive_ok_required
|
||||||
if clear_after_checks is None else clear_after_checks),
|
if clear_after_checks is None else clear_after_checks),
|
||||||
notify_no_data=r.notify_no_data if notify_no_data is None else notify_no_data)
|
notify_no_data=r.notify_no_data if notify_no_data is None else notify_no_data,
|
||||||
|
covers=r.covers, cover_group=(r.cover_group or "") if group is None else group)
|
||||||
await alert_svc.apply_rule_change(db, r, fields, enabled=enabled,
|
await alert_svc.apply_rule_change(db, r, fields, enabled=enabled,
|
||||||
actor_id=current_caller().user_id)
|
actor_id=current_caller().user_id)
|
||||||
|
if r.covers != "resource":
|
||||||
|
await alert_svc.reconcile_children(db, r, actor_id=current_caller().user_id)
|
||||||
|
row = await _rule_row(db, r)
|
||||||
changed = sorted(k for k, v in {"name": name, "source_module": source_module,
|
changed = sorted(k for k, v in {"name": name, "source_module": source_module,
|
||||||
"resource": resource, "metric": metric, "operator": operator,
|
"resource": resource, "metric": metric, "operator": operator,
|
||||||
"threshold": threshold, "consecutive_checks": consecutive_checks,
|
"threshold": threshold, "consecutive_checks": consecutive_checks,
|
||||||
"clear_after_checks": clear_after_checks,
|
"clear_after_checks": clear_after_checks,
|
||||||
"notify_no_data": notify_no_data,
|
"notify_no_data": notify_no_data,
|
||||||
"enabled": enabled}.items() if v is not None)
|
"enabled": enabled, "group": group}.items() if v is not None)
|
||||||
await audit_mcp("alert_rule.updated", "alert_rule", r.name, {"rule_id": r.id, "changed": changed})
|
await audit_mcp("alert_rule.updated", "alert_rule", r.name, {"rule_id": r.id, "changed": changed})
|
||||||
return _rule_row(r)
|
return row
|
||||||
|
|
||||||
|
|
||||||
async def delete_alert_rule(rule: str) -> dict:
|
async def delete_alert_rule(rule: str) -> dict:
|
||||||
"""Delete an alert rule and its history. To silence it but keep it,
|
"""Delete an alert rule and its history. To silence it but keep it,
|
||||||
update_alert_rule(enabled=false). Admin only; recorded in the audit log."""
|
update_alert_rule(enabled=false). Deleting a rule that covers hosts
|
||||||
|
deletes its per-host rules and their history too. Admin only; recorded
|
||||||
|
in the audit log."""
|
||||||
async with db_session() as db, db.begin():
|
async with db_session() as db, db.begin():
|
||||||
r = await _rule(db, rule)
|
r = await _rule(db, rule)
|
||||||
|
await _not_a_child(db, r)
|
||||||
await db.delete(r)
|
await db.delete(r)
|
||||||
await audit_mcp("alert_rule.deleted", "alert_rule", r.name, {"rule_id": r.id})
|
await audit_mcp("alert_rule.deleted", "alert_rule", r.name, {"rule_id": r.id})
|
||||||
return {"deleted": r.name, "rule_id": r.id}
|
return {"deleted": r.name, "rule_id": r.id}
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
"""A rule can cover every host, or every host in an inventory group (#5464):
|
||||||
|
the covering rule keeps one child rule per host under it.
|
||||||
|
|
||||||
|
Revision ID: 0041_alert_rule_covers
|
||||||
|
Revises: 0040_alert_event_actor
|
||||||
|
Create Date: 2026-10-09
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
revision: str = "0041_alert_rule_covers"
|
||||||
|
down_revision: Union[str, None] = "0040_alert_event_actor"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("alert_rules", sa.Column("covers", sa.String(16), nullable=False,
|
||||||
|
server_default="resource"))
|
||||||
|
op.add_column("alert_rules", sa.Column("cover_group", sa.String(128), nullable=True))
|
||||||
|
op.add_column("alert_rules", sa.Column(
|
||||||
|
"parent_id", sa.String(36), sa.ForeignKey("alert_rules.id", ondelete="CASCADE"),
|
||||||
|
nullable=True))
|
||||||
|
op.create_index("ix_alert_rules_parent_id", "alert_rules", ["parent_id"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_alert_rules_parent_id", table_name="alert_rules")
|
||||||
|
op.drop_column("alert_rules", "parent_id")
|
||||||
|
op.drop_column("alert_rules", "cover_group")
|
||||||
|
op.drop_column("alert_rules", "covers")
|
||||||
@@ -55,6 +55,18 @@ class AlertRule(Base):
|
|||||||
created_at: Mapped[datetime] = mapped_column(
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
|
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc)
|
||||||
)
|
)
|
||||||
|
# What the rule covers (#5464): "resource" watches resource_name itself;
|
||||||
|
# "all_hosts" and "group" make it a covering rule over hosts (every host,
|
||||||
|
# or every host in the inventory group cover_group). A covering rule has
|
||||||
|
# no state and is never evaluated: Steward keeps one ordinary child rule
|
||||||
|
# per covered host (parent_id), so each host's alert keeps its own state,
|
||||||
|
# history and acknowledgement. Fixed when the rule is created.
|
||||||
|
covers: Mapped[str] = mapped_column(String(16), nullable=False, default="resource",
|
||||||
|
server_default="resource")
|
||||||
|
cover_group: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||||
|
parent_id: Mapped[str | None] = mapped_column(
|
||||||
|
String(36), ForeignKey("alert_rules.id", ondelete="CASCADE"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class AlertState(Base):
|
class AlertState(Base):
|
||||||
|
|||||||
@@ -216,6 +216,9 @@ strong, b, th { font-weight: 500; }
|
|||||||
/* The Docker page's one-line answer, above its tiles (#5298). */
|
/* The Docker page's one-line answer, above its tiles (#5298). */
|
||||||
.page-verdict { margin: 0 0 0.75rem; font-size: 0.95rem; }
|
.page-verdict { margin: 0 0 0.75rem; font-size: 0.95rem; }
|
||||||
.table tr.row-off > td { opacity: 0.55; }
|
.table tr.row-off > td { opacity: 0.55; }
|
||||||
|
/* A covering alert rule's per-host rules (#5464), indented under it. */
|
||||||
|
.table tr.row-child > td { border-top-style: dashed; }
|
||||||
|
.table tr.row-child .cell-name { padding-left: 2rem; }
|
||||||
/* A row that opens as a whole: its one .row-target link stretches over the
|
/* A row that opens as a whole: its one .row-target link stretches over the
|
||||||
row, and the actions cell sits above it so its buttons keep their own
|
row, and the actions cell sits above it so its buttons keep their own
|
||||||
clicks. One real link per row, so keyboard and screen readers meet it once. */
|
clicks. One real link per row, so keyboard and screen readers meet it once. */
|
||||||
|
|||||||
@@ -1,16 +1,49 @@
|
|||||||
{# alerts/_rule_fields.html — the resource and metric pickers for one source;
|
{# alerts/_rule_fields.html — the resource and metric pickers for one source;
|
||||||
the rule form includes it and swaps it in again when the source changes. #}
|
the rule form includes it and swaps it in again when the source changes.
|
||||||
|
For a source whose resources are hosts, the rule can instead cover every
|
||||||
|
host, or every host in an inventory group (#5464): Steward then keeps one
|
||||||
|
rule per host under it. What a rule covers is fixed once it exists. #}
|
||||||
|
{% set covers = current_covers|default("resource") %}
|
||||||
|
{% set one = covers == "resource" %}
|
||||||
|
{% if host_source|default(false) %}
|
||||||
<div class="form-group">
|
<div class="form-group">
|
||||||
|
<label for="rule-covers">Covers</label>
|
||||||
|
{% if covers_locked|default(false) %}
|
||||||
|
<input type="hidden" name="covers" value="{{ covers }}">
|
||||||
|
<p style="margin:0;">{% if one %}One host, or one part of a host{% elif covers == "all_hosts" %}Every host{% else %}Every host in a group{% endif %}
|
||||||
|
<span class="tone-dim">· fixed once the rule exists</span></p>
|
||||||
|
{% else %}
|
||||||
|
<select id="rule-covers" name="covers" data-covers>
|
||||||
|
<option value="resource" {% if one %}selected{% endif %}>One host, or one part of a host</option>
|
||||||
|
<option value="all_hosts" {% if covers == "all_hosts" %}selected{% endif %}>Every host</option>
|
||||||
|
<option value="group" {% if covers == "group" %}selected{% endif %} {% if not groups %}disabled{% endif %}>Every host in a group{% if not groups %} (no inventory groups yet){% endif %}</option>
|
||||||
|
</select>
|
||||||
|
{% endif %}
|
||||||
|
<p class="field-hint">Covering hosts keeps one alert per host under this rule, each named
|
||||||
|
"rule: host". A host that starts reporting, or joins the group, is covered within a minute;
|
||||||
|
one that leaves loses its alert and that alert's history.</p>
|
||||||
|
</div>
|
||||||
|
<div class="form-group" data-cover-group {% if covers != "group" %}hidden{% endif %}>
|
||||||
|
<label for="rule-group">Group</label>
|
||||||
|
<select id="rule-group" name="cover_group" {% if covers != "group" %}disabled{% endif %}>
|
||||||
|
{% for g in groups %}
|
||||||
|
<option value="{{ g }}" {% if g == current_group|default("") %}selected{% endif %}>{{ g }}</option>
|
||||||
|
{% endfor %}
|
||||||
|
</select>
|
||||||
|
<p class="field-hint">A host is in a group through its linked Ansible target (Ansible → Inventory).</p>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
<div class="form-group" data-cover-resource {% if not one %}hidden{% endif %}>
|
||||||
<label>Resource</label>
|
<label>Resource</label>
|
||||||
{% if resources %}
|
{% if resources %}
|
||||||
<select name="resource_name" required>
|
<select name="resource_name" required {% if not one %}disabled{% endif %}>
|
||||||
{% if not current_resource %}<option value="" disabled selected>— pick a resource —</option>{% endif %}
|
{% if not current_resource %}<option value="" disabled selected>— pick a resource —</option>{% endif %}
|
||||||
{% for r in resources %}
|
{% for r in resources %}
|
||||||
<option value="{{ r }}" {% if r == current_resource %}selected{% endif %}>{{ r }}</option>
|
<option value="{{ r }}" {% if r == current_resource %}selected{% endif %}>{{ r }}</option>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</select>
|
</select>
|
||||||
{% else %}
|
{% else %}
|
||||||
<select name="resource_name" required disabled>
|
<select name="resource_name" required disabled data-none="1">
|
||||||
<option value="">Nothing has reported from this source yet</option>
|
<option value="">Nothing has reported from this source yet</option>
|
||||||
</select>
|
</select>
|
||||||
<p class="field-hint">
|
<p class="field-hint">
|
||||||
|
|||||||
@@ -2,7 +2,8 @@
|
|||||||
(alerts/_active.html, polled), then every rule: what it is doing now and
|
(alerts/_active.html, polled), then every rule: what it is doing now and
|
||||||
when it last fired, what it fires on, and its History and Disable. The
|
when it last fired, what it fires on, and its History and Disable. The
|
||||||
whole row opens the rule (.row-link, its name is the link); deleting one is
|
whole row opens the rule (.row-link, its name is the link); deleting one is
|
||||||
on its own page, behind the confirm dialog. #}
|
on its own page, behind the confirm dialog. A rule covering hosts (#5464)
|
||||||
|
lists its per-host rules under it; each opens its history. #}
|
||||||
{% extends "base.html" %}
|
{% extends "base.html" %}
|
||||||
{% block title %}Alerts — Steward{% endblock %}
|
{% block title %}Alerts — Steward{% endblock %}
|
||||||
{% from "_macros.html" import page_header, status_badge, empty_state %}
|
{% from "_macros.html" import page_header, status_badge, empty_state %}
|
||||||
@@ -36,11 +37,13 @@
|
|||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{% for rule in rules %}
|
{% for rule in rules %}
|
||||||
|
{% set kids = (children or {}).get(rule.id, []) %}
|
||||||
{% set fired = (last_fired or {}).get(rule.id) %}
|
{% set fired = (last_fired or {}).get(rule.id) %}
|
||||||
<tr class="row-link{% if not rule.enabled %} row-off{% endif %}">
|
<tr class="row-link{% if not rule.enabled %} row-off{% endif %}">
|
||||||
<td class="cell-name">
|
<td class="cell-name">
|
||||||
<a class="row-target" href="{{ ('/alerts/rules/' ~ rule.id ~ '/edit') if can_edit else ('/alerts/history?rule=' ~ rule.id) }}" title="{{ rule.name }}">{{ rule.name }}</a>
|
<a class="row-target" href="{{ ('/alerts/rules/' ~ rule.id ~ '/edit') if can_edit else ('/alerts/history?rule=' ~ rule.id) }}" title="{{ rule.name }}">{{ rule.name }}</a>
|
||||||
<div class="cell-sub">{{ source_label(rule.source_module) }}</div>
|
<div class="cell-sub">{{ source_label(rule.source_module) }}
|
||||||
|
{%- if rule.covers != "resource" %} · {{ kids|length }} host{{ "s" if kids|length != 1 }}{% endif %}</div>
|
||||||
</td>
|
</td>
|
||||||
<td>
|
<td>
|
||||||
{%- set now_label = (rule_state or {}).get(rule.id) %}
|
{%- set now_label = (rule_state or {}).get(rule.id) %}
|
||||||
@@ -50,7 +53,7 @@
|
|||||||
<div class="cell-sub">{% if fired %}last fired {{ fired|ago }}{% else %}never fired{% endif %}</div>
|
<div class="cell-sub">{% if fired %}last fired {{ fired|ago }}{% else %}never fired{% endif %}</div>
|
||||||
</td>
|
</td>
|
||||||
<td>
|
<td>
|
||||||
{{ rule.resource_name }} · {{ rule_condition(rule) }}
|
{% if rule.covers == "group" %}every host in {{ rule.cover_group }}{% elif rule.covers == "all_hosts" %}every host{% else %}{{ rule.resource_name }}{% endif %} · {{ rule_condition(rule) }}
|
||||||
<div class="cell-sub">
|
<div class="cell-sub">
|
||||||
{%- set n_fire, n_ok = rule.consecutive_failures_required, rule.consecutive_ok_required %}
|
{%- set n_fire, n_ok = rule.consecutive_failures_required, rule.consecutive_ok_required %}
|
||||||
{%- if n_fire > 1 %}after {{ n_fire }} checks in a row{% else %}on the first check{% endif %}
|
{%- if n_fire > 1 %}after {{ n_fire }} checks in a row{% else %}on the first check{% endif %}
|
||||||
@@ -58,7 +61,7 @@
|
|||||||
</div>
|
</div>
|
||||||
</td>
|
</td>
|
||||||
<td class="td-actions">
|
<td class="td-actions">
|
||||||
<a href="/alerts/history?rule={{ rule.id }}" class="btn btn-sm btn-ghost">History</a>
|
{% if rule.covers == "resource" %}<a href="/alerts/history?rule={{ rule.id }}" class="btn btn-sm btn-ghost">History</a>{% endif %}
|
||||||
{% if can_edit %}
|
{% if can_edit %}
|
||||||
<form method="post" action="/alerts/rules/{{ rule.id }}/toggle" style="display:inline;">{{ csrf_field() }}
|
<form method="post" action="/alerts/rules/{{ rule.id }}/toggle" style="display:inline;">{{ csrf_field() }}
|
||||||
<button type="submit" class="btn btn-sm btn-ghost">
|
<button type="submit" class="btn btn-sm btn-ghost">
|
||||||
@@ -68,6 +71,23 @@
|
|||||||
{% endif %}
|
{% endif %}
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
{% for kid in kids %}
|
||||||
|
{% set kid_label = (rule_state or {}).get(kid.id) %}
|
||||||
|
{% set kid_fired = (last_fired or {}).get(kid.id) %}
|
||||||
|
<tr class="row-link row-child{% if not kid.enabled %} row-off{% endif %}">
|
||||||
|
<td class="cell-name">
|
||||||
|
<a class="row-target" href="/alerts/history?rule={{ kid.id }}" title="{{ kid.name }}">{{ kid.resource_name }}</a>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{%- if not kid.enabled %}<span class="tone-dim">off</span>
|
||||||
|
{%- elif kid_label and kid_label not in ('inactive', 'resolved') %}{{ status_badge(kid_label) }}
|
||||||
|
{%- else %}<span class="tone-ok" title="Within its threshold">quiet</span>{% endif -%}
|
||||||
|
<div class="cell-sub">{% if kid_fired %}last fired {{ kid_fired|ago }}{% else %}never fired{% endif %}</div>
|
||||||
|
</td>
|
||||||
|
<td class="tone-dim">as above</td>
|
||||||
|
<td class="td-actions"><a href="/alerts/history?rule={{ kid.id }}" class="btn btn-sm btn-ghost">History</a></td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -17,6 +17,13 @@
|
|||||||
{%- if fired %}Last fired {{ fired|ago }}{% else %}Never fired{% endif -%}
|
{%- if fired %}Last fired {{ fired|ago }}{% else %}Never fired{% endif -%}
|
||||||
{%- if history_count is defined and history_count is not none %} · {{ history_count }} change{{ "s" if history_count != 1 }} of state on record{% endif -%}
|
{%- if history_count is defined and history_count is not none %} · {{ history_count }} change{{ "s" if history_count != 1 }} of state on record{% endif -%}
|
||||||
</p>
|
</p>
|
||||||
|
{% if rule.covers != "resource" %}
|
||||||
|
<p class="field-hint" style="margin-top:-0.5rem;">
|
||||||
|
Covers {% if rule.covers == "group" %}every host in <strong>{{ rule.cover_group }}</strong>{% else %}every host{% endif %}:
|
||||||
|
{% if covered %}{{ covered|length }} now ({{ covered|join(", ") }}), each with its own alert and history.
|
||||||
|
{%- else %}no host yet; one is covered within a minute of reporting.{% endif %}
|
||||||
|
Changes here apply to all of them.</p>
|
||||||
|
{% endif %}
|
||||||
{% else %}
|
{% else %}
|
||||||
{{ page_header("New alert rule") }}
|
{{ page_header("New alert rule") }}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
@@ -212,6 +219,16 @@
|
|||||||
document.addEventListener("input", function (e) {
|
document.addEventListener("input", function (e) {
|
||||||
if (e.target.name === "threshold" && !e.target.form.querySelector("[data-threshold-echo]").hidden) showDuration(e.target.form);
|
if (e.target.name === "threshold" && !e.target.form.querySelector("[data-threshold-echo]").hidden) showDuration(e.target.form);
|
||||||
});
|
});
|
||||||
|
// Covering hosts swaps the resource picker for the group picker (#5464).
|
||||||
|
document.addEventListener("change", function (e) {
|
||||||
|
if (!e.target.hasAttribute("data-covers")) return;
|
||||||
|
var form = e.target.form, v = e.target.value;
|
||||||
|
var res = form.querySelector("[data-cover-resource]"), grp = form.querySelector("[data-cover-group]");
|
||||||
|
res.hidden = v !== "resource";
|
||||||
|
res.querySelectorAll("select").forEach(function (s) { s.disabled = v !== "resource" || s.dataset.none === "1"; });
|
||||||
|
grp.hidden = v !== "group";
|
||||||
|
grp.querySelector("select").disabled = v !== "group";
|
||||||
|
});
|
||||||
// The playbook's fields show only while "run a playbook" is ticked.
|
// The playbook's fields show only while "run a playbook" is ticked.
|
||||||
document.addEventListener("change", function (e) {
|
document.addEventListener("change", function (e) {
|
||||||
if (!e.target.hasAttribute("data-ansible-toggle")) return;
|
if (!e.target.hasAttribute("data-ansible-toggle")) return;
|
||||||
|
|||||||
@@ -17,7 +17,8 @@ def _rule(metric="cpu_pct", op=AlertOperator.gt, threshold=90.0, resource="web-1
|
|||||||
return SimpleNamespace(id=rid, name=f"{resource} {metric}", source_module="host_agent",
|
return SimpleNamespace(id=rid, name=f"{resource} {metric}", source_module="host_agent",
|
||||||
resource_name=resource, metric_name=metric, operator=op,
|
resource_name=resource, metric_name=metric, operator=op,
|
||||||
threshold=threshold, enabled=True,
|
threshold=threshold, enabled=True,
|
||||||
consecutive_failures_required=1, consecutive_ok_required=1)
|
consecutive_failures_required=1, consecutive_ok_required=1,
|
||||||
|
covers="resource", cover_group=None, parent_id=None)
|
||||||
|
|
||||||
|
|
||||||
def _said(says):
|
def _said(says):
|
||||||
@@ -272,3 +273,35 @@ async def test_the_rule_form_speaks_in_names_and_units(app):
|
|||||||
assert '<option value="down" selected>' in page # the binary choice, shown
|
assert '<option value="down" selected>' in page # the binary choice, shown
|
||||||
assert 'name="threshold" step="any" required disabled' in page # the raw threshold, off
|
assert 'name="threshold" step="any" required disabled' in page # the raw threshold, off
|
||||||
assert ">is_up<" not in page and "SNMP" not in page # no raw key; empty source skipped
|
assert ">is_up<" not in page and "SNMP" not in page # no raw key; empty source skipped
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_rule_covering_hosts_lists_its_hosts_under_it(app):
|
||||||
|
"""#5464: a covering rule reads as what it covers, with each host's rule
|
||||||
|
nested under it opening that host's history; its page names the hosts."""
|
||||||
|
parent = _rule("seconds_since_seen", AlertOperator.gt, 180.0, "", rid="p1")
|
||||||
|
parent.name, parent.covers, parent.cover_group = "Agent offline", "group", "swarm"
|
||||||
|
parent.ansible_action, parent.notify_no_data = None, True
|
||||||
|
kids = []
|
||||||
|
for i, host in enumerate(("node-a", "node-b")):
|
||||||
|
k = _rule("seconds_since_seen", AlertOperator.gt, 180.0, host, rid=f"k{i}")
|
||||||
|
k.name, k.parent_id = f"Agent offline: {host}", "p1"
|
||||||
|
kids.append(k)
|
||||||
|
async with app.test_request_context("/alerts/"):
|
||||||
|
session["user_role"] = "operator"
|
||||||
|
listing = await render_template(
|
||||||
|
"alerts/list.html", active=[], rules=[parent], children={"p1": kids},
|
||||||
|
rule_state={"p1": "firing", "k0": "firing", "k1": "inactive"}, operators=[])
|
||||||
|
page = await render_template(
|
||||||
|
"alerts/rules_form.html", rule=parent, operators=[], source_modules=["host_agent"],
|
||||||
|
initial_source="host_agent", resources=[], metrics=["seconds_since_seen"],
|
||||||
|
ansible_sources=[], catalog={}, history_count=0, fired=None, notify=[],
|
||||||
|
covered=["node-a", "node-b"], host_source=True, groups=["swarm"],
|
||||||
|
current_covers="group", current_group="swarm", covers_locked=True)
|
||||||
|
listing, page = " ".join(listing.split()), " ".join(page.split())
|
||||||
|
assert "every host in swarm" in listing and "2 hosts" in listing
|
||||||
|
assert listing.index("Agent offline</a>") < listing.index('href="/alerts/history?rule=k0"') \
|
||||||
|
< listing.index('href="/alerts/history?rule=k1"')
|
||||||
|
assert 'class="row-link row-child"' in listing
|
||||||
|
assert "Covers every host in <strong>swarm</strong>: 2 now (node-a, node-b)" in page
|
||||||
|
assert 'name="covers" value="group"' in page and "fixed once the rule exists" in page
|
||||||
|
assert 'data-cover-resource hidden' in page
|
||||||
|
|||||||
@@ -0,0 +1,173 @@
|
|||||||
|
"""Integration: a rule covering hosts (#5464) keeps one ordinary rule per
|
||||||
|
covered host, adds one for a host that joins its group, removes one for a host
|
||||||
|
that leaves, passes edits and on/off down to them, and refuses a change made
|
||||||
|
to a host's rule alone. Each host's rule fires on its own readings."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def app():
|
||||||
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
||||||
|
pytest.skip("needs Postgres")
|
||||||
|
from steward.app import create_app
|
||||||
|
return create_app(testing=False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_group_rule_keeps_one_rule_per_host_in_the_group(app):
|
||||||
|
from sqlalchemy import select
|
||||||
|
from steward.alerts import service as alert_svc
|
||||||
|
from steward.core.alerts import record_metric
|
||||||
|
from steward.models.alerts import AlertRule, AlertState, AlertStateEnum
|
||||||
|
from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget
|
||||||
|
from steward.models.hosts import Host
|
||||||
|
from steward.models.users import User, UserRole
|
||||||
|
|
||||||
|
async def _go():
|
||||||
|
tag = uuid.uuid4().hex[:8]
|
||||||
|
names = [f"cv-{tag}-{i}" for i in range(3)]
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
async with s.begin():
|
||||||
|
user = User(username=f"cv-{tag}", email=f"cv-{tag}@test", password_hash="x",
|
||||||
|
role=UserRole.admin)
|
||||||
|
hosts = [Host(name=n, address=f"10.8.{i}.1") for i, n in enumerate(names)]
|
||||||
|
s.add_all([user, *hosts])
|
||||||
|
await s.flush()
|
||||||
|
group = AnsibleGroup(name=f"web {tag}")
|
||||||
|
targets = [AnsibleTarget(name=f"t-{n}", address=h.address, host_id=h.id)
|
||||||
|
for n, h in zip(names, hosts)]
|
||||||
|
for t in targets[:2]:
|
||||||
|
t.groups = [group]
|
||||||
|
s.add_all([group, *targets])
|
||||||
|
for n in names:
|
||||||
|
await record_metric(s, "host_agent", n, "cpu_pct", 10.0)
|
||||||
|
fields = await alert_svc.rule_fields(
|
||||||
|
s, name=f"Hot {tag}", source_module="host_agent", resource_name="",
|
||||||
|
metric_name="cpu_pct", operator=">", threshold=90,
|
||||||
|
covers="group", cover_group=group.name)
|
||||||
|
parent = await alert_svc.add_rule(s, fields, created_by=user.id)
|
||||||
|
ids = {"parent": parent.id, "user": user.id, "third": targets[2].id,
|
||||||
|
"group": group.id}
|
||||||
|
|
||||||
|
async def kids():
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
rows = (await s.execute(select(AlertRule).where(
|
||||||
|
AlertRule.parent_id == ids["parent"]))).scalars().all()
|
||||||
|
return {r.resource_name: r for r in rows}
|
||||||
|
|
||||||
|
async def reconcile():
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
async with s.begin():
|
||||||
|
return await alert_svc.reconcile_children(
|
||||||
|
s, await s.get(AlertRule, ids["parent"]))
|
||||||
|
|
||||||
|
# The two hosts in the group, each its own rule; the parent has no state.
|
||||||
|
got = await kids()
|
||||||
|
assert sorted(got) == names[:2]
|
||||||
|
assert got[names[0]].name == f"Hot {tag}: {names[0]}"
|
||||||
|
assert got[names[0]].threshold == 90 and got[names[0]].covers == "resource"
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
assert (await s.execute(select(AlertState).where(
|
||||||
|
AlertState.rule_id == ids["parent"]))).scalar_one_or_none() is None
|
||||||
|
|
||||||
|
# A host's own readings fire its own rule only.
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
async with s.begin():
|
||||||
|
await record_metric(s, "host_agent", names[0], "cpu_pct", 97.0)
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
states = {st.rule_id: st.state for st in (await s.execute(select(AlertState).where(
|
||||||
|
AlertState.rule_id.in_([r.id for r in got.values()])))).scalars()}
|
||||||
|
assert states[got[names[0]].id] == AlertStateEnum.firing
|
||||||
|
assert states[got[names[1]].id] == AlertStateEnum.inactive
|
||||||
|
|
||||||
|
# The third host joins the group: covered on the next reconcile.
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
async with s.begin():
|
||||||
|
from sqlalchemy.orm import selectinload
|
||||||
|
t = (await s.execute(select(AnsibleTarget).where(AnsibleTarget.id == ids["third"])
|
||||||
|
.options(selectinload(AnsibleTarget.groups)))).scalar_one()
|
||||||
|
t.groups = [await s.get(AnsibleGroup, ids["group"])]
|
||||||
|
assert await reconcile() == ([names[2]], [])
|
||||||
|
assert sorted(await kids()) == names
|
||||||
|
|
||||||
|
# Raising the threshold passes down; the firing host's alert resets.
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
async with s.begin():
|
||||||
|
p = await s.get(AlertRule, ids["parent"])
|
||||||
|
fields = await alert_svc.rule_fields(
|
||||||
|
s, name=p.name, source_module="host_agent", resource_name="",
|
||||||
|
metric_name="cpu_pct", operator=">", threshold=99,
|
||||||
|
covers="group", cover_group=p.cover_group)
|
||||||
|
await alert_svc.apply_rule_change(s, p, fields, actor_id=ids["user"])
|
||||||
|
await alert_svc.reconcile_children(s, p, actor_id=ids["user"])
|
||||||
|
got = await kids()
|
||||||
|
assert {r.threshold for r in got.values()} == {99}
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
st = (await s.execute(select(AlertState).where(
|
||||||
|
AlertState.rule_id == got[names[0]].id))).scalar_one()
|
||||||
|
assert st.state == AlertStateEnum.inactive
|
||||||
|
|
||||||
|
# Disabling the parent disables every host's rule.
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
async with s.begin():
|
||||||
|
p = await s.get(AlertRule, ids["parent"])
|
||||||
|
await alert_svc.apply_rule_change(s, p, {}, enabled=False)
|
||||||
|
await alert_svc.reconcile_children(s, p)
|
||||||
|
assert not any(r.enabled for r in (await kids()).values())
|
||||||
|
|
||||||
|
# A host leaving the group loses its rule.
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
async with s.begin():
|
||||||
|
t = (await s.execute(select(AnsibleTarget).where(AnsibleTarget.id == ids["third"])
|
||||||
|
.options(selectinload(AnsibleTarget.groups)))).scalar_one()
|
||||||
|
t.groups = []
|
||||||
|
assert await reconcile() == ([], [names[2]])
|
||||||
|
assert sorted(await kids()) == names[:2]
|
||||||
|
|
||||||
|
asyncio.run(_go())
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_host_rule_is_changed_through_its_covering_rule(app):
|
||||||
|
"""Over MCP, a host's rule under a covering rule refuses an edit made to it
|
||||||
|
alone, and the covering rule lists the hosts it covers."""
|
||||||
|
from steward.alerts import service as alert_svc
|
||||||
|
from steward.core.alerts import record_metric
|
||||||
|
from steward.core.api_tokens import Caller
|
||||||
|
from steward.mcp import _context
|
||||||
|
from steward.mcp.tools import alerts as a
|
||||||
|
from steward.models.hosts import Host
|
||||||
|
from steward.models.users import User, UserRole
|
||||||
|
|
||||||
|
async def _go():
|
||||||
|
tag = uuid.uuid4().hex[:8]
|
||||||
|
name = f"cvm-{tag}"
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
async with s.begin():
|
||||||
|
user = User(username=f"cvm-{tag}", email=f"cvm-{tag}@test", password_hash="x",
|
||||||
|
role=UserRole.admin)
|
||||||
|
s.add_all([user, Host(name=name, address="10.8.9.1")])
|
||||||
|
await s.flush()
|
||||||
|
await record_metric(s, "host_agent", name, "seconds_since_seen", 5.0)
|
||||||
|
uid = user.id
|
||||||
|
_context._app_ctx.set(app)
|
||||||
|
_context._caller_ctx.set(Caller(user_id=uid, username=f"cvm-{tag}",
|
||||||
|
role=UserRole.admin, token_id=7))
|
||||||
|
made = await a.create_alert_rule(f"Offline {tag}", "host_agent", "",
|
||||||
|
"seconds_since_seen", ">", 180, covers="all_hosts")
|
||||||
|
assert made["covers"] == "all_hosts" and name in made["hosts"]
|
||||||
|
with pytest.raises(alert_svc.PartOfCoveringRule, match="change or disable it there"):
|
||||||
|
await a.update_alert_rule(f"Offline {tag}: {name}", threshold=60)
|
||||||
|
listed = {r["rule_name"]: r for r in (await a.list_alert_rules(limit=500))["rules"]}
|
||||||
|
assert listed[f"Offline {tag}: {name}"]["part_of"] == f"Offline {tag}"
|
||||||
|
await a.update_alert_rule(f"Offline {tag}", threshold=300)
|
||||||
|
listed = {r["rule_name"]: r for r in (await a.list_alert_rules(limit=500))["rules"]}
|
||||||
|
assert listed[f"Offline {tag}: {name}"]["condition"].startswith("seconds_since_seen > 300")
|
||||||
|
|
||||||
|
asyncio.run(_go())
|
||||||
Reference in New Issue
Block a user