feat(alerts): pick the host, then its part, and see what the metric reads (#5475)

#5475 "Rule form: pick the host, then its part; metrics that part reports;
a link to groups; a preview of the metric", from the operator's review of
the New rule page: hosts and their parts shared one long picker.

- Host agent rules pick a Host, then a Part: "The whole host", then its
  mounts, CPU cores, disks, interfaces and sensors under a heading each,
  cores in number order. A part that reports nothing a rule can watch
  (an interface's traffic) isn't offered.
- The Metric list holds what the picked resource reports, in catalog order.
- Under it, a preview: the reading now, and the last 24 h low to high with
  an hourly sparkline drawn from zero (UI ruling 8), read from one series.
- The group picker links to Ansible > Inventory to make a group, in a new
  tab, and re-reads the groups when the tab comes back into view.
- Picking a host or part swaps the pickers alone (/alerts/api/rule_target);
  picking a metric swaps the preview (/alerts/api/metric_preview).
- A rule's own resource and metric stay on offer after they go quiet.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/routes.py

@@ -256,21 +256,65 @@ HISTORY_SUMMARY_ROWS = 2000
 @alerts_bp.get("/api/rule_fields")
 @require_role(UserRole.viewer)
 async def rule_fields_api():
+    """Everything the rule watches, for a newly picked source."""
     source = request.args.get("source_module", "")
-    current_resource = request.args.get("current_resource", "")
-    current_metric = request.args.get("current_metric", "")
     async with current_app.db_sessionmaker() as db:
-        resources = await alert_svc.resources_for(db, source) if source else []
-        metrics = await alert_svc.metrics_for(db, source)
-        cover = await _cover_choices(db, source)
-    return await render_template(
-        "alerts/_rule_fields.html",
-        resources=resources,
-        metrics=metrics,
-        current_resource=current_resource,
-        current_metric=current_metric,
-        **cover,
-    )
+        ctx = {**await _cover_choices(db, source), **await _target(db, source)}
+    return await render_template("alerts/_rule_fields.html", **ctx)
+
+
+@alerts_bp.get("/api/rule_target")
+@require_role(UserRole.viewer)
+async def rule_target_api():
+    """The host, part and metric pickers and the preview, after a host or a
+    part is picked (#5475)."""
+    async with current_app.db_sessionmaker() as db:
+        ctx = await _target_from_args(db)
+    return await render_template("alerts/_rule_target.html", **ctx)
+
+
+@alerts_bp.get("/api/metric_preview")
+@require_role(UserRole.viewer)
+async def metric_preview_api():
+    """What the picked metric reads now and over the last day (#5475)."""
+    async with current_app.db_sessionmaker() as db:
+        ctx = await _target_from_args(db)
+    return await render_template("alerts/_metric_preview.html", **ctx)
+
+
+@alerts_bp.get("/api/cover_groups")
+@require_role(UserRole.viewer)
+async def cover_groups_api():
+    """The inventory groups, re-read when the reader comes back from making one."""
+    from steward.hosts.service import group_names
+    async with current_app.db_sessionmaker() as db:
+        groups = await group_names(db)
+    return await render_template("alerts/_cover_groups.html", groups=groups,
+                                 current_group=request.args.get("cover_group", ""),
+                                 current_covers=request.args.get("covers", "resource"))
+
+
+async def _target(db, source: str, *, host: str = "", resource: str = "", metric: str = "",
+                  covers: str = "resource") -> dict:
+    """The rule form's target pickers and the chosen metric's preview (#5475)."""
+    from steward.core.status import sparkline_svg
+    target = await alert_svc.rule_target(db, source, host=host, resource=resource, metric=metric)
+    preview = None
+    if target["resource"] and target["metric"]:
+        preview = await alert_svc.metric_preview(db, source, target["resource"], target["metric"])
+        if preview:
+            # From zero to the series' own peak (UI & Design ruling 8), so a
+            # disk at 61-62% reads flat rather than as a cliff.
+            preview["spark"] = sparkline_svg(preview["series"], width=120, height=20,
+                                             stroke="currentColor", floor=1.0)
+    return {**target, "preview": preview, "current_covers": covers}
+
+
+async def _target_from_args(db) -> dict:
+    a = request.args
+    return await _target(db, a.get("source_module", ""), host=a.get("pick_host", ""),
+                         resource=a.get("resource_name", ""), metric=a.get("metric_name", ""),
+                         covers=a.get("covers", "resource"))
 
 
 async def _cover_choices(db, source: str, rule: AlertRule | None = None) -> dict:
@@ -290,16 +334,13 @@ async def _cover_choices(db, source: str, rule: AlertRule | None = None) -> dict
 async def new_rule():
     first_source = alert_svc.source_modules()[0]
     async with current_app.db_sessionmaker() as db:
-        resources = await alert_svc.resources_for(db, first_source)
-        cover = await _cover_choices(db, first_source)
+        ctx = {**await _cover_choices(db, first_source), **await _target(db, first_source)}
     return await render_template(
         "alerts/rules_form.html",
-        rule=None, **cover,
+        rule=None, **ctx,
         operators=_OPERATORS,
         source_modules=alert_svc.source_modules(),
         initial_source=first_source,
-        resources=resources,
-        metrics=alert_svc.metric_catalog().get(first_source, []),
         catalog=alert_svc.metric_catalog(),
         ansible_sources=_ansible_source_names(),
         notify=await _notify(),
@@ -317,8 +358,9 @@ async def edit_rule(rule_id: str):
         if rule.parent_id:
             # A host's rule under a covering rule is changed through it.
             return redirect(f"/alerts/rules/{rule.parent_id}/edit")
-        resources = await alert_svc.resources_for(db, rule.source_module)
-        cover = await _cover_choices(db, rule.source_module, rule)
+        ctx = {**await _cover_choices(db, rule.source_module, rule),
+               **await _target(db, rule.source_module, resource=rule.resource_name,
+                               metric=rule.metric_name, covers=rule.covers)}
         hosts = sorted((c.resource_name for c in (await db.execute(
             select(AlertRule).where(AlertRule.parent_id == rule.id))).scalars()), key=str.lower)
         history_count = (await db.execute(
@@ -327,12 +369,10 @@ async def edit_rule(rule_id: str):
     return await render_template(
         "alerts/rules_form.html",
         rule=rule, history_count=history_count, fired=fired, notify=await _notify(),
-        covered=hosts, **cover,
+        covered=hosts, **ctx,
         operators=_OPERATORS,
         source_modules=alert_svc.source_modules(),
         initial_source=rule.source_module,
-        resources=resources,
-        metrics=alert_svc.metric_catalog().get(rule.source_module, []),
         catalog=alert_svc.metric_catalog(),
         ansible_sources=_ansible_source_names(),
     )

steward/alerts/service.py

@@ -358,6 +358,124 @@ async def resources_for(db, source_module: str) -> list[str]:
     return sorted(resources)
 
 
+# A host's parts, by the shape of the name after "<host>:" (#5475): the order
+# and heading each kind takes in the rule form's Part picker.
+PART_KINDS = (("mount", "Mounts"), ("core", "CPU cores"), ("disk", "Disks"),
+              ("net", "Interfaces"), ("temp", "Temperatures"), ("other", "Other"))
+
+
+def part_kind(part: str) -> tuple[str, str]:
+    """(kind, label) for a host part: "/var" is a mount, "core3" a core,
+    "diskio:sda" a disk, "net:eth0" an interface, "temp:Core 0" a sensor."""
+    if part.startswith("/"):
+        return "mount", part
+    if part.startswith("core") and part[4:].isdigit():
+        return "core", f"core {part[4:]}"
+    for prefix, kind in (("diskio:", "disk"), ("net:", "net"), ("temp:", "temp")):
+        if part.startswith(prefix):
+            return kind, part[len(prefix):]
+    return "other", part
+
+
+def _part_sort(part: str):
+    kind, label = part_kind(part)
+    return (int(part[4:]), "") if kind == "core" else (0, label.lower())
+
+
+def alertable(source_module: str, reported: set[str]) -> list[str]:
+    """The metrics a rule on a resource can watch, from what it reports: the
+    source's catalog, in its order, narrowed to those; every reported name
+    for a source with no catalog (snmp)."""
+    catalog = metric_catalog().get(source_module, [])
+    return [m for m in catalog if m in reported] if catalog else sorted(reported)
+
+
+async def rule_target(db, source_module: str, *, host: str = "", resource: str = "",
+                      metric: str = "") -> dict:
+    """What the rule form's target pickers offer (#5475), from the latest-value
+    table: the resources that report something a rule can watch, and for the
+    chosen one the metrics it reports. For a host source the resources split
+    into a Host picker and a Part picker ("the whole host", then the host's
+    mounts, cores and so on). A resource or metric already chosen stays on
+    offer even when it has gone quiet, so editing a rule never drops them.
+
+    Returns host_source, hosts, host, parts [(heading, [(value, label)])],
+    resources, resource, metrics, metric."""
+    catalog = metric_catalog().get(source_module, [])
+    seen: dict[str, set[str]] = {}
+    for res, name in (await db.execute(
+            select(PluginMetricLatest.resource_name, PluginMetricLatest.metric_name)
+            .where(PluginMetricLatest.source_module == source_module))).all():
+        seen.setdefault(res, set()).add(name)
+    pickable = {r: ms for r, names in seen.items() if (ms := alertable(source_module, names))}
+    if source_module in MONITOR_TYPE_VALUES:
+        for (name,) in (await db.execute(select(Monitor.name).where(Monitor.type == source_module))).all():
+            pickable.setdefault(name, list(catalog))
+    if resource and resource not in pickable:
+        pickable[resource] = list(catalog) or ([metric] if metric else [])
+
+    out: dict = {"host_source": source_module in HOST_SOURCES, "hosts": [], "host": "",
+                 "parts": [], "resources": [], "resource": resource}
+    if out["host_source"]:
+        hosts = sorted({r.split(":", 1)[0] for r in pickable}, key=str.lower)
+        host = host or resource.split(":", 1)[0]
+        if host not in hosts:
+            host = hosts[0] if hosts else ""
+        mine = [r for r in pickable if r == host or r.startswith(host + ":")]
+        if resource not in mine:
+            resource = host if host in pickable else (sorted(mine)[0] if mine else "")
+        by_kind: dict[str, list] = {}
+        for r in sorted((r for r in mine if r != host), key=lambda r: _part_sort(r[len(host) + 1:])):
+            kind, label = part_kind(r[len(host) + 1:])
+            by_kind.setdefault(kind, []).append((r, label))
+        parts = [("", [(host, "The whole host")])] if host in pickable else []
+        parts += [(heading, by_kind[k]) for k, heading in PART_KINDS if k in by_kind]
+        out.update(hosts=hosts, host=host, parts=parts, resource=resource)
+    else:
+        out["resources"] = sorted(pickable, key=str.lower)
+    metrics = list(pickable.get(out["resource"]) or catalog)
+    if metric and metric not in metrics and (metric in catalog or not catalog):
+        metrics.append(metric)
+    out.update(metrics=metrics, metric=metric if metric in metrics else "")
+    return out
+
+
+# How far back the rule form's metric preview looks: a day shows the daily
+# swing (backups, batch jobs) that a threshold has to sit above.
+PREVIEW_HOURS = 24
+
+
+async def metric_preview(db, source_module: str, resource: str, metric: str,
+                         now: datetime | None = None) -> dict | None:
+    """What the chosen metric reads (#5475), so a threshold is set against real
+    numbers: the latest reading and when, and the last PREVIEW_HOURS as hourly
+    averages with the lowest and highest. Reads one series: the latest-value
+    row, then a recorded_at range on plugin_metrics' series index. None when
+    it has no reading."""
+    from datetime import timedelta
+    from steward.models.metrics import PluginMetric
+    latest = (await db.execute(select(PluginMetricLatest).where(
+        PluginMetricLatest.source_module == source_module,
+        PluginMetricLatest.resource_name == resource,
+        PluginMetricLatest.metric_name == metric))).scalar_one_or_none()
+    if latest is None:
+        return None
+    now = now or datetime.now(timezone.utc)
+    hour = func.date_trunc("hour", PluginMetric.recorded_at)
+    rows = (await db.execute(
+        select(hour, func.avg(PluginMetric.value), func.min(PluginMetric.value),
+               func.max(PluginMetric.value))
+        .where(PluginMetric.source_module == source_module,
+               PluginMetric.resource_name == resource,
+               PluginMetric.metric_name == metric,
+               PluginMetric.recorded_at >= now - timedelta(hours=PREVIEW_HOURS))
+        .group_by(hour).order_by(hour))).all()
+    return {"now": latest.value, "at": latest.recorded_at,
+            "series": [avg for _, avg, _, _ in rows],
+            "low": min((lo for _, _, lo, _ in rows), default=None),
+            "high": max((hi for _, _, _, hi in rows), default=None)}
+
+
 async def metrics_for(db, source_module: str) -> list[str]:
     """The metric names a rule on this source can watch: the static catalog, or
     for dynamic sources (snmp) the names recorded so far."""