feat(alerts): pick the host, then its part, and see what the metric reads (#5475)

#5475 "Rule form: pick the host, then its part; metrics that part reports;
a link to groups; a preview of the metric", from the operator's review of
the New rule page: hosts and their parts shared one long picker.

- Host agent rules pick a Host, then a Part: "The whole host", then its
  mounts, CPU cores, disks, interfaces and sensors under a heading each,
  cores in number order. A part that reports nothing a rule can watch
  (an interface's traffic) isn't offered.
- The Metric list holds what the picked resource reports, in catalog order.
- Under it, a preview: the reading now, and the last 24 h low to high with
  an hourly sparkline drawn from zero (UI ruling 8), read from one series.
- The group picker links to Ansible > Inventory to make a group, in a new
  tab, and re-reads the groups when the tab comes back into view.
- Picking a host or part swaps the pickers alone (/alerts/api/rule_target);
  picking a metric swaps the preview (/alerts/api/metric_preview).
- A rule's own resource and metric stay on offer after they go quiet.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/routes.py

@@ -256,21 +256,65 @@ HISTORY_SUMMARY_ROWS = 2000
 @alerts_bp.get("/api/rule_fields")
 @require_role(UserRole.viewer)
 async def rule_fields_api():
+    """Everything the rule watches, for a newly picked source."""
     source = request.args.get("source_module", "")
-    current_resource = request.args.get("current_resource", "")
-    current_metric = request.args.get("current_metric", "")
     async with current_app.db_sessionmaker() as db:
-        resources = await alert_svc.resources_for(db, source) if source else []
-        metrics = await alert_svc.metrics_for(db, source)
-        cover = await _cover_choices(db, source)
-    return await render_template(
-        "alerts/_rule_fields.html",
-        resources=resources,
-        metrics=metrics,
-        current_resource=current_resource,
-        current_metric=current_metric,
-        **cover,
-    )
+        ctx = {**await _cover_choices(db, source), **await _target(db, source)}
+    return await render_template("alerts/_rule_fields.html", **ctx)
+
+
+@alerts_bp.get("/api/rule_target")
+@require_role(UserRole.viewer)
+async def rule_target_api():
+    """The host, part and metric pickers and the preview, after a host or a
+    part is picked (#5475)."""
+    async with current_app.db_sessionmaker() as db:
+        ctx = await _target_from_args(db)
+    return await render_template("alerts/_rule_target.html", **ctx)
+
+
+@alerts_bp.get("/api/metric_preview")
+@require_role(UserRole.viewer)
+async def metric_preview_api():
+    """What the picked metric reads now and over the last day (#5475)."""
+    async with current_app.db_sessionmaker() as db:
+        ctx = await _target_from_args(db)
+    return await render_template("alerts/_metric_preview.html", **ctx)
+
+
+@alerts_bp.get("/api/cover_groups")
+@require_role(UserRole.viewer)
+async def cover_groups_api():
+    """The inventory groups, re-read when the reader comes back from making one."""
+    from steward.hosts.service import group_names
+    async with current_app.db_sessionmaker() as db:
+        groups = await group_names(db)
+    return await render_template("alerts/_cover_groups.html", groups=groups,
+                                 current_group=request.args.get("cover_group", ""),
+                                 current_covers=request.args.get("covers", "resource"))
+
+
+async def _target(db, source: str, *, host: str = "", resource: str = "", metric: str = "",
+                  covers: str = "resource") -> dict:
+    """The rule form's target pickers and the chosen metric's preview (#5475)."""
+    from steward.core.status import sparkline_svg
+    target = await alert_svc.rule_target(db, source, host=host, resource=resource, metric=metric)
+    preview = None
+    if target["resource"] and target["metric"]:
+        preview = await alert_svc.metric_preview(db, source, target["resource"], target["metric"])
+        if preview:
+            # From zero to the series' own peak (UI & Design ruling 8), so a
+            # disk at 61-62% reads flat rather than as a cliff.
+            preview["spark"] = sparkline_svg(preview["series"], width=120, height=20,
+                                             stroke="currentColor", floor=1.0)
+    return {**target, "preview": preview, "current_covers": covers}
+
+
+async def _target_from_args(db) -> dict:
+    a = request.args
+    return await _target(db, a.get("source_module", ""), host=a.get("pick_host", ""),
+                         resource=a.get("resource_name", ""), metric=a.get("metric_name", ""),
+                         covers=a.get("covers", "resource"))
 
 
 async def _cover_choices(db, source: str, rule: AlertRule | None = None) -> dict:
@@ -290,16 +334,13 @@ async def _cover_choices(db, source: str, rule: AlertRule | None = None) -> dict
 async def new_rule():
     first_source = alert_svc.source_modules()[0]
     async with current_app.db_sessionmaker() as db:
-        resources = await alert_svc.resources_for(db, first_source)
-        cover = await _cover_choices(db, first_source)
+        ctx = {**await _cover_choices(db, first_source), **await _target(db, first_source)}
     return await render_template(
         "alerts/rules_form.html",
-        rule=None, **cover,
+        rule=None, **ctx,
         operators=_OPERATORS,
         source_modules=alert_svc.source_modules(),
         initial_source=first_source,
-        resources=resources,
-        metrics=alert_svc.metric_catalog().get(first_source, []),
         catalog=alert_svc.metric_catalog(),
         ansible_sources=_ansible_source_names(),
         notify=await _notify(),
@@ -317,8 +358,9 @@ async def edit_rule(rule_id: str):
         if rule.parent_id:
             # A host's rule under a covering rule is changed through it.
             return redirect(f"/alerts/rules/{rule.parent_id}/edit")
-        resources = await alert_svc.resources_for(db, rule.source_module)
-        cover = await _cover_choices(db, rule.source_module, rule)
+        ctx = {**await _cover_choices(db, rule.source_module, rule),
+               **await _target(db, rule.source_module, resource=rule.resource_name,
+                               metric=rule.metric_name, covers=rule.covers)}
         hosts = sorted((c.resource_name for c in (await db.execute(
             select(AlertRule).where(AlertRule.parent_id == rule.id))).scalars()), key=str.lower)
         history_count = (await db.execute(
@@ -327,12 +369,10 @@ async def edit_rule(rule_id: str):
     return await render_template(
         "alerts/rules_form.html",
         rule=rule, history_count=history_count, fired=fired, notify=await _notify(),
-        covered=hosts, **cover,
+        covered=hosts, **ctx,
         operators=_OPERATORS,
         source_modules=alert_svc.source_modules(),
         initial_source=rule.source_module,
-        resources=resources,
-        metrics=alert_svc.metric_catalog().get(rule.source_module, []),
         catalog=alert_svc.metric_catalog(),
         ansible_sources=_ansible_source_names(),
     )

steward/alerts/service.py

@@ -358,6 +358,124 @@ async def resources_for(db, source_module: str) -> list[str]:
     return sorted(resources)
 
 
+# A host's parts, by the shape of the name after "<host>:" (#5475): the order
+# and heading each kind takes in the rule form's Part picker.
+PART_KINDS = (("mount", "Mounts"), ("core", "CPU cores"), ("disk", "Disks"),
+              ("net", "Interfaces"), ("temp", "Temperatures"), ("other", "Other"))
+
+
+def part_kind(part: str) -> tuple[str, str]:
+    """(kind, label) for a host part: "/var" is a mount, "core3" a core,
+    "diskio:sda" a disk, "net:eth0" an interface, "temp:Core 0" a sensor."""
+    if part.startswith("/"):
+        return "mount", part
+    if part.startswith("core") and part[4:].isdigit():
+        return "core", f"core {part[4:]}"
+    for prefix, kind in (("diskio:", "disk"), ("net:", "net"), ("temp:", "temp")):
+        if part.startswith(prefix):
+            return kind, part[len(prefix):]
+    return "other", part
+
+
+def _part_sort(part: str):
+    kind, label = part_kind(part)
+    return (int(part[4:]), "") if kind == "core" else (0, label.lower())
+
+
+def alertable(source_module: str, reported: set[str]) -> list[str]:
+    """The metrics a rule on a resource can watch, from what it reports: the
+    source's catalog, in its order, narrowed to those; every reported name
+    for a source with no catalog (snmp)."""
+    catalog = metric_catalog().get(source_module, [])
+    return [m for m in catalog if m in reported] if catalog else sorted(reported)
+
+
+async def rule_target(db, source_module: str, *, host: str = "", resource: str = "",
+                      metric: str = "") -> dict:
+    """What the rule form's target pickers offer (#5475), from the latest-value
+    table: the resources that report something a rule can watch, and for the
+    chosen one the metrics it reports. For a host source the resources split
+    into a Host picker and a Part picker ("the whole host", then the host's
+    mounts, cores and so on). A resource or metric already chosen stays on
+    offer even when it has gone quiet, so editing a rule never drops them.
+
+    Returns host_source, hosts, host, parts [(heading, [(value, label)])],
+    resources, resource, metrics, metric."""
+    catalog = metric_catalog().get(source_module, [])
+    seen: dict[str, set[str]] = {}
+    for res, name in (await db.execute(
+            select(PluginMetricLatest.resource_name, PluginMetricLatest.metric_name)
+            .where(PluginMetricLatest.source_module == source_module))).all():
+        seen.setdefault(res, set()).add(name)
+    pickable = {r: ms for r, names in seen.items() if (ms := alertable(source_module, names))}
+    if source_module in MONITOR_TYPE_VALUES:
+        for (name,) in (await db.execute(select(Monitor.name).where(Monitor.type == source_module))).all():
+            pickable.setdefault(name, list(catalog))
+    if resource and resource not in pickable:
+        pickable[resource] = list(catalog) or ([metric] if metric else [])
+
+    out: dict = {"host_source": source_module in HOST_SOURCES, "hosts": [], "host": "",
+                 "parts": [], "resources": [], "resource": resource}
+    if out["host_source"]:
+        hosts = sorted({r.split(":", 1)[0] for r in pickable}, key=str.lower)
+        host = host or resource.split(":", 1)[0]
+        if host not in hosts:
+            host = hosts[0] if hosts else ""
+        mine = [r for r in pickable if r == host or r.startswith(host + ":")]
+        if resource not in mine:
+            resource = host if host in pickable else (sorted(mine)[0] if mine else "")
+        by_kind: dict[str, list] = {}
+        for r in sorted((r for r in mine if r != host), key=lambda r: _part_sort(r[len(host) + 1:])):
+            kind, label = part_kind(r[len(host) + 1:])
+            by_kind.setdefault(kind, []).append((r, label))
+        parts = [("", [(host, "The whole host")])] if host in pickable else []
+        parts += [(heading, by_kind[k]) for k, heading in PART_KINDS if k in by_kind]
+        out.update(hosts=hosts, host=host, parts=parts, resource=resource)
+    else:
+        out["resources"] = sorted(pickable, key=str.lower)
+    metrics = list(pickable.get(out["resource"]) or catalog)
+    if metric and metric not in metrics and (metric in catalog or not catalog):
+        metrics.append(metric)
+    out.update(metrics=metrics, metric=metric if metric in metrics else "")
+    return out
+
+
+# How far back the rule form's metric preview looks: a day shows the daily
+# swing (backups, batch jobs) that a threshold has to sit above.
+PREVIEW_HOURS = 24
+
+
+async def metric_preview(db, source_module: str, resource: str, metric: str,
+                         now: datetime | None = None) -> dict | None:
+    """What the chosen metric reads (#5475), so a threshold is set against real
+    numbers: the latest reading and when, and the last PREVIEW_HOURS as hourly
+    averages with the lowest and highest. Reads one series: the latest-value
+    row, then a recorded_at range on plugin_metrics' series index. None when
+    it has no reading."""
+    from datetime import timedelta
+    from steward.models.metrics import PluginMetric
+    latest = (await db.execute(select(PluginMetricLatest).where(
+        PluginMetricLatest.source_module == source_module,
+        PluginMetricLatest.resource_name == resource,
+        PluginMetricLatest.metric_name == metric))).scalar_one_or_none()
+    if latest is None:
+        return None
+    now = now or datetime.now(timezone.utc)
+    hour = func.date_trunc("hour", PluginMetric.recorded_at)
+    rows = (await db.execute(
+        select(hour, func.avg(PluginMetric.value), func.min(PluginMetric.value),
+               func.max(PluginMetric.value))
+        .where(PluginMetric.source_module == source_module,
+               PluginMetric.resource_name == resource,
+               PluginMetric.metric_name == metric,
+               PluginMetric.recorded_at >= now - timedelta(hours=PREVIEW_HOURS))
+        .group_by(hour).order_by(hour))).all()
+    return {"now": latest.value, "at": latest.recorded_at,
+            "series": [avg for _, avg, _, _ in rows],
+            "low": min((lo for _, _, lo, _ in rows), default=None),
+            "high": max((hi for _, _, _, hi in rows), default=None)}
+
+
 async def metrics_for(db, source_module: str) -> list[str]:
     """The metric names a rule on this source can watch: the static catalog, or
     for dynamic sources (snmp) the names recorded so far."""

steward/static/steward.css

@@ -219,6 +219,11 @@ strong, b, th { font-weight: 500; }
 /* A covering alert rule's per-host rules (#5464), indented under it. */
 .table tr.row-child > td { border-top-style: dashed; }
 .table tr.row-child .cell-name { padding-left: 2rem; }
+/* The rule form's metric preview (#5475): now, then the last day beside its sparkline. */
+.metric-preview { display: flex; flex-wrap: wrap; align-items: center; gap: 0.25rem 1rem;
+  margin-top: 0.4rem; font-size: 0.85rem; }
+.metric-preview-day { display: inline-flex; align-items: center; gap: 0.4rem; }
+.metric-preview-spark { color: var(--text-muted); line-height: 0; }
 /* A row that opens as a whole: its one .row-target link stretches over the
    row, and the actions cell sits above it so its buttons keep their own
    clicks. One real link per row, so keyboard and screen readers meet it once. */

steward/templates/alerts/_cover_groups.html

@@ -0,0 +1,9 @@
+{# alerts/_cover_groups.html — the inventory groups a rule can cover (#5464),
+   re-read when the reader returns from making one (#5475). #}
+<select id="rule-group" name="cover_group" {% if current_covers|default("resource") != "group" %}disabled{% endif %}>
+  {% for g in groups %}
+  <option value="{{ g }}" {% if g == current_group|default("") %}selected{% endif %}>{{ g }}</option>
+  {% else %}
+  <option value="" disabled selected>No inventory groups yet</option>
+  {% endfor %}
+</select>

steward/templates/alerts/_metric_preview.html

@@ -0,0 +1,18 @@
+{# alerts/_metric_preview.html — what the chosen metric reads (#5475), so a
+   threshold is set against real numbers: now, and the last day's hourly
+   averages with the lowest and highest reading. A rule covering hosts
+   previews on the host picked above it, or none. #}
+{% if preview %}
+<div class="metric-preview">
+  <span>Now <strong>{{ metric_value(metric, preview.now) }}</strong>
+    <span class="tone-dim" title="{{ preview.at|when }}">{{ preview.at|ago }}</span></span>
+  {% if preview.series|length > 1 %}
+  <span class="metric-preview-day">
+    <span class="tone-dim">last 24 h</span> {{ metric_value(metric, preview.low) }} – {{ metric_value(metric, preview.high) }}
+    <span class="metric-preview-spark" aria-hidden="true">{{ preview.spark|safe }}</span>
+  </span>
+  {% endif %}
+</div>
+{% elif metric and resource %}
+<p class="field-hint">No reading of this yet from {{ resource }}.</p>
+{% endif %}

steward/templates/alerts/_rule_fields.html

@@ -1,8 +1,10 @@
-{# alerts/_rule_fields.html — the resource and metric pickers for one source;
-   the rule form includes it and swaps it in again when the source changes.
-   For a source whose resources are hosts, the rule can instead cover every
-   host, or every host in an inventory group (#5464): Steward then keeps one
-   rule per host under it. What a rule covers is fixed once it exists. #}
+{# alerts/_rule_fields.html — what a rule watches, for one source; the rule
+   form includes it and swaps it in again when the source changes. For a
+   source whose resources are hosts, the rule can instead cover every host,
+   or every host in an inventory group (#5464): Steward then keeps one rule
+   per host under it. What a rule covers is fixed once it exists. The host,
+   part and metric pickers are alerts/_rule_target.html, swapped on their
+   own as a host or part is picked (#5475). #}
 {% set covers = current_covers|default("resource") %}
 {% set one = covers == "resource" %}
 {% if host_source|default(false) %}
@@ -16,7 +18,7 @@
   <select id="rule-covers" name="covers" data-covers>
     <option value="resource" {% if one %}selected{% endif %}>One host, or one part of a host</option>
     <option value="all_hosts" {% if covers == "all_hosts" %}selected{% endif %}>Every host</option>
-    <option value="group" {% if covers == "group" %}selected{% endif %} {% if not groups %}disabled{% endif %}>Every host in a group{% if not groups %} (no inventory groups yet){% endif %}</option>
+    <option value="group" {% if covers == "group" %}selected{% endif %}>Every host in a group</option>
   </select>
   {% endif %}
   <p class="field-hint">Covering hosts keeps one alert per host under this rule, each named
@@ -25,43 +27,17 @@
 </div>
 <div class="form-group" data-cover-group {% if covers != "group" %}hidden{% endif %}>
   <label for="rule-group">Group</label>
-  <select id="rule-group" name="cover_group" {% if covers != "group" %}disabled{% endif %}>
-    {% for g in groups %}
-    <option value="{{ g }}" {% if g == current_group|default("") %}selected{% endif %}>{{ g }}</option>
-    {% endfor %}
-  </select>
-  <p class="field-hint">A host is in a group through its linked Ansible target (Ansible → Inventory).</p>
+  {# Re-read when the reader comes back to this tab, so a group just made in
+     Inventory is here without losing what was typed. #}
+  <div id="rule-group-picker" hx-get="/alerts/api/cover_groups" hx-trigger="visibilitychange[document.visibilityState === 'visible'] from:document"
+       hx-include="[name='cover_group'],[name='covers']" hx-swap="innerHTML">
+    {% include "alerts/_cover_groups.html" %}
+  </div>
+  <p class="field-hint">A host is in a group through its linked Ansible target.
+    <a href="/ansible/inventory/?add=group" target="_blank" rel="noopener">Make a group, or change who is in one, in Ansible → Inventory</a>;
+    it shows here when you come back.</p>
 </div>
 {% endif %}
-<div class="form-group" data-cover-resource {% if not one %}hidden{% endif %}>
-  <label>Resource</label>
-  {% if resources %}
-  <select name="resource_name" required {% if not one %}disabled{% endif %}>
-    {% if not current_resource %}<option value="" disabled selected>— pick a resource —</option>{% endif %}
-    {% for r in resources %}
-    <option value="{{ r }}" {% if r == current_resource %}selected{% endif %}>{{ r }}</option>
-    {% endfor %}
-  </select>
-  {% else %}
-  <select name="resource_name" required disabled data-none="1">
-    <option value="">Nothing has reported from this source yet</option>
-  </select>
-  <p class="field-hint">
-    The list fills from readings this source has recorded. Come back once it has reported.
-  </p>
-  {% endif %}
-</div>
-
-<div class="form-group">
-  <label>Metric</label>
-  <select name="metric_name" required data-metric-picker>
-    {% if not current_metric %}<option value="" disabled selected>— pick a metric —</option>{% endif %}
-    {% for m in metrics %}
-    <option value="{{ m }}" data-unit="{{ metric_unit(m) }}" data-about="{{ metric_about(m) }}" {% if m in ("is_up", "is_healthy") %}data-binary{% endif %}
-            {% if m == current_metric %}selected{% endif %}>{{ metric_label(m) }}</option>
-    {% endfor %}
-  </select>
-  {% if current_metric and metric_about(current_metric) %}
-  <p class="field-hint" data-metric-about>{{ metric_about(current_metric) }}</p>
-  {% else %}<p class="field-hint" data-metric-about hidden></p>{% endif %}
+<div id="rule-target">
+{% include "alerts/_rule_target.html" %}
 </div>

steward/templates/alerts/_rule_target.html

@@ -0,0 +1,72 @@
+{# alerts/_rule_target.html — what one rule watches (#5475). For a host
+   source: the Host, then its Part ("the whole host", then mounts, cores and
+   so on), so a host's parts never crowd the list of hosts. Any other source:
+   one Resource picker. Then the metrics that resource reports, and a preview
+   of the chosen one. Picking a host or part swaps this block alone
+   (/alerts/api/rule_target); picking a metric swaps the preview. #}
+{% set one = current_covers|default("resource") == "resource" %}
+<div data-cover-resource {% if not one %}hidden{% endif %}>
+  {% if host_source %}
+  <div class="form-pair">
+    <div class="form-group">
+      <label for="rule-host">Host</label>
+      {% if hosts %}
+      <select id="rule-host" name="pick_host" {% if not one %}disabled{% endif %}
+              hx-get="/alerts/api/rule_target" hx-trigger="change" hx-target="#rule-target"
+              hx-include="closest form" hx-params="source_module,pick_host,metric_name,covers">
+        {% for h in hosts %}<option value="{{ h }}" {% if h == host %}selected{% endif %}>{{ h }}</option>{% endfor %}
+      </select>
+      {% else %}
+      <select id="rule-host" disabled data-none="1"><option>No host has reported yet</option></select>
+      {% endif %}
+    </div>
+    <div class="form-group">
+      <label for="rule-part">Part</label>
+      <select id="rule-part" name="resource_name" required {% if not one or not parts %}disabled{% endif %} {% if not parts %}data-none="1"{% endif %}
+              hx-get="/alerts/api/rule_target" hx-trigger="change" hx-target="#rule-target"
+              hx-include="closest form" hx-params="source_module,pick_host,resource_name,metric_name,covers">
+        {% for heading, items in parts %}
+        {% if heading %}<optgroup label="{{ heading }}">{% endif %}
+          {% for value, label in items %}<option value="{{ value }}" {% if value == resource %}selected{% endif %}>{{ label }}</option>{% endfor %}
+        {% if heading %}</optgroup>{% endif %}
+        {% endfor %}
+      </select>
+    </div>
+  </div>
+  {% if not hosts %}<p class="field-hint" style="margin-top:-0.5rem;">The list fills once an agent reports.</p>{% endif %}
+  {% else %}
+  <div class="form-group">
+    <label for="rule-resource">Resource</label>
+    {% if resources %}
+    <select id="rule-resource" name="resource_name" required {% if not one %}disabled{% endif %}
+            hx-get="/alerts/api/rule_target" hx-trigger="change" hx-target="#rule-target"
+            hx-include="closest form" hx-params="source_module,resource_name,metric_name,covers">
+      {% if not resource %}<option value="" disabled selected>— pick a resource —</option>{% endif %}
+      {% for r in resources %}<option value="{{ r }}" {% if r == resource %}selected{% endif %}>{{ r }}</option>{% endfor %}
+    </select>
+    {% else %}
+    <select id="rule-resource" name="resource_name" required disabled data-none="1">
+      <option value="">Nothing has reported from this source yet</option>
+    </select>
+    <p class="field-hint">The list fills from readings this source has recorded. Come back once it has reported.</p>
+    {% endif %}
+  </div>
+  {% endif %}
+</div>
+
+<div class="form-group">
+  <label for="rule-metric">Metric</label>
+  <select id="rule-metric" name="metric_name" required data-metric-picker
+          hx-get="/alerts/api/metric_preview" hx-trigger="change" hx-target="#metric-preview"
+          hx-include="closest form" hx-params="source_module,resource_name,pick_host,metric_name,covers">
+    {% if not metric %}<option value="" disabled selected>— pick a metric —</option>{% endif %}
+    {% for m in metrics %}
+    <option value="{{ m }}" data-unit="{{ metric_unit(m) }}" data-about="{{ metric_about(m) }}" {% if m in ("is_up", "is_healthy") %}data-binary{% endif %}
+            {% if m == metric %}selected{% endif %}>{{ metric_label(m) }}</option>
+    {% endfor %}
+  </select>
+  {% if metric and metric_about(metric) %}
+  <p class="field-hint" data-metric-about>{{ metric_about(metric) }}</p>
+  {% else %}<p class="field-hint" data-metric-about hidden></p>{% endif %}
+  <div id="metric-preview" aria-live="polite">{% include "alerts/_metric_preview.html" %}</div>
+</div>

steward/templates/alerts/rules_form.html

@@ -56,9 +56,7 @@
 
       {# ── Dynamic resource + metric ────────────────────────────────────────── #}
       <div id="rule-fields">
-        {% with current_resource = rule.resource_name if rule else "", current_metric = rule.metric_name if rule else "" %}
         {% include "alerts/_rule_fields.html" %}
-        {% endwith %}
       </div>
       {# ── Metric reference ────────────────────────────────────────────────── #}
       <details style="margin-bottom:1rem;">