feat(prometheus): fold state sets on the target page; forget series an endpoint drops (#5689)

#5689 "Prometheus target page: fold state sets, show epoch times, size
bytes", step 1 of milestone 508 "Prometheus widget templates".

- A state set (name ending in _state, a state label, every series 0 or 1;
  or the OpenMetrics form, the label named after the metric) reads as one
  row per thing it describes, naming the current state, the others on
  hover. fabledarchives_jobs{state} is a count and stays as it is.
- label_pairs / state_label / fold_states in service.py, for the widget
  templates to reuse.
- After a good scrape, series it no longer has lose their newest value
  (forget_gone): Archives' old build_info row read as current beside the
  new one. History stays; a failed scrape forgets nothing; alert rules keep
  their own last-sample time.
- Epoch times and _bytes sizes were already read by shown_value /
  metric_value.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/prometheus/routes.py

@@ -20,8 +20,8 @@ from steward.models.users import UserRole
 
 from .models import PrometheusTarget
 from .scraper import ScrapeError, scrape
-from .service import (OWN_METRICS, SOURCE, FormError, form_values, map_families, parse_form,
-                      patterns, target_state)
+from .service import (OWN_METRICS, SOURCE, FormError, fold_states, form_values, map_families,
+                      parse_form, patterns, target_state)
 
 prometheus_bp = Blueprint("prometheus", __name__, template_folder="templates")
 
@@ -256,7 +256,9 @@ async def test_target():
 
 def group_series(target_name: str, latest: list) -> list[dict]:
     """The target's stored series grouped by metric, each with its rows:
-    the labels that tell them apart, and the newest value."""
+    the labels that tell them apart, and the newest value. A state set is
+    folded to one row per thing it describes, naming the current state
+    (`states` set on the group)."""
     groups: dict[str, list] = {}
     for row in latest:
         if row.resource_name == target_name and row.metric_name in OWN_METRICS:
@@ -264,9 +266,18 @@ def group_series(target_name: str, latest: list) -> list[dict]:
         labels = row.resource_name[len(target_name) + 1:] if row.resource_name != target_name else ""
         groups.setdefault(row.metric_name, []).append({
             "labels": labels, "resource": row.resource_name, "at": row.recorded_at,
-            **shown_value(row.metric_name, row.value)})
-    return [{"metric": m, "rows": sorted(rows, key=lambda r: r["labels"])}
-            for m, rows in sorted(groups.items())]
+            "value": row.value, **shown_value(row.metric_name, row.value)})
+    out = []
+    for metric, rows in sorted(groups.items()):
+        folded = fold_states(metric, rows)
+        if folded is not None:
+            out.append({"metric": metric, "states": True, "rows": [
+                {"labels": f["labels"], "resource": f["resource"], "moment": None,
+                 "text": f["state"] or "none", "states": f["states"]} for f in folded]})
+        else:
+            out.append({"metric": metric, "states": False,
+                        "rows": sorted(rows, key=lambda r: r["labels"])})
+    return out
 
 
 @prometheus_bp.get("/targets/<target_id>")

plugins/prometheus/scheduler.py

@@ -83,18 +83,46 @@ async def scrape_due(app: "Quart", scrape=None) -> None:
     await record(app, targets, results, datetime.now(timezone.utc))
 
 
+async def forget_gone(session, target: str, kept: set[tuple[str, str]]) -> None:
+    """Drop the newest value of each series of `target` that its last scrape
+    didn't keep: the endpoint stopped reporting it (an old build_info after an
+    upgrade, a deleted job), or a filter or the cap now leaves it out. Pages,
+    widgets and query_metrics read the newest values, so they show what the
+    endpoint says now. History stays for charts and ages out with retention.
+    Alert rules keep their own last-sample time, so one on a gone series
+    still goes no_data."""
+    from sqlalchemy import delete, tuple_
+
+    from steward.models.metrics import PluginMetricLatest
+
+    from .service import SOURCE
+
+    L = PluginMetricLatest
+    stored = (await session.execute(select(L.resource_name, L.metric_name).where(
+        L.source_module == SOURCE,
+        (L.resource_name == target) | L.resource_name.startswith(target + ":", autoescape=True)))).all()
+    gone = [(r, m) for r, m in stored if (r, m) not in kept]
+    if gone:
+        await session.execute(delete(L).where(
+            L.source_module == SOURCE, tuple_(L.resource_name, L.metric_name).in_(gone)))
+
+
 async def record(app: "Quart", targets, results, now: datetime) -> None:
     """Each target's outcome on its row, and its kept series (counters as
-    rates) plus its own is_up / scrape_ms / series counts as metrics."""
+    rates) plus its own is_up / scrape_ms / series counts as metrics. After a
+    good scrape, the series it no longer has are forgotten (forget_gone); a
+    failed scrape says nothing about them, so they stay."""
     from steward.core.alerts import MetricBatch
 
     from .models import PrometheusTarget
     from .scraper import ScrapeError
-    from .service import (IS_UP, SCRAPE_MS, SERIES_CAPPED, SERIES_KEPT, SOURCE, map_families)
+    from .service import (IS_UP, OWN_METRICS, SCRAPE_MS, SERIES_CAPPED, SERIES_KEPT, SOURCE,
+                          map_families)
 
     async with app.db_sessionmaker() as session:
         async with session.begin():
             batch = MetricBatch()
+            answered: list[tuple[str, set[tuple[str, str]]]] = []
             for target, (result, ms) in zip(targets, results):
                 row = await session.get(PrometheusTarget, target.id)
                 if row is None:          # deleted while it was being scraped
@@ -114,6 +142,9 @@ async def record(app: "Quart", targets, results, now: datetime) -> None:
                 row.last_kept = len(mapped.series)
                 row.last_dropped = mapped.dropped
                 row.last_capped = mapped.capped
+                # A counter's first scrape gives no rate yet; it is still kept.
+                answered.append((row.name, {(x.resource, x.metric) for x in mapped.series}
+                                 | {(row.name, m) for m in OWN_METRICS}))
                 for series in mapped.series:
                     value = series.value
                     if series.counter:
@@ -128,3 +159,5 @@ async def record(app: "Quart", targets, results, now: datetime) -> None:
                     batch.add(source_module=SOURCE, resource_name=row.name,
                               metric_name=metric_name, value=value, recorded_at=now)
             await batch.record(session)
+            for name, kept in answered:
+                await forget_gone(session, name, kept)

plugins/prometheus/service.py

@@ -60,6 +60,67 @@ def rate_name(sample_name: str) -> str:
     return base + "_per_sec"
 
 
+def label_pairs(labels: str) -> list[tuple[str, str]]:
+    """resource_for's labels read back: "target=S,state=late" →
+    [("target", "S"), ("state", "late")]. A piece without "=" was part of the
+    value before it, a value that held a comma."""
+    pairs: list[tuple[str, str]] = []
+    for piece in labels.split(",") if labels else []:
+        key, eq, value = piece.partition("=")
+        if eq and key:
+            pairs.append((key, value))
+        elif pairs:
+            pairs[-1] = (pairs[-1][0], pairs[-1][1] + "," + piece)
+    return pairs
+
+
+def state_label(metric: str, pairs: list[tuple[str, str]]) -> str | None:
+    """Which label names a state set's state: `state`, or the metric's own
+    name as OpenMetrics writes a stateset (foo{foo="a"} 1)."""
+    keys = {k for k, _ in pairs}
+    for key in ("state", metric):
+        if key in keys:
+            return key
+    return None
+
+
+def fold_states(metric: str, rows: list[dict]) -> list[dict] | None:
+    """A state set's series folded to one row per thing it describes.
+
+    A state set is one series per possible state, the current one 1 and the
+    rest 0: Archives' fabledarchives_backup_target_state{target, state}. Each
+    row in `rows` has `labels` (resource_for's form), `value` and `resource`.
+    A metric is folded when its name ends in _state (or its state label is
+    its own name) and every series carries the state label and reads 0 or 1;
+    anything else is None, and shown series by series. The name matters: a
+    count split by a `state` label (fabledarchives_jobs{state="waiting"})
+    often reads 0 or 1 too, and is not a state set.
+
+    Each folded row: `labels` without the state, `state` (the one set to 1;
+    several are joined, none is ""), `states` (every state, in order) and
+    `resource` (the current state's series, else the first)."""
+    if not rows:
+        return None
+    sets: dict[str, list[tuple[str, dict]]] = {}
+    for row in rows:
+        pairs = label_pairs(row["labels"])
+        key = state_label(metric, pairs)
+        if key is None or row["value"] not in (0, 1):
+            return None
+        if key == "state" and not metric.endswith("_state"):
+            return None
+        rest = ",".join(f"{k}={v}" for k, v in pairs if k != key)
+        sets.setdefault(rest, []).append((dict(pairs)[key], row))
+    folded = []
+    for rest, members in sorted(sets.items()):
+        members.sort(key=lambda m: m[0])
+        current = [(s, r) for s, r in members if r["value"] == 1]
+        folded.append({"labels": rest, "state": ", ".join(s for s, _ in current),
+                       "states": [s for s, _ in members],
+                       "resource": (current or members)[0][1]["resource"]})
+    return folded
+
+
 @dataclass
 class Series:
     resource: str

plugins/prometheus/templates/prometheus/target.html

@@ -1,7 +1,8 @@
 {# plugins/prometheus/templates/prometheus/target.html — one scrape target
    (milestone 506): what its newest scrape found, then every series it
    stored, grouped by metric, each with its labels and newest value and a
-   link to its chart. #}
+   link to its chart. A state set reads as one row per thing it describes,
+   naming the current state (#5689). #}
 {% extends "base.html" %}
 {% from "_macros.html" import crumbs, page_header, status_badge, stat_tile %}
 {% block title %}Prometheus — {{ target.name }} — Steward{% endblock %}
@@ -54,10 +55,12 @@
     {% for g in groups %}
       {% for r in g.rows %}
       <tr>
-        {% if loop.first %}<td class="mono" rowspan="{{ g.rows|length }}" style="vertical-align:top;overflow-wrap:anywhere;">{{ g.metric }}</td>{% endif %}
+        {% if loop.first %}<td class="mono" rowspan="{{ g.rows|length }}" style="vertical-align:top;overflow-wrap:anywhere;">{{ g.metric }}
+          {%- if g.states %}<div class="cell-sub">a state set: the current state is shown</div>{% endif %}</td>{% endif %}
         <td class="mono" style="overflow-wrap:anywhere;font-size:0.82rem;">{{ r.labels or "—" }}</td>
         <td style="text-align:right;white-space:nowrap;">
-          <a href="/plugins/prometheus/targets/{{ target.id }}/series?resource={{ r.resource|urlencode }}&metric={{ g.metric|urlencode }}">{% if r.moment %}{{ r.moment|ago }}{% else %}{{ r.text }}{% endif %}</a>
+          <a href="/plugins/prometheus/targets/{{ target.id }}/series?resource={{ r.resource|urlencode }}&metric={{ g.metric|urlencode }}"
+             {%- if r.states %} title="One of: {{ r.states|join(', ') }}"{% endif %}>{% if r.moment %}{{ r.moment|ago }}{% else %}{{ r.text }}{% endif %}</a>
         </td>
       </tr>
       {% endfor %}