feat(prometheus): fold state sets on the target page; forget series an endpoint drops (#5689)

#5689 "Prometheus target page: fold state sets, show epoch times, size
bytes", step 1 of milestone 508 "Prometheus widget templates".

- A state set (name ending in _state, a state label, every series 0 or 1;
  or the OpenMetrics form, the label named after the metric) reads as one
  row per thing it describes, naming the current state, the others on
  hover. fabledarchives_jobs{state} is a count and stays as it is.
- label_pairs / state_label / fold_states in service.py, for the widget
  templates to reuse.
- After a good scrape, series it no longer has lose their newest value
  (forget_gone): Archives' old build_info row read as current beside the
  new one. History stays; a failed scrape forgets nothing; alert rules keep
  their own last-sample time.
- Epoch times and _bytes sizes were already read by shown_value /
  metric_value.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/prometheus/scheduler.py

@@ -83,18 +83,46 @@ async def scrape_due(app: "Quart", scrape=None) -> None:
     await record(app, targets, results, datetime.now(timezone.utc))
 
 
+async def forget_gone(session, target: str, kept: set[tuple[str, str]]) -> None:
+    """Drop the newest value of each series of `target` that its last scrape
+    didn't keep: the endpoint stopped reporting it (an old build_info after an
+    upgrade, a deleted job), or a filter or the cap now leaves it out. Pages,
+    widgets and query_metrics read the newest values, so they show what the
+    endpoint says now. History stays for charts and ages out with retention.
+    Alert rules keep their own last-sample time, so one on a gone series
+    still goes no_data."""
+    from sqlalchemy import delete, tuple_
+
+    from steward.models.metrics import PluginMetricLatest
+
+    from .service import SOURCE
+
+    L = PluginMetricLatest
+    stored = (await session.execute(select(L.resource_name, L.metric_name).where(
+        L.source_module == SOURCE,
+        (L.resource_name == target) | L.resource_name.startswith(target + ":", autoescape=True)))).all()
+    gone = [(r, m) for r, m in stored if (r, m) not in kept]
+    if gone:
+        await session.execute(delete(L).where(
+            L.source_module == SOURCE, tuple_(L.resource_name, L.metric_name).in_(gone)))
+
+
 async def record(app: "Quart", targets, results, now: datetime) -> None:
     """Each target's outcome on its row, and its kept series (counters as
-    rates) plus its own is_up / scrape_ms / series counts as metrics."""
+    rates) plus its own is_up / scrape_ms / series counts as metrics. After a
+    good scrape, the series it no longer has are forgotten (forget_gone); a
+    failed scrape says nothing about them, so they stay."""
     from steward.core.alerts import MetricBatch
 
     from .models import PrometheusTarget
     from .scraper import ScrapeError
-    from .service import (IS_UP, SCRAPE_MS, SERIES_CAPPED, SERIES_KEPT, SOURCE, map_families)
+    from .service import (IS_UP, OWN_METRICS, SCRAPE_MS, SERIES_CAPPED, SERIES_KEPT, SOURCE,
+                          map_families)
 
     async with app.db_sessionmaker() as session:
         async with session.begin():
             batch = MetricBatch()
+            answered: list[tuple[str, set[tuple[str, str]]]] = []
             for target, (result, ms) in zip(targets, results):
                 row = await session.get(PrometheusTarget, target.id)
                 if row is None:          # deleted while it was being scraped
@@ -114,6 +142,9 @@ async def record(app: "Quart", targets, results, now: datetime) -> None:
                 row.last_kept = len(mapped.series)
                 row.last_dropped = mapped.dropped
                 row.last_capped = mapped.capped
+                # A counter's first scrape gives no rate yet; it is still kept.
+                answered.append((row.name, {(x.resource, x.metric) for x in mapped.series}
+                                 | {(row.name, m) for m in OWN_METRICS}))
                 for series in mapped.series:
                     value = series.value
                     if series.counter:
@@ -128,3 +159,5 @@ async def record(app: "Quart", targets, results, now: datetime) -> None:
                     batch.add(source_module=SOURCE, resource_name=row.name,
                               metric_name=metric_name, value=value, recorded_at=now)
             await batch.record(session)
+            for name, kept in answered:
+                await forget_gone(session, name, kept)