feat(prometheus): scrape any /metrics endpoint into Steward's metrics, with pages (#5668, #5669)

A bundled prometheus plugin, on by default like the other protocol plugins.
Targets are rows edited on its page: address, bearer token (encrypted),
interval, keep/leave-out name patterns and a series cap. Steward does the
filtering; the endpoint serves everything (Archives' ruling).

- A 5 s tick scrapes due targets at once, each within a 30 s deadline, and
  writes the outcome to its row and its series through record_metrics, so
  alert rules can watch any of them.
- Gauges are stored as they are; counters (and a histogram's or summary's
  _sum/_count) as <name>_per_sec rates from the second scrape. Buckets and
  quantiles are skipped. A labelled series is stored under
  "<target>:k=v,…". Each target also records is_up, scrape_ms, series_kept
  and series_capped.
- Pages: targets not answering first, with why; a target's series grouped
  by metric with their newest values; a chart per series; the editor with
  Test, which scrapes once and says what would be stored.
- The counter-to-rate keeper moves to steward/core/counter_rates.py; SNMP's
  RateKeeper now scales its rate on top of it.
- metric_value reads Prometheus's _seconds as a span and _per_sec as a rate.
- The catalog guard treats an empty list as a dynamic source (snmp,
  prometheus), as the rule form does.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_metric_catalog.py

@@ -9,7 +9,7 @@ import steward
 from steward.alerts.service import CORE_METRICS
 
 _REPO = pathlib.Path(steward.__file__).parent.parent
-_PLUGINS = ("traefik", "unifi", "docker", "host_agent", "snmp")
+_PLUGINS = ("traefik", "unifi", "docker", "host_agent", "snmp", "prometheus")
 
 
 def catalog() -> dict[str, set[str]]:
@@ -98,5 +98,7 @@ def test_every_recorded_metric_is_in_the_catalog():
     assert ("traefik", "error_rate_5xx_pct") in pairs and ("docker", "is_healthy") in pairs
     assert ("host_agent", "seconds_since_seen") in pairs
     cat = catalog()
-    missing = sorted(p for p in pairs if p[1] not in cat.get(p[0], set()))
+    # An empty list is a dynamic source (snmp, prometheus): the rule form
+    # offers whatever it has recorded, so any name it writes is offerable.
+    missing = sorted(p for p in pairs if cat.get(p[0]) != set() and p[1] not in cat.get(p[0], set()))
     assert not missing, f"recorded but not in the alert catalog: {missing}"

tests/integration/test_prometheus_scrape.py

@@ -0,0 +1,130 @@
+"""Integration: a Prometheus scrape writes each target's outcome to its row,
+its kept series to the metric tables (a counter as a rate), and its own
+is_up, which an alert rule can watch (#5668). Requires STEWARD_DATABASE_URL."""
+from __future__ import annotations
+
+import asyncio
+import importlib
+import os
+import uuid
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+TEXT = """\
+# TYPE fa_backup_target_state gauge
+fa_backup_target_state{target="Steward",state="late"} 0
+# TYPE fa_requests_total counter
+fa_requests_total {count}
+# TYPE noise gauge
+noise 1
+"""
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def _prom(sub):
+    """A submodule of the prometheus plugin as the app loaded it: it is on by
+    default, so its models are already mapped under the loader's name, and
+    importing plugins.prometheus.models would define the table a second time."""
+    from steward.core.plugin_manager import get_loaded_plugin_modules
+    loaded = get_loaded_plugin_modules().get("prometheus")
+    return importlib.import_module(f"{loaded.__name__ if loaded else 'plugins.prometheus'}.{sub}")
+
+
+def test_a_scrape_records_series_rates_and_a_down_target_fires_its_rule(app):
+    from sqlalchemy import select
+
+    from steward.alerts import service as alert_svc
+    from steward.core import prometheus
+    from steward.core.alerts import record_metrics
+    from steward.core.crypto import encrypt_secret
+    from steward.models.alerts import AlertState, AlertStateEnum
+    from steward.models.metrics import PluginMetric, PluginMetricLatest
+    from steward.models.users import User, UserRole
+
+    scheduler = _prom("scheduler")
+    PrometheusTarget = _prom("models").PrometheusTarget
+    ScrapeError = _prom("scraper").ScrapeError
+
+    tag = uuid.uuid4().hex[:8]
+    up, down = f"arch-{tag}", f"gone-{tag}"
+    counts = iter([1_000, 1_011])
+    tokens = []
+
+    async def fake_scrape(url, token, verify_tls):
+        tokens.append(token)
+        if "gone" in url:
+            raise ScrapeError("Couldn't connect: connection refused.")
+        return prometheus.parse(TEXT.format(count=next(counts)))
+
+    async def latest(name_prefix):
+        async with app.db_sessionmaker() as s:
+            rows = (await s.execute(select(PluginMetricLatest).where(
+                PluginMetricLatest.source_module == "prometheus",
+                PluginMetricLatest.resource_name.startswith(name_prefix)))).scalars()
+            return {(r.resource_name, r.metric_name): r.value for r in rows}
+
+    async def go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                user = User(username=f"pr-{tag}", email=f"pr-{tag}@test", password_hash="x",
+                            role=UserRole.admin)
+                s.add_all([user,
+                           PrometheusTarget(name=up, url=f"http://{up}/metrics",
+                                            token=encrypt_secret("scrape-token"), keep=["fa_*"]),
+                           PrometheusTarget(name=down, url=f"http://{down}/metrics")])
+                await s.flush()
+                await record_metrics(s, [PluginMetric(source_module="prometheus", resource_name=down,
+                                                      metric_name="is_up", value=1.0)])
+                await s.flush()
+                fields = await alert_svc.rule_fields(
+                    s, name=f"down {tag}", source_module="prometheus", resource_name=down,
+                    metric_name="is_up", operator="<", threshold=1)
+                rule = await alert_svc.add_rule(s, fields, created_by=user.id)
+            rule_id = rule.id
+
+        await scheduler.scrape_due(app, scrape=fake_scrape)
+        first = await latest(up)
+        # Every target was just scraped, so nothing is due on the next tick.
+        await scheduler.scrape_due(app, scrape=fake_scrape)
+        assert len(tokens) == 2
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                for t in (await s.execute(select(PrometheusTarget).where(
+                        PrometheusTarget.name.in_([up, down])))).scalars():
+                    t.last_scraped_at = t.last_scraped_at.replace(year=2000)
+        await asyncio.sleep(1.1)
+        await scheduler.scrape_due(app, scrape=fake_scrape)
+        second = await latest(up)
+        async with app.db_sessionmaker() as s:
+            rows = {t.name: t for t in (await s.execute(select(PrometheusTarget).where(
+                PrometheusTarget.name.in_([up, down])))).scalars()}
+            state = (await s.execute(select(AlertState).where(
+                AlertState.rule_id == rule_id))).scalar_one()
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                for t in rows.values():
+                    await s.delete(await s.get(PrometheusTarget, t.id))
+        return first, second, rows, state, await latest(down)
+
+    first, second, rows, state, down_latest = asyncio.run(go())
+    assert "scrape-token" in tokens
+    assert first[(f"{up}:target=Steward,state=late", "fa_backup_target_state")] == 0.0
+    assert (up, "fa_requests_per_sec") not in first                  # a counter needs two scrapes
+    assert first[(up, "is_up")] == 1.0 and first[(up, "series_kept")] == 2.0
+    assert not any(m == "noise" for _, m in first)                   # not in the keep list
+    # 11 requests over the 1.1 s or more between the scrapes.
+    assert 0 < second[(up, "fa_requests_per_sec")] <= 10.0
+    assert rows[up].last_error is None and rows[up].last_kept == 2 and rows[up].last_dropped == 1
+    assert rows[down].last_error == "Couldn't connect: connection refused."
+    assert rows[down].last_ok_at is None
+    assert down_latest[(down, "is_up")] == 0.0
+    assert state.state == AlertStateEnum.firing

tests/plugins/prometheus/test_pages.py

@@ -0,0 +1,100 @@
+"""The Prometheus pages (#5669): targets not answering first and saying why,
+what was left out, the series a target stored, and the editor."""
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace as NS
+
+from plugins.prometheus.routes import group_series, shown_value
+from tests._templates import render_plugin_template
+from tests.test_style_tokens import hex_colours
+
+
+def _view(name, reach, says="Answering.", ok_minutes=None, capped=0):
+    now = datetime.now(timezone.utc)
+    return {"id": f"id-{name}", "name": name, "url": f"http://{name}:8081/metrics", "reach": reach,
+            "says": says, "scraped_at": now - timedelta(seconds=20),
+            "ok_at": now - timedelta(minutes=ok_minutes) if ok_minutes is not None else None,
+            "interval": 60, "kept": 120, "dropped": 30, "capped": capped, "cap": 1000,
+            "keep": ["fabledarchives_*"], "drop": [], "scrape_ms": 42.4}
+
+
+async def test_a_target_not_answering_says_why_and_the_cap_is_shown(app):
+    from quart import session
+    targets = [_view("archives", "down", "Answered 401: the token was refused.", ok_minutes=180),
+               _view("busy", "up", capped=37), _view("new", "pending")]
+    async with app.test_request_context("/plugins/prometheus/"):
+        session["user_role"] = "admin"
+        page = await render_plugin_template(app, "prometheus", "prometheus/index.html", targets=targets)
+    page = " ".join(page.split())
+    assert '<span class="badge badge-crit">Not answering</span>' in page
+    assert "Answered 401: the token was refused." in page and "Last answered" in page
+    assert "37 series weren't stored" in page and "the cap of 1000" in page
+    assert '<span class="stat-tile-value">42 ms</span>' in page
+    assert "the first scrape runs within seconds of saving" in page
+    assert 'href="/plugins/prometheus/targets/new"' in page and 'href="/plugins/prometheus/targets/id-busy/edit"' in page
+    main = page[page.index("<main"):]
+    assert not hex_colours(main), hex_colours(main)
+
+
+async def test_a_viewer_sees_no_editing_and_an_empty_page_offers_to_add(app):
+    from quart import session
+    async with app.test_request_context("/plugins/prometheus/"):
+        session["user_role"] = "viewer"
+        page = await render_plugin_template(app, "prometheus", "prometheus/index.html",
+                                            targets=[_view("archives", "up")])
+        session["user_role"] = "admin"
+        empty = await render_plugin_template(app, "prometheus", "prometheus/index.html", targets=[])
+    assert "/edit" not in page and "targets/new" not in page
+    assert "No targets yet." in empty and 'href="/plugins/prometheus/targets/new"' in empty
+
+
+def test_series_are_grouped_by_metric_with_their_labels_and_units():
+    at = datetime(2026, 10, 12, 12, 0, tzinfo=timezone.utc)
+    stamp = (at - timedelta(hours=2)).timestamp()
+    rows = [NS(resource_name="A", metric_name="is_up", value=1.0, recorded_at=at),
+            NS(resource_name="A", metric_name="fa_disk_free_bytes", value=2 * 1024 ** 3, recorded_at=at),
+            NS(resource_name="A:target=S,state=late", metric_name="fa_state", value=0.0, recorded_at=at),
+            NS(resource_name="A:target=S,state=fresh", metric_name="fa_state", value=1.0, recorded_at=at),
+            NS(resource_name="A:target=S", metric_name="fa_last_backup_timestamp_seconds", value=stamp,
+               recorded_at=at)]
+    groups = group_series("A", rows)
+    assert [g["metric"] for g in groups] == ["fa_disk_free_bytes", "fa_last_backup_timestamp_seconds", "fa_state"]
+    assert groups[0]["rows"][0]["labels"] == "" and groups[0]["rows"][0]["text"] == "2.0 GiB"
+    assert groups[1]["rows"][0]["moment"] == datetime.fromtimestamp(stamp, timezone.utc)
+    assert [r["labels"] for r in groups[2]["rows"]] == ["target=S,state=fresh", "target=S,state=late"]
+    assert shown_value("x_timestamp_seconds", 0) == {"text": "never", "moment": None}
+    assert shown_value("jobs_per_sec", 0.25)["text"] == "0.25/s"
+    assert shown_value("job_seconds", 90)["text"] == "1m 30s"
+    assert shown_value("anything", None)["text"] == "—"
+
+
+async def test_the_target_page_leads_with_its_state_then_its_series(app):
+    from quart import session
+    groups = [{"metric": "fa_state", "rows": [
+        {"labels": "target=S,state=late", "resource": "archives:target=S,state=late", "at": None,
+         "text": "0", "moment": None}]}]
+    async with app.test_request_context("/plugins/prometheus/targets/id-archives"):
+        session["user_role"] = "viewer"
+        page = await render_plugin_template(app, "prometheus", "prometheus/target.html",
+                                            target=_view("archives", "up", capped=3), groups=groups,
+                                            total=1, q="")
+    page = " ".join(page.split())
+    assert page.index("Answering") < page.index("Series stored") < page.index("fa_state")
+    assert "3 series weren't stored" in page and "Keeping fabledarchives_*" in page
+    assert "resource=archives%3Atarget%3DS%2Cstate%3Dlate" in page and "metric=fa_state" in page
+    assert 'href="/plugins/prometheus/">Prometheus</a>' in page          # its breadcrumb
+    assert "/edit" not in page
+
+
+async def test_the_form_never_echoes_the_token_and_offers_test(app):
+    from quart import session
+    target = NS(id="t1", name="archives")
+    v = {"name": "archives", "url": "http://archives:8081/metrics", "interval_seconds": 60,
+         "series_cap": 1000, "keep": "fabledarchives_*", "drop": "", "verify_tls": True,
+         "enabled": True, "has_token": True}
+    async with app.test_request_context("/plugins/prometheus/targets/t1/edit"):
+        session["user_role"] = "admin"
+        page = await render_plugin_template(app, "prometheus", "prometheus/form.html",
+                                            target=target, v=v, error=None)
+    assert 'name="token" type="password"' in page and "kept; type to change" in page
+    assert 'name="clear_token"' in page and 'hx-post="/plugins/prometheus/targets/test"' in page
+    assert "fabledarchives_*" in page and "Delete target" in page

tests/plugins/prometheus/test_service.py

@@ -0,0 +1,150 @@
+"""The Prometheus plugin's pure parts: which series are stored and under what
+names, the form, and what a page says about the newest scrape (#5668)."""
+from datetime import datetime, timedelta, timezone
+
+import httpx
+import pytest
+
+from plugins.prometheus.models import PrometheusTarget
+from plugins.prometheus.scheduler import is_due
+from plugins.prometheus.scraper import ScrapeError, scrape
+from plugins.prometheus.service import (FormError, map_families, parse_form, patterns,
+                                        rate_name, resource_for, target_state, wanted)
+from steward.core import prometheus
+from steward.core.counter_rates import CounterRates
+
+TEXT = """\
+# TYPE fa_backup_target_state gauge
+fa_backup_target_state{target="Steward",state="fresh"} 1
+fa_backup_target_state{target="Steward",state="late"} 0
+# TYPE fa_http_requests_total counter
+fa_http_requests_total{route="GET /",status="200"} 100
+# TYPE fa_job_seconds histogram
+fa_job_seconds_bucket{le="1"} 2
+fa_job_seconds_bucket{le="+Inf"} 3
+fa_job_seconds_sum 4.5
+fa_job_seconds_count 3
+# TYPE fa_rpc summary
+fa_rpc{quantile="0.5"} 0.2
+fa_rpc_count 9
+fa_disk_free_bytes 1e9
+fa_broken NaN
+other_thing 7
+"""
+
+
+def _mapped(keep=(), drop=(), cap=1000, target="Archives"):
+    return map_families(target, prometheus.parse(TEXT), list(keep), list(drop), cap)
+
+
+def test_gauges_stay_counters_become_rates_and_distributions_keep_sum_and_count():
+    got = {(s.resource, s.metric): (s.value, s.counter) for s in _mapped().series}
+    assert got == {
+        ("Archives:target=Steward,state=fresh", "fa_backup_target_state"): (1.0, False),
+        ("Archives:target=Steward,state=late", "fa_backup_target_state"): (0.0, False),
+        ("Archives:route=GET /,status=200", "fa_http_requests_per_sec"): (100.0, True),
+        ("Archives", "fa_job_seconds_sum_per_sec"): (4.5, True),
+        ("Archives", "fa_job_seconds_count_per_sec"): (3.0, True),
+        ("Archives", "fa_rpc_count_per_sec"): (9.0, True),
+        ("Archives", "fa_disk_free_bytes"): (1e9, False),
+        ("Archives", "other_thing"): (7.0, False),
+    }
+
+
+def test_keep_and_drop_patterns_choose_families_and_drop_wins():
+    m = _mapped(keep=["fa_*"], drop=["fa_http_*"])
+    assert {s.metric for s in m.series} == {
+        "fa_backup_target_state", "fa_job_seconds_sum_per_sec", "fa_job_seconds_count_per_sec",
+        "fa_rpc_count_per_sec", "fa_disk_free_bytes"}
+    assert m.dropped == 2          # the counter's one sample, other_thing's one
+    assert wanted("x", [], []) and not wanted("x", ["y*"], []) and not wanted("y1", ["y*"], ["y1"])
+
+
+def test_the_series_cap_and_overlong_names_are_counted_not_stored():
+    m = _mapped(cap=3)
+    assert len(m.series) == 3 and m.capped == 5
+    long = map_families("t" * 250, prometheus.parse('m{a="bcdefgh"} 1\n'), [], [], 10)
+    assert long.series == [] and long.capped == 1
+
+
+def test_names():
+    assert rate_name("jobs_total") == "jobs_per_sec"
+    assert rate_name("job_seconds_sum") == "job_seconds_sum_per_sec"
+    assert resource_for("T", ()) == "T"
+    assert resource_for("T", (("a", "1"), ("b", "x"))) == "T:a=1,b=x"
+    assert patterns("a_*, b_*\n\n a_* \nc") == ["a_*", "b_*", "c"]
+
+
+def test_a_counter_becomes_a_rate_and_a_reset_is_skipped():
+    rates = CounterRates()
+    t0 = datetime(2026, 10, 12, 12, 0, tzinfo=timezone.utc)
+    key = ("t1", "Archives", "x_per_sec")
+    assert rates.rate(key, 100, t0) is None
+    assert rates.rate(key, 160, t0 + timedelta(seconds=60)) == 1.0
+    assert rates.rate(key, 5, t0 + timedelta(seconds=120)) is None        # the service restarted
+    assert rates.rate(key, 35, t0 + timedelta(seconds=150)) == 1.0
+    rates.forget("t1")
+    assert rates.rate(key, 99, t0 + timedelta(seconds=180)) is None
+
+
+class _Form(dict):
+    def getlist(self, key):
+        v = self.get(key)
+        return v if isinstance(v, list) else ([v] if v else [])
+
+
+def test_the_form_checks_what_it_stores_and_keeps_a_blank_token():
+    values = parse_form(_Form(name="Archives", url="http://archives:8081/metrics", token="s3cret",
+                              interval_seconds="30", series_cap="500", keep="fabledarchives_*",
+                              enabled="on", verify_tls="on"))
+    assert values["token"].startswith("enc:") or values["token"] == "s3cret"
+    assert (values["interval_seconds"], values["series_cap"], values["keep"], values["enabled"]) == (
+        30, 500, ["fabledarchives_*"], True)
+    existing = PrometheusTarget(name="Archives", url="http://a/metrics", token="enc:v1:kept")
+    again = parse_form(_Form(name="Archives", url="http://a/metrics"), existing)
+    assert again["token"] == "enc:v1:kept" and again["enabled"] is False
+    cleared = parse_form(_Form(name="Archives", url="http://a/metrics", clear_token="on"), existing)
+    assert cleared["token"] == ""
+    for bad in (dict(name="", url="http://a"), dict(name="a:b", url="http://a"),
+                dict(name="a", url="ftp://a"), dict(name="a", url="http://a", interval_seconds="5"),
+                dict(name="a", url="http://a", series_cap="0")):
+        with pytest.raises(FormError):
+            parse_form(_Form(bad))
+
+
+def test_state_and_due():
+    now = datetime(2026, 10, 12, 12, 0, tzinfo=timezone.utc)
+    t = PrometheusTarget(name="a", url="http://a", enabled=True, interval_seconds=60)
+    assert target_state(t, now).reach == "pending" and is_due(t, now)
+    t.last_scraped_at = now - timedelta(seconds=30)
+    assert target_state(t, now).reach == "up" and not is_due(t, now)
+    t.last_scraped_at = now - timedelta(seconds=58)
+    assert is_due(t, now)                          # half a tick of slack
+    t.last_error = "Couldn't connect: refused."
+    t.last_scraped_at = now - timedelta(seconds=30)
+    assert (target_state(t, now).reach, target_state(t, now).says) == ("down", "Couldn't connect: refused.")
+    t.last_scraped_at = now - timedelta(minutes=10)
+    assert target_state(t, now).reach == "stale"
+    t.enabled = False
+    assert target_state(t, now).reach == "off" and not is_due(t, now)
+
+
+@pytest.mark.asyncio
+async def test_a_scrape_sends_the_token_and_says_why_it_failed():
+    seen = {}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen["auth"] = request.headers.get("Authorization")
+        if request.url.path == "/denied":
+            return httpx.Response(401)
+        if request.url.path == "/html":
+            return httpx.Response(200, text="<html>hello</html>")
+        return httpx.Response(200, text="up_thing 1\n")
+
+    transport = httpx.MockTransport(handler)
+    fams = await scrape("http://svc/metrics", "tok", transport=transport)
+    assert list(fams) == ["up_thing"] and seen["auth"] == "Bearer tok"
+    with pytest.raises(ScrapeError, match="token was refused"):
+        await scrape("http://svc/denied", "bad", transport=transport)
+    with pytest.raises(ScrapeError, match="no metrics"):
+        await scrape("http://svc/html", transport=transport)