feat(prometheus): scrape any /metrics endpoint into Steward's metrics, with pages (#5668, #5669)

A bundled prometheus plugin, on by default like the other protocol plugins.
Targets are rows edited on its page: address, bearer token (encrypted),
interval, keep/leave-out name patterns and a series cap. Steward does the
filtering; the endpoint serves everything (Archives' ruling).

- A 5 s tick scrapes due targets at once, each within a 30 s deadline, and
  writes the outcome to its row and its series through record_metrics, so
  alert rules can watch any of them.
- Gauges are stored as they are; counters (and a histogram's or summary's
  _sum/_count) as <name>_per_sec rates from the second scrape. Buckets and
  quantiles are skipped. A labelled series is stored under
  "<target>:k=v,…". Each target also records is_up, scrape_ms, series_kept
  and series_capped.
- Pages: targets not answering first, with why; a target's series grouped
  by metric with their newest values; a chart per series; the editor with
  Test, which scrapes once and says what would be stored.
- The counter-to-rate keeper moves to steward/core/counter_rates.py; SNMP's
  RateKeeper now scales its rate on top of it.
- metric_value reads Prometheus's _seconds as a span and _per_sec as a rate.
- The catalog guard treats an empty list as a dynamic source (snmp,
  prometheus), as the rule form does.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/counter_rates.py

@@ -0,0 +1,34 @@
+"""A counter's previous sample, to turn the next one into a per-second rate.
+
+Scrapers that read monotonic counters (SNMP octets, Prometheus *_total)
+store the rate, not the running total. Keys are tuples whose first element
+is the owner (a device, a target), so forgetting an owner drops all of its
+counters at once.
+"""
+from __future__ import annotations
+
+from datetime import datetime
+
+
+class CounterRates:
+    """In memory: after a restart the first sample of each counter gives no
+    rate and the second does. A sample lower than the one before (the source
+    restarted, or a 32-bit counter wrapped) gives no rate rather than a huge
+    negative or wrapped one."""
+
+    def __init__(self) -> None:
+        self._last: dict[tuple, tuple[float, datetime]] = {}
+
+    def rate(self, key: tuple, value: float, at: datetime) -> float | None:
+        prev = self._last.get(key)
+        self._last[key] = (value, at)
+        if prev is None:
+            return None
+        seconds = (at - prev[1]).total_seconds()
+        if seconds <= 0 or value < prev[0]:
+            return None
+        return (value - prev[0]) / seconds
+
+    def forget(self, owner) -> None:
+        for key in [k for k in self._last if k[0] == owner]:
+            del self._last[key]

steward/core/formatting.py

@@ -245,7 +245,8 @@ def worst_reading(readings, thresholds: dict):
 def metric_value(name: str, value: Any) -> str:
     """A metric reading in its unit, from the name's suffix: _pct → 94%,
     _ms → 120 ms, _bytes → 1.2 GiB, _bps → its rate, _secs / seconds_ → a
-    span, _c → 71°C, _days → 12 days; is_up / is_healthy read as yes or no.
+    span (Prometheus's _seconds too), _per_sec → 1.5/s, _c → 71°C, _days →
+    12 days; is_up / is_healthy read as yes or no.
     None (no reading) is "—"."""
     if value is None:
         return "—"
@@ -260,7 +261,10 @@ def metric_value(name: str, value: Any) -> str:
         return human_bytes(value)
     if n.endswith(("_bps", "bytes_rate")):
         return human_rate(value)
-    if n.endswith("_secs") or n.startswith("seconds_"):
+    if n.endswith("_per_sec"):
+        return f"{value:.3g}/s"
+    if n.endswith("_secs") or n.startswith("seconds_") or (
+            n.endswith("_seconds") and not n.endswith("_timestamp_seconds")):
         return human_duration(value)
     if n.endswith("_c"):
         return f"{value:.0f}°C"
@@ -284,6 +288,9 @@ METRICS: dict[str, tuple[str, str]] = {
     "cleanup_secs": ("Hourly cleanup took", "how long the last retention and rollup run took"),
     "open_fds": ("Open files", "files Steward itself has open"),
     "open_fds_pct": ("Open files (% of limit)", "files Steward has open, against its limit"),
+    "scrape_ms": ("Scrape time", "how long the target took to hand over its metrics"),
+    "series_kept": ("Series stored", "series from the target's last scrape that were stored"),
+    "series_capped": ("Series over the cap", "series the last scrape left out: past the target's series cap"),
     "request_rate": ("Requests per second", "requests through this router"),
     "error_rate_4xx_pct": ("4xx responses", "share of responses that were client errors"),
     "error_rate_5xx_pct": ("5xx responses", "share of responses that were server errors"),
@@ -321,7 +328,7 @@ SOURCES: dict[str, str] = {
     "icmp": "Ping monitors", "tcp": "TCP monitors", "dns": "DNS monitors",
     "http": "HTTP monitors", "steward": "Steward itself", "host_agent": "Host agent",
     "docker": "Docker containers", "traefik": "Traefik routers", "unifi": "UniFi",
-    "snmp": "SNMP devices",
+    "snmp": "SNMP devices", "prometheus": "Prometheus targets",
 }
 
 

steward/core/settings.py

@@ -142,6 +142,7 @@ DEFAULTS: dict[str, Any] = {
     "plugin.docker": {"enabled": True},
     "plugin.host_agent": {"enabled": True},
     "plugin.snmp": {"enabled": True},
+    "plugin.prometheus": {"enabled": True},
     # OIDC single-sign-on
     "oidc.enabled": False,
     "oidc.discovery_url": "",