bvandeusenandClaude Opus 5.5
3cb2a95054
A bundled prometheus plugin, on by default like the other protocol plugins. Targets are rows edited on its page: address, bearer token (encrypted), interval, keep/leave-out name patterns and a series cap. Steward does the filtering; the endpoint serves everything (Archives' ruling). - A 5 s tick scrapes due targets at once, each within a 30 s deadline, and writes the outcome to its row and its series through record_metrics, so alert rules can watch any of them. - Gauges are stored as they are; counters (and a histogram's or summary's _sum/_count) as <name>_per_sec rates from the second scrape. Buckets and quantiles are skipped. A labelled series is stored under "<target>:k=v,…". Each target also records is_up, scrape_ms, series_kept and series_capped. - Pages: targets not answering first, with why; a target's series grouped by metric with their newest values; a chart per series; the editor with Test, which scrapes once and says what would be stored. - The counter-to-rate keeper moves to steward/core/counter_rates.py; SNMP's RateKeeper now scales its rate on top of it. - metric_value reads Prometheus's _seconds as a span and _per_sec as a rate. - The catalog guard treats an empty list as a dynamic source (snmp, prometheus), as the rule form does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
59 lines
2.5 KiB
Python
59 lines
2.5 KiB
Python
# plugins/prometheus/scraper.py
|
|
"""Fetching one /metrics endpoint. Every failure comes back as one sentence
|
|
a page can show, never a stack trace."""
|
|
from __future__ import annotations
|
|
|
|
import httpx
|
|
|
|
from steward.core import prometheus
|
|
|
|
TIMEOUT_SECONDS = 10.0
|
|
# A /metrics page past this is not one to store a series of; reading stops.
|
|
MAX_BYTES = 20 * 1024 * 1024
|
|
|
|
ACCEPT = "text/plain;version=0.0.4;q=1,*/*;q=0.1"
|
|
|
|
|
|
class ScrapeError(Exception):
|
|
"""Why a scrape got nothing, said to the person."""
|
|
|
|
|
|
async def scrape(url: str, token: str = "", verify_tls: bool = True,
|
|
transport: httpx.AsyncBaseTransport | None = None) -> dict[str, prometheus.Family]:
|
|
"""The endpoint's families. `transport` stands in for the network in tests."""
|
|
headers = {"Accept": ACCEPT}
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
try:
|
|
async with httpx.AsyncClient(timeout=TIMEOUT_SECONDS, verify=verify_tls,
|
|
transport=transport, follow_redirects=False) as client:
|
|
async with client.stream("GET", url, headers=headers) as response:
|
|
if response.status_code in (401, 403):
|
|
raise ScrapeError(f"Answered {response.status_code}: the token was refused. "
|
|
"Check the token, or make a new one on the service.")
|
|
if response.status_code != 200:
|
|
raise ScrapeError(f"Answered {response.status_code} {response.reason_phrase}, not metrics.")
|
|
body = bytearray()
|
|
async for chunk in response.aiter_bytes():
|
|
body.extend(chunk)
|
|
if len(body) > MAX_BYTES:
|
|
raise ScrapeError(f"The page is over {MAX_BYTES // (1024 * 1024)} MB; "
|
|
"is this a /metrics address?")
|
|
except ScrapeError:
|
|
raise
|
|
except httpx.TimeoutException:
|
|
raise ScrapeError(f"No answer within {TIMEOUT_SECONDS:.0f} s.") from None
|
|
except httpx.ConnectError as exc:
|
|
raise ScrapeError(f"Couldn't connect: {_reason(exc)}.") from None
|
|
except httpx.HTTPError as exc:
|
|
raise ScrapeError(f"The request failed: {_reason(exc)}.") from None
|
|
families = prometheus.parse(body.decode("utf-8", errors="replace"))
|
|
if not families:
|
|
raise ScrapeError("It answered, but with no metrics in Prometheus format.")
|
|
return families
|
|
|
|
|
|
def _reason(exc: Exception) -> str:
|
|
text = str(exc).strip() or type(exc).__name__
|
|
return text.rstrip(".")
|