bvandeusenandClaude Opus 5.5
3cb2a95054
A bundled prometheus plugin, on by default like the other protocol plugins. Targets are rows edited on its page: address, bearer token (encrypted), interval, keep/leave-out name patterns and a series cap. Steward does the filtering; the endpoint serves everything (Archives' ruling). - A 5 s tick scrapes due targets at once, each within a 30 s deadline, and writes the outcome to its row and its series through record_metrics, so alert rules can watch any of them. - Gauges are stored as they are; counters (and a histogram's or summary's _sum/_count) as <name>_per_sec rates from the second scrape. Buckets and quantiles are skipped. A labelled series is stored under "<target>:k=v,…". Each target also records is_up, scrape_ms, series_kept and series_capped. - Pages: targets not answering first, with why; a target's series grouped by metric with their newest values; a chart per series; the editor with Test, which scrapes once and says what would be stored. - The counter-to-rate keeper moves to steward/core/counter_rates.py; SNMP's RateKeeper now scales its rate on top of it. - metric_value reads Prometheus's _seconds as a span and _per_sec as a rate. - The catalog guard treats an empty list as a dynamic source (snmp, prometheus), as the rule form does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
314 lines
15 KiB
Python
314 lines
15 KiB
Python
"""The Prometheus pages. Every GET reads Steward's own records: the target
|
|
rows the editor writes and the scraper annotates, and the series it stores.
|
|
Only Test, an admin's explicit action, asks an endpoint anything (System
|
|
178's ruling)."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from quart import Blueprint, abort, current_app, redirect, render_template, request, session
|
|
from sqlalchemy import delete, func, select, update
|
|
|
|
from steward.auth.middleware import require_role
|
|
from steward.core.audit import log_audit
|
|
from steward.core.crypto import decrypt_secret
|
|
from steward.core.formatting import metric_value
|
|
from steward.core.metric_series import latest_values, points
|
|
from steward.models.metrics import PluginMetric, PluginMetricLatest
|
|
from steward.models.users import UserRole
|
|
|
|
from .models import PrometheusTarget
|
|
from .scraper import ScrapeError, scrape
|
|
from .service import (OWN_METRICS, SOURCE, FormError, form_values, map_families, parse_form,
|
|
patterns, target_state)
|
|
|
|
prometheus_bp = Blueprint("prometheus", __name__, template_folder="templates")
|
|
|
|
_REACH_ORDER = {"down": 0, "stale": 1, "pending": 2, "up": 3, "off": 4}
|
|
|
|
|
|
def _actor() -> tuple[str | None, str]:
|
|
return session.get("user_id"), session.get("username", "")
|
|
|
|
|
|
async def _all_targets(db) -> list[PrometheusTarget]:
|
|
return list((await db.execute(select(PrometheusTarget).order_by(PrometheusTarget.name))).scalars().all())
|
|
|
|
|
|
def view(target: PrometheusTarget, own: dict, now: datetime) -> dict:
|
|
"""What a page says about a target: its state in words and the numbers
|
|
from its newest scrape."""
|
|
state = target_state(target, now)
|
|
return {"id": target.id, "name": target.name, "url": target.url, "reach": state.reach,
|
|
"says": state.says, "scraped_at": target.last_scraped_at, "ok_at": target.last_ok_at,
|
|
"interval": target.interval_seconds, "kept": target.last_kept,
|
|
"dropped": target.last_dropped, "capped": target.last_capped, "cap": target.series_cap,
|
|
"keep": list(target.keep or []), "drop": list(target.drop or []),
|
|
"scrape_ms": own.get("scrape_ms")}
|
|
|
|
|
|
def shown_value(metric: str, value: float | None) -> dict:
|
|
"""A series' value in its unit, from the Prometheus naming conventions.
|
|
A *_timestamp_seconds is a moment: it comes back as `moment` for the page
|
|
to show as "3m ago" in the reader's zone; 0 there means "never"."""
|
|
if value is None:
|
|
return {"text": "—", "moment": None}
|
|
if metric.endswith("_timestamp_seconds"):
|
|
if value <= 0:
|
|
return {"text": "never", "moment": None}
|
|
return {"text": "", "moment": datetime.fromtimestamp(value, timezone.utc)}
|
|
return {"text": metric_value(metric, value), "moment": None}
|
|
|
|
|
|
@prometheus_bp.get("/")
|
|
@require_role(UserRole.viewer)
|
|
async def index():
|
|
async with current_app.db_sessionmaker() as db:
|
|
targets = await _all_targets(db)
|
|
own = {}
|
|
if targets:
|
|
rows = (await db.execute(select(PluginMetricLatest).where(
|
|
PluginMetricLatest.source_module == SOURCE,
|
|
PluginMetricLatest.resource_name.in_([t.name for t in targets]),
|
|
PluginMetricLatest.metric_name == "scrape_ms"))).scalars()
|
|
own = {r.resource_name: {"scrape_ms": r.value} for r in rows}
|
|
now = datetime.now(timezone.utc)
|
|
shown = [view(t, own.get(t.name, {}), now) for t in targets]
|
|
shown.sort(key=lambda v: (_REACH_ORDER[v["reach"]], v["name"].lower()))
|
|
return await render_template("prometheus/index.html", targets=shown)
|
|
|
|
|
|
# ── The editor ────────────────────────────────────────────────────────────────
|
|
|
|
async def _render_form(target: PrometheusTarget | None, values: dict, error: str | None = None,
|
|
status: int = 200):
|
|
return await render_template("prometheus/form.html", target=target, v=values, error=error), status
|
|
|
|
|
|
def _typed(form, target: PrometheusTarget | None = None) -> dict:
|
|
"""What was typed, to show the form again with an error. The token is not
|
|
echoed back."""
|
|
return {"name": form.get("name", ""), "url": form.get("url", ""),
|
|
"interval_seconds": form.get("interval_seconds", ""), "series_cap": form.get("series_cap", ""),
|
|
"keep": form.get("keep", ""), "drop": form.get("drop", ""),
|
|
"verify_tls": bool(form.get("verify_tls")), "enabled": bool(form.get("enabled")),
|
|
"has_token": bool(target and target.token)}
|
|
|
|
|
|
async def _name_taken(db, name: str, other_than: str | None = None) -> bool:
|
|
q = select(PrometheusTarget.id).where(PrometheusTarget.name == name)
|
|
if other_than:
|
|
q = q.where(PrometheusTarget.id != other_than)
|
|
return (await db.execute(q)).first() is not None
|
|
|
|
|
|
async def _target_or_404(db, target_id: str) -> PrometheusTarget:
|
|
target = await db.get(PrometheusTarget, target_id)
|
|
if target is None:
|
|
abort(404)
|
|
return target
|
|
|
|
|
|
@prometheus_bp.get("/targets/new")
|
|
@require_role(UserRole.admin)
|
|
async def new_target():
|
|
return await _render_form(None, form_values(None))
|
|
|
|
|
|
@prometheus_bp.post("/targets/new")
|
|
@require_role(UserRole.admin)
|
|
async def create_target():
|
|
form = await request.form
|
|
try:
|
|
values = parse_form(form)
|
|
except FormError as exc:
|
|
return await _render_form(None, _typed(form), str(exc), 422)
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
if await _name_taken(db, values["name"]):
|
|
return await _render_form(None, _typed(form),
|
|
f"A target is already named “{values['name']}”.", 422)
|
|
target = PrometheusTarget(**values)
|
|
db.add(target)
|
|
await db.flush()
|
|
target_id = target.id
|
|
await log_audit(current_app, *_actor(), "prometheus.target_created", entity_type="prometheus_target",
|
|
entity_id=target_id, detail={"name": values["name"], "url": values["url"]})
|
|
return redirect(f"/plugins/prometheus/targets/{target_id}")
|
|
|
|
|
|
@prometheus_bp.get("/targets/<target_id>/edit")
|
|
@require_role(UserRole.admin)
|
|
async def edit_target(target_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
target = await _target_or_404(db, target_id)
|
|
return await _render_form(target, form_values(target))
|
|
|
|
|
|
@prometheus_bp.post("/targets/<target_id>/edit")
|
|
@require_role(UserRole.admin)
|
|
async def save_target(target_id: str):
|
|
form = await request.form
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
target = await _target_or_404(db, target_id)
|
|
try:
|
|
values = parse_form(form, target)
|
|
except FormError as exc:
|
|
return await _render_form(target, _typed(form, target), str(exc), 422)
|
|
if await _name_taken(db, values["name"], other_than=target.id):
|
|
return await _render_form(target, _typed(form, target),
|
|
f"A target is already named “{values['name']}”.", 422)
|
|
old_name = target.name
|
|
for column, value in values.items():
|
|
setattr(target, column, value)
|
|
# Where it is or what is kept may have changed: the last outcome no
|
|
# longer describes it, so the page says "not scraped yet" until the next scrape.
|
|
target.last_scraped_at = target.last_ok_at = target.last_error = None
|
|
target.last_kept = target.last_dropped = target.last_capped = 0
|
|
if old_name != target.name:
|
|
# Its history is stored under its name, and its labelled series
|
|
# under "<name>:…": move them with the rename.
|
|
for model in (PluginMetric, PluginMetricLatest):
|
|
await db.execute(update(model).where(
|
|
model.source_module == SOURCE, model.resource_name == old_name
|
|
).values(resource_name=target.name))
|
|
# "<old>:labels" keeps its ":labels" (substr is 1-based).
|
|
await db.execute(update(model).where(
|
|
model.source_module == SOURCE,
|
|
model.resource_name.startswith(old_name + ":", autoescape=True),
|
|
).values(resource_name=target.name + func.substr(model.resource_name, len(old_name) + 1)))
|
|
from . import scheduler
|
|
scheduler.RATES.forget(target_id)
|
|
detail = {"name": values["name"]}
|
|
if old_name != values["name"]:
|
|
detail["old_name"] = old_name
|
|
await log_audit(current_app, *_actor(), "prometheus.target_updated", entity_type="prometheus_target",
|
|
entity_id=target_id, detail=detail)
|
|
return redirect(f"/plugins/prometheus/targets/{target_id}")
|
|
|
|
|
|
@prometheus_bp.post("/targets/<target_id>/delete")
|
|
@require_role(UserRole.admin)
|
|
async def delete_target(target_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
target = await _target_or_404(db, target_id)
|
|
name = target.name
|
|
await db.delete(target)
|
|
# Its newest values go with it, so nothing reads it as a target
|
|
# that stopped answering. Its history ages out with retention.
|
|
await db.execute(delete(PluginMetricLatest).where(
|
|
PluginMetricLatest.source_module == SOURCE,
|
|
(PluginMetricLatest.resource_name == name)
|
|
| PluginMetricLatest.resource_name.startswith(name + ":", autoescape=True)))
|
|
from . import scheduler
|
|
scheduler.RATES.forget(target_id)
|
|
await log_audit(current_app, *_actor(), "prometheus.target_deleted", entity_type="prometheus_target",
|
|
entity_id=target_id, detail={"name": name})
|
|
return redirect("/plugins/prometheus/")
|
|
|
|
|
|
TEST_FAMILIES_SHOWN = 40
|
|
|
|
|
|
@prometheus_bp.post("/targets/test")
|
|
@require_role(UserRole.admin)
|
|
async def test_target():
|
|
"""Scrape the (unsaved) form's address once and say what would be stored.
|
|
An admin's explicit action, so it may contact the endpoint."""
|
|
form = await request.form
|
|
existing = None
|
|
if form.get("target_id"):
|
|
async with current_app.db_sessionmaker() as db:
|
|
existing = await db.get(PrometheusTarget, form["target_id"])
|
|
url = (form.get("url") or "").strip()
|
|
token = (form.get("token") or "").strip()
|
|
if not token and existing is not None and existing.token and not form.get("clear_token"):
|
|
token = decrypt_secret(existing.token, context=f"prometheus target {existing.name} token")
|
|
found: dict = {}
|
|
if not url.startswith(("http://", "https://")):
|
|
found["error"] = "Enter the address first: http://… or https://…"
|
|
else:
|
|
try:
|
|
families = await scrape(url, token, bool(form.get("verify_tls")))
|
|
except ScrapeError as exc:
|
|
found["error"] = str(exc)
|
|
else:
|
|
try:
|
|
cap = max(1, int(form.get("series_cap") or 1000))
|
|
except ValueError:
|
|
cap = 1000
|
|
keep, drop = patterns(form.get("keep") or ""), patterns(form.get("drop") or "")
|
|
mapped = map_families("t", families, keep, drop, cap)
|
|
kept_names = {}
|
|
for s in mapped.series:
|
|
kept_names[s.metric] = kept_names.get(s.metric, 0) + 1
|
|
found = {"families": len(families), "kept": len(mapped.series), "dropped": mapped.dropped,
|
|
"capped": mapped.capped,
|
|
"names": sorted(kept_names.items())[:TEST_FAMILIES_SHOWN],
|
|
"more": max(0, len(kept_names) - TEST_FAMILIES_SHOWN)}
|
|
return await render_template("prometheus/_test.html", found=found, url=url)
|
|
|
|
|
|
# ── The target page and one series ───────────────────────────────────────────
|
|
|
|
def group_series(target_name: str, latest: list) -> list[dict]:
|
|
"""The target's stored series grouped by metric, each with its rows:
|
|
the labels that tell them apart, and the newest value."""
|
|
groups: dict[str, list] = {}
|
|
for row in latest:
|
|
if row.resource_name == target_name and row.metric_name in OWN_METRICS:
|
|
continue
|
|
labels = row.resource_name[len(target_name) + 1:] if row.resource_name != target_name else ""
|
|
groups.setdefault(row.metric_name, []).append({
|
|
"labels": labels, "resource": row.resource_name, "at": row.recorded_at,
|
|
**shown_value(row.metric_name, row.value)})
|
|
return [{"metric": m, "rows": sorted(rows, key=lambda r: r["labels"])}
|
|
for m, rows in sorted(groups.items())]
|
|
|
|
|
|
@prometheus_bp.get("/targets/<target_id>")
|
|
@require_role(UserRole.viewer)
|
|
async def target_detail(target_id: str):
|
|
q = (request.args.get("q") or "").strip()
|
|
async with current_app.db_sessionmaker() as db:
|
|
target = await _target_or_404(db, target_id)
|
|
latest = await latest_values(db, SOURCE, target.name, include_children=True)
|
|
own = {r.metric_name: r.value for r in latest if r.resource_name == target.name
|
|
and r.metric_name in OWN_METRICS}
|
|
groups = group_series(target.name, latest)
|
|
total = sum(len(g["rows"]) for g in groups)
|
|
if q:
|
|
groups = [g for g in groups if q.lower() in g["metric"].lower()]
|
|
now = datetime.now(timezone.utc)
|
|
return await render_template("prometheus/target.html", target=view(target, own, now), groups=groups,
|
|
total=total, q=q)
|
|
|
|
|
|
RANGES = (1, 6, 24, 168)
|
|
|
|
|
|
@prometheus_bp.get("/targets/<target_id>/series")
|
|
@require_role(UserRole.viewer)
|
|
async def series_detail(target_id: str):
|
|
resource = request.args.get("resource") or ""
|
|
metric = request.args.get("metric") or ""
|
|
hours = request.args.get("hours", type=int, default=24)
|
|
if hours not in RANGES:
|
|
hours = 24
|
|
async with current_app.db_sessionmaker() as db:
|
|
target = await _target_or_404(db, target_id)
|
|
if resource != target.name and not resource.startswith(target.name + ":"):
|
|
abort(404)
|
|
now_row = await db.get(PluginMetricLatest, (SOURCE, resource, metric))
|
|
series = await points(db, SOURCE, resource, metric,
|
|
datetime.now(timezone.utc) - timedelta(hours=hours))
|
|
labels = resource[len(target.name) + 1:] if resource != target.name else ""
|
|
stamp_axis = metric.endswith("_timestamp_seconds")
|
|
return await render_template(
|
|
"prometheus/series.html", target=target, resource=resource, metric=metric, labels=labels,
|
|
now=shown_value(metric, now_row.value if now_row else None), hours=hours, ranges=RANGES,
|
|
points_json=json.dumps([(t.isoformat(), v) for t, v in series]), stamp_axis=stamp_axis)
|