Files
FabledSteward/plugins/prometheus/routes.py
T
bvandeusenandClaude Opus 5.5 bd762307e0
CI / lint (push) Successful in 2s
CI / integration (push) Successful in 1m16s
CI / unit (push) Failing after 1m29s
CI / publish (push) Skipped
feat(prometheus): the widget template library: list, edit with live preview, paste in, upload, export, duplicate (#5692)
Prometheus gains a "Widget templates" tab beside Targets. It lists the
built-in templates and an admin's own, any that no longer pass the checker
first. A template opens as its YAML beside a live preview drawn over a
chosen target exactly as the dashboard widget draws it (routes.draw, now
shared by both). An admin writes or pastes a new one, reads a .yaml upload
into the editor to look over, duplicates any template under a free name,
edits and deletes their own; anyone can export the YAML. Built-in
templates are read-only.

Custom templates are rows in prometheus_widget_templates (migration
prometheus_002_templates), kept in backups, keyed "custom:<id>" so a
rename keeps its widgets. Saving needs a template that passes the checker
and a name no other template has. Create, update and delete are audited
and link to the template.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 09:22:31 -04:00

583 lines
27 KiB
Python

"""The Prometheus pages. Every GET reads Steward's own records: the target
rows the editor writes and the scraper annotates, and the series it stores.
Only Test, an admin's explicit action, asks an endpoint anything (System
178's ruling)."""
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from quart import Blueprint, Response, abort, current_app, redirect, render_template, request, session
from sqlalchemy import delete, func, select, update
from steward.auth.middleware import require_role
from steward.core.audit import log_audit
from steward.core.crypto import decrypt_secret
from steward.core.metric_series import latest_values, points
from steward.models.metrics import PluginMetric, PluginMetricLatest
from steward.models.users import UserRole
from . import library
from .models import PrometheusTarget, PrometheusWidgetTemplate
from .scraper import ScrapeError, scrape
from .service import (OWN_METRICS, SOURCE, FormError, fold_states, form_values, map_families,
parse_form, patterns, shown_value, target_state)
prometheus_bp = Blueprint("prometheus", __name__, template_folder="templates")
_REACH_ORDER = {"down": 0, "stale": 1, "pending": 2, "up": 3, "off": 4}
def _actor() -> tuple[str | None, str]:
return session.get("user_id"), session.get("username", "")
async def _all_targets(db) -> list[PrometheusTarget]:
return list((await db.execute(select(PrometheusTarget).order_by(PrometheusTarget.name))).scalars().all())
def view(target: PrometheusTarget, own: dict, now: datetime) -> dict:
"""What a page says about a target: its state in words and the numbers
from its newest scrape."""
state = target_state(target, now)
return {"id": target.id, "name": target.name, "url": target.url, "reach": state.reach,
"says": state.says, "scraped_at": target.last_scraped_at, "ok_at": target.last_ok_at,
"interval": target.interval_seconds, "kept": target.last_kept,
"dropped": target.last_dropped, "capped": target.last_capped, "cap": target.series_cap,
"keep": list(target.keep or []), "drop": list(target.drop or []),
"scrape_ms": own.get("scrape_ms")}
@prometheus_bp.get("/")
@require_role(UserRole.viewer)
async def index():
async with current_app.db_sessionmaker() as db:
targets = await _all_targets(db)
own = {}
if targets:
rows = (await db.execute(select(PluginMetricLatest).where(
PluginMetricLatest.source_module == SOURCE,
PluginMetricLatest.resource_name.in_([t.name for t in targets]),
PluginMetricLatest.metric_name == "scrape_ms"))).scalars()
own = {r.resource_name: {"scrape_ms": r.value} for r in rows}
now = datetime.now(timezone.utc)
shown = [view(t, own.get(t.name, {}), now) for t in targets]
shown.sort(key=lambda v: (_REACH_ORDER[v["reach"]], v["name"].lower()))
return await render_template("prometheus/index.html", targets=shown)
# ── The editor ────────────────────────────────────────────────────────────────
async def _render_form(target: PrometheusTarget | None, values: dict, error: str | None = None,
status: int = 200):
return await render_template("prometheus/form.html", target=target, v=values, error=error), status
def _typed(form, target: PrometheusTarget | None = None) -> dict:
"""What was typed, to show the form again with an error. The token is not
echoed back."""
return {"name": form.get("name", ""), "url": form.get("url", ""),
"interval_seconds": form.get("interval_seconds", ""), "series_cap": form.get("series_cap", ""),
"keep": form.get("keep", ""), "drop": form.get("drop", ""),
"verify_tls": bool(form.get("verify_tls")), "enabled": bool(form.get("enabled")),
"has_token": bool(target and target.token)}
async def _name_taken(db, name: str, other_than: str | None = None) -> bool:
q = select(PrometheusTarget.id).where(PrometheusTarget.name == name)
if other_than:
q = q.where(PrometheusTarget.id != other_than)
return (await db.execute(q)).first() is not None
async def _target_or_404(db, target_id: str) -> PrometheusTarget:
target = await db.get(PrometheusTarget, target_id)
if target is None:
abort(404)
return target
@prometheus_bp.get("/targets/new")
@require_role(UserRole.admin)
async def new_target():
return await _render_form(None, form_values(None))
@prometheus_bp.post("/targets/new")
@require_role(UserRole.admin)
async def create_target():
form = await request.form
try:
values = parse_form(form)
except FormError as exc:
return await _render_form(None, _typed(form), str(exc), 422)
async with current_app.db_sessionmaker() as db:
async with db.begin():
if await _name_taken(db, values["name"]):
return await _render_form(None, _typed(form),
f"A target is already named “{values['name']}”.", 422)
target = PrometheusTarget(**values)
db.add(target)
await db.flush()
target_id = target.id
await log_audit(current_app, *_actor(), "prometheus.target_created", entity_type="prometheus_target",
entity_id=target_id, detail={"name": values["name"], "url": values["url"]})
return redirect(f"/plugins/prometheus/targets/{target_id}")
@prometheus_bp.get("/targets/<target_id>/edit")
@require_role(UserRole.admin)
async def edit_target(target_id: str):
async with current_app.db_sessionmaker() as db:
target = await _target_or_404(db, target_id)
return await _render_form(target, form_values(target))
@prometheus_bp.post("/targets/<target_id>/edit")
@require_role(UserRole.admin)
async def save_target(target_id: str):
form = await request.form
async with current_app.db_sessionmaker() as db:
async with db.begin():
target = await _target_or_404(db, target_id)
try:
values = parse_form(form, target)
except FormError as exc:
return await _render_form(target, _typed(form, target), str(exc), 422)
if await _name_taken(db, values["name"], other_than=target.id):
return await _render_form(target, _typed(form, target),
f"A target is already named “{values['name']}”.", 422)
old_name = target.name
for column, value in values.items():
setattr(target, column, value)
# Where it is or what is kept may have changed: the last outcome no
# longer describes it, so the page says "not scraped yet" until the next scrape.
target.last_scraped_at = target.last_ok_at = target.last_error = None
target.last_kept = target.last_dropped = target.last_capped = 0
if old_name != target.name:
# Its history is stored under its name, and its labelled series
# under "<name>:…": move them with the rename.
for model in (PluginMetric, PluginMetricLatest):
await db.execute(update(model).where(
model.source_module == SOURCE, model.resource_name == old_name
).values(resource_name=target.name))
# "<old>:labels" keeps its ":labels" (substr is 1-based).
await db.execute(update(model).where(
model.source_module == SOURCE,
model.resource_name.startswith(old_name + ":", autoescape=True),
).values(resource_name=target.name + func.substr(model.resource_name, len(old_name) + 1)))
from . import scheduler
scheduler.RATES.forget(target_id)
detail = {"name": values["name"]}
if old_name != values["name"]:
detail["old_name"] = old_name
await log_audit(current_app, *_actor(), "prometheus.target_updated", entity_type="prometheus_target",
entity_id=target_id, detail=detail)
return redirect(f"/plugins/prometheus/targets/{target_id}")
@prometheus_bp.post("/targets/<target_id>/delete")
@require_role(UserRole.admin)
async def delete_target(target_id: str):
async with current_app.db_sessionmaker() as db:
async with db.begin():
target = await _target_or_404(db, target_id)
name = target.name
await db.delete(target)
# Its newest values go with it, so nothing reads it as a target
# that stopped answering. Its history ages out with retention.
await db.execute(delete(PluginMetricLatest).where(
PluginMetricLatest.source_module == SOURCE,
(PluginMetricLatest.resource_name == name)
| PluginMetricLatest.resource_name.startswith(name + ":", autoescape=True)))
from . import scheduler
scheduler.RATES.forget(target_id)
await log_audit(current_app, *_actor(), "prometheus.target_deleted", entity_type="prometheus_target",
entity_id=target_id, detail={"name": name})
return redirect("/plugins/prometheus/")
TEST_FAMILIES_SHOWN = 40
@prometheus_bp.post("/targets/test")
@require_role(UserRole.admin)
async def test_target():
"""Scrape the (unsaved) form's address once and say what would be stored.
An admin's explicit action, so it may contact the endpoint."""
form = await request.form
existing = None
if form.get("target_id"):
async with current_app.db_sessionmaker() as db:
existing = await db.get(PrometheusTarget, form["target_id"])
url = (form.get("url") or "").strip()
token = (form.get("token") or "").strip()
if not token and existing is not None and existing.token and not form.get("clear_token"):
token = decrypt_secret(existing.token, context=f"prometheus target {existing.name} token")
found: dict = {}
if not url.startswith(("http://", "https://")):
found["error"] = "Enter the address first: http://… or https://…"
else:
try:
families = await scrape(url, token, bool(form.get("verify_tls")))
except ScrapeError as exc:
found["error"] = str(exc)
else:
try:
cap = max(1, int(form.get("series_cap") or 1000))
except ValueError:
cap = 1000
keep, drop = patterns(form.get("keep") or ""), patterns(form.get("drop") or "")
mapped = map_families("t", families, keep, drop, cap)
kept_names = {}
for s in mapped.series:
kept_names[s.metric] = kept_names.get(s.metric, 0) + 1
found = {"families": len(families), "kept": len(mapped.series), "dropped": mapped.dropped,
"capped": mapped.capped,
"names": sorted(kept_names.items())[:TEST_FAMILIES_SHOWN],
"more": max(0, len(kept_names) - TEST_FAMILIES_SHOWN)}
return await render_template("prometheus/_test.html", found=found, url=url)
# ── The target page and one series ───────────────────────────────────────────
def group_series(target_name: str, latest: list) -> list[dict]:
"""The target's stored series grouped by metric, each with its rows:
the labels that tell them apart, and the newest value. A state set is
folded to one row per thing it describes, naming the current state
(`states` set on the group)."""
groups: dict[str, list] = {}
for row in latest:
if row.resource_name == target_name and row.metric_name in OWN_METRICS:
continue
labels = row.resource_name[len(target_name) + 1:] if row.resource_name != target_name else ""
groups.setdefault(row.metric_name, []).append({
"labels": labels, "resource": row.resource_name, "at": row.recorded_at,
"value": row.value, **shown_value(row.metric_name, row.value)})
out = []
for metric, rows in sorted(groups.items()):
folded = fold_states(metric, rows)
if folded is not None:
out.append({"metric": metric, "states": True, "rows": [
{"labels": f["labels"], "resource": f["resource"], "moment": None,
"text": f["state"] or "none", "states": f["states"]} for f in folded]})
else:
out.append({"metric": metric, "states": False,
"rows": sorted(rows, key=lambda r: r["labels"])})
return out
@prometheus_bp.get("/targets/<target_id>")
@require_role(UserRole.viewer)
async def target_detail(target_id: str):
q = (request.args.get("q") or "").strip()
async with current_app.db_sessionmaker() as db:
target = await _target_or_404(db, target_id)
latest = await latest_values(db, SOURCE, target.name, include_children=True)
own = {r.metric_name: r.value for r in latest if r.resource_name == target.name
and r.metric_name in OWN_METRICS}
groups = group_series(target.name, latest)
total = sum(len(g["rows"]) for g in groups)
if q:
groups = [g for g in groups if q.lower() in g["metric"].lower()]
now = datetime.now(timezone.utc)
return await render_template("prometheus/target.html", target=view(target, own, now), groups=groups,
total=total, q=q)
RANGES = (1, 6, 24, 168)
@prometheus_bp.get("/targets/<target_id>/series")
@require_role(UserRole.viewer)
async def series_detail(target_id: str):
resource = request.args.get("resource") or ""
metric = request.args.get("metric") or ""
hours = request.args.get("hours", type=int, default=24)
if hours not in RANGES:
hours = 24
async with current_app.db_sessionmaker() as db:
target = await _target_or_404(db, target_id)
if resource != target.name and not resource.startswith(target.name + ":"):
abort(404)
now_row = await db.get(PluginMetricLatest, (SOURCE, resource, metric))
series = await points(db, SOURCE, resource, metric,
datetime.now(timezone.utc) - timedelta(hours=hours))
labels = resource[len(target.name) + 1:] if resource != target.name else ""
stamp_axis = metric.endswith("_timestamp_seconds")
return await render_template(
"prometheus/series.html", target=target, resource=resource, metric=metric, labels=labels,
now=shown_value(metric, now_row.value if now_row else None), hours=hours, ranges=RANGES,
points_json=json.dumps([(t.isoformat(), v) for t, v in series]), stamp_axis=stamp_axis)
# ── The dashboard widget ─────────────────────────────────────────────────────
SPARK_HOURS = 24
def spark_heights(values: list[float]) -> list[float]:
"""Bar heights in % of the series' own peak over the window (System
154's ruling on small history bars); a flat zero draws flat."""
peak = max((abs(v) for v in values), default=0.0)
return [round(100.0 * abs(v) / peak, 1) if peak else 0.0 for v in values]
async def draw(db, template, target: PrometheusTarget, now: datetime):
"""`template` drawn over `target`'s stored values, with the last day's bars
for the items that ask for them. The dashboard widget and the template
editor's preview both draw through here."""
from .template_view import render
latest = await latest_values(db, SOURCE, target.name, include_children=True)
shown = render(template, target.name, latest, now)
if shown.wants:
since = now - timedelta(hours=SPARK_HOURS)
history = {}
for resource, metric in shown.wants:
got = await points(db, SOURCE, resource, metric, since, target=SPARK_HOURS)
history[(resource, metric)] = spark_heights([v for _, v in got])
shown = render(template, target.name, latest, now, history=history)
return shown
@prometheus_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
"""The Prometheus dashboard widget: one target drawn by one template
(System 181's ruling), in widget shape. Reads stored values only."""
target_id = request.args.get("target") or ""
key = request.args.get("template") or ""
now = datetime.now(timezone.utc)
async with current_app.db_sessionmaker() as db:
target = await db.get(PrometheusTarget, target_id) if target_id else None
entry = await library.find(db, key) if key else None
problem = ("This widget has no target; remove it and add it again with one." if not target_id
else "The target this widget showed has been deleted." if target is None
else "This widget has no template; remove it and add it again with one." if not key
else "The template this widget used is gone." if entry is None
else f"The template “{entry.name}” has a problem: {entry.problems[0]}"
if entry.template is None else "")
if problem:
return await render_template("prometheus/widget.html", problem=problem, target=None,
view=None)
shown = await draw(db, entry.template, target, now)
return await render_template("prometheus/widget.html", problem="", view=shown,
target=view_target(target, now), spark_hours=SPARK_HOURS)
def view_target(target: PrometheusTarget, now: datetime) -> dict:
state = target_state(target, now)
return {"id": target.id, "name": target.name, "reach": state.reach, "says": state.says,
"ok_at": target.last_ok_at, "scraped_at": target.last_scraped_at}
# ── The template library (#5692) ─────────────────────────────────────────────
# Bundled templates are read-only: they can be looked at, exported and
# duplicated. An admin writes, pastes or uploads their own, edits and deletes
# them. Nothing is fetched from anywhere (System 181's ruling).
def _shown_entry(entry, used: int) -> dict:
return {"key": entry.key, "id": entry.id, "name": entry.name, "bundled": entry.bundled,
"version": entry.template.version if entry.template else None,
"description": entry.template.description if entry.template else "",
"problem": str(entry.problems[0]) if entry.problems else "",
"updated_at": entry.updated_at, "used": used}
@prometheus_bp.get("/templates/")
@require_role(UserRole.viewer)
async def templates_index():
async with current_app.db_sessionmaker() as db:
entries = await library.entries(db)
used = await library.used_by(db)
shown = [_shown_entry(e, used.get(e.key, 0)) for e in entries]
# One that no longer passes the checker first: its widgets say so too.
shown.sort(key=lambda t: (not t["problem"], t["name"].lower()))
return await render_template("prometheus/templates.html", templates=shown)
async def _editor(entry=None, text: str = "", problems=(), note: str = "", status: int = 200,
saving: bool = False):
"""The template page: an admin's editor, or a read-only view of a
bundled template (or of any template, to a viewer). Both preview it."""
async with current_app.db_sessionmaker() as db:
targets = await library.target_options(db)
used = (await library.used_by(db)).get(entry.key, 0) if entry is not None else 0
editable = session.get("user_role") == "admin" and (entry is None or not entry.bundled)
return await render_template(
"prometheus/template.html", entry=entry, title=entry.name if entry else "New template",
text=text if entry is None or text else entry.text,
problems=[str(p) for p in problems], note=note, targets=targets, editable=editable,
used=used, saving=saving), status
async def _entry_or_404(db, key: str):
entry = await library.find(db, key)
if entry is None:
abort(404)
return entry
@prometheus_bp.get("/templates/new")
@require_role(UserRole.admin)
async def new_template():
"""A blank editor, or a copy of another template (?from=<key>) under a
name nobody has."""
source = request.args.get("from") or ""
if not source:
return await _editor()
async with current_app.db_sessionmaker() as db:
entry = await _entry_or_404(db, source)
taken = {e.name for e in await library.entries(db)}
return await _editor(text=library.renamed(entry.text, library.copy_name(entry.name, taken)),
note=f"A copy of “{entry.name}”. Nothing is saved until you press Save.")
async def _checked(db, text: str, other_than: str | None = None):
"""The template `text` describes and its problems: the checker's, and a
name another template already has."""
from .template_format import Problem, TemplateError, check
try:
template = check(text)
except TemplateError as exc:
return None, exc.problems
if (await library.name_taken(db, template.name, other_than)
or any(e.name == template.name for e in library.bundled())):
line = next((i for i, ln in enumerate(text.splitlines(), 1) if ln.startswith("name:")), 0)
return None, [Problem(line, f"A template is already named “{template.name}”; give this one its own name.")]
return template, []
def _typed_text(form) -> str:
return (form.get("text") or "").replace("\r\n", "\n")
@prometheus_bp.post("/templates/new")
@require_role(UserRole.admin)
async def create_template():
form = await request.form
text = _typed_text(form)
async with current_app.db_sessionmaker() as db:
async with db.begin():
template, problems = await _checked(db, text)
if problems:
return await _editor(text=text, problems=problems, status=422, saving=True)
row = PrometheusWidgetTemplate(name=template.name, text=text)
db.add(row)
await db.flush()
row_id = row.id
await log_audit(current_app, *_actor(), "prometheus.template_created", entity_type="prometheus_template",
entity_id=row_id, detail={"name": template.name, "version": template.version})
return redirect(f"/plugins/prometheus/templates/custom:{row_id}")
UPLOAD_LIMIT = 64 * 1024
@prometheus_bp.post("/templates/upload")
@require_role(UserRole.admin)
async def upload_template():
"""A .yaml file read into a new editor. Nothing is saved until Save."""
files = await request.files
upload = files.get("file")
if upload is None or not upload.filename:
return await _editor(problems=["Choose a .yaml file to upload."], status=422)
raw = upload.read(UPLOAD_LIMIT + 1)
if len(raw) > UPLOAD_LIMIT:
return await _editor(problems=[f"A template is at most {UPLOAD_LIMIT // 1024} KB."], status=422)
try:
text = raw.decode("utf-8-sig").replace("\r\n", "\n")
except UnicodeDecodeError:
return await _editor(problems=[f"{upload.filename} isn't a text file (UTF-8)."], status=422)
return await _editor(text=text, note=f"Read from {upload.filename}. Nothing is saved until you press Save.")
@prometheus_bp.get("/templates/<key>")
@require_role(UserRole.viewer)
async def template_detail(key: str):
async with current_app.db_sessionmaker() as db:
entry = await _entry_or_404(db, key)
return await _editor(entry, problems=entry.problems)
async def _custom_or_404(db, template_id: str) -> PrometheusWidgetTemplate:
row = await db.get(PrometheusWidgetTemplate, template_id)
if row is None:
abort(404)
return row
@prometheus_bp.post("/templates/custom:<template_id>/edit")
@require_role(UserRole.admin)
async def save_template(template_id: str):
form = await request.form
text = _typed_text(form)
async with current_app.db_sessionmaker() as db:
async with db.begin():
row = await _custom_or_404(db, template_id)
template, problems = await _checked(db, text, other_than=row.id)
if problems:
entry = library.custom_entry(row)
return await _editor(entry, text=text, problems=problems, status=422, saving=True)
old_name = row.name
row.name, row.text = template.name, text
row.updated_at = datetime.now(timezone.utc)
detail = {"name": template.name, "version": template.version}
if old_name != template.name:
detail["old_name"] = old_name
await log_audit(current_app, *_actor(), "prometheus.template_updated", entity_type="prometheus_template",
entity_id=template_id, detail=detail)
return redirect(f"/plugins/prometheus/templates/custom:{template_id}")
@prometheus_bp.post("/templates/custom:<template_id>/delete")
@require_role(UserRole.admin)
async def delete_template(template_id: str):
async with current_app.db_sessionmaker() as db:
async with db.begin():
row = await _custom_or_404(db, template_id)
name = row.name
await db.delete(row)
await log_audit(current_app, *_actor(), "prometheus.template_deleted", entity_type="prometheus_template",
entity_id=template_id, detail={"name": name})
return redirect("/plugins/prometheus/templates/")
@prometheus_bp.get("/templates/<key>/export")
@require_role(UserRole.viewer)
async def export_template(key: str):
"""The template's YAML as a file, to share or keep."""
async with current_app.db_sessionmaker() as db:
entry = await _entry_or_404(db, key)
return Response(entry.text, mimetype="application/yaml", headers={
"Content-Disposition": f'attachment; filename="{library.file_name(entry.name)}"'})
@prometheus_bp.post("/templates/preview")
@require_role(UserRole.viewer)
async def preview_template():
"""The editor's live preview: what's typed, checked, and drawn over a
target's stored values as the widget would draw it. Saves nothing."""
from .template_format import TemplateError, check
form = await request.form
text = _typed_text(form)
now = datetime.now(timezone.utc)
try:
template = check(text) if text.strip() else None
except TemplateError as exc:
return await render_template("prometheus/_preview.html", problems=[str(p) for p in exc.problems],
view=None, target=None)
if template is None:
return await render_template("prometheus/_preview.html", problems=[], view=None, target=None,
empty=True)
async with current_app.db_sessionmaker() as db:
target = await db.get(PrometheusTarget, form.get("target") or "") if form.get("target") else None
if target is None:
return await render_template("prometheus/_preview.html", problems=[], view=None, target=None,
template=template)
shown = await draw(db, template, target, now)
return await render_template("prometheus/_preview.html", problems=[], view=shown, template=template,
target=view_target(target, now), spark_hours=SPARK_HOURS)