Files
FabledSteward/plugins/prometheus/routes.py
T
bvandeusenandClaude Opus 5.5 f7a856cb90
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 1m37s
CI / integration (push) Failing after 1m54s
CI / publish (push) Skipped
feat(homeassistant): it shows system problems and installs updates on a weekly slot, and leaves device readings alone (#6748, #6749, #6747)
The connection's job is keeping Home Assistant running. Each poll now reads open
repairs, Supervisor health, disk, recovery mode and add-ons that should be running
over its websocket, for the page, the overview and the problems_open, disk_free_pct
and update_failed metrics. That needs a websocket client, so websockets joins the
dependencies.

A weekly slot, set on the page or through MCP and off until then, installs add-ons,
then Core, then the OS, each after its own confirmed backup, ending at the first
failure. The slot arithmetic moves from the host agent's cadence to steward.core.weekly
so both share it.

The device scrape goes: the page's Metrics section and a target's token_from from
4c9ed02 are removed, and a migration deletes targets using it, as they'd otherwise ask
with no token. A refused token still stops a scrape.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-01 14:16:42 -05:00

606 lines
28 KiB
Python

"""The Prometheus pages. Every GET reads Steward's own records: the target
rows the editor writes and the scraper annotates, and the series it stores.
Only Test, an admin's explicit action, asks an endpoint anything (System
178's ruling)."""
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from quart import Blueprint, Response, abort, current_app, redirect, render_template, request, session
from sqlalchemy import delete, func, select, update
from steward.auth.middleware import require_role
from steward.core.audit import log_audit
from steward.core.crypto import decrypt_secret
from steward.core.metric_series import latest_values, points
from steward.models.metrics import PluginMetric, PluginMetricLatest
from steward.models.users import UserRole
from . import library
from .models import PrometheusTarget, PrometheusWidgetTemplate
from .scraper import ScrapeError, scrape
from .service import (OWN_METRICS, SOURCE, FormError, fold_states, form_values, is_moment, map_families,
parse_form, patterns, shown_value, target_state)
prometheus_bp = Blueprint("prometheus", __name__, template_folder="templates")
_REACH_ORDER = {"down": 0, "stale": 1, "pending": 2, "up": 3, "paused": 4}
def _actor() -> tuple[str | None, str]:
return session.get("user_id"), session.get("username", "")
async def _all_targets(db) -> list[PrometheusTarget]:
return list((await db.execute(select(PrometheusTarget).order_by(PrometheusTarget.name))).scalars().all())
def view(target: PrometheusTarget, own: dict, now: datetime) -> dict:
"""What a page says about a target: its state in words and the numbers
from its newest scrape."""
state = target_state(target, now)
return {"id": target.id, "name": target.name, "url": target.url, "enabled": target.enabled,
"reach": state.reach,
"says": state.says, "scraped_at": target.last_scraped_at, "ok_at": target.last_ok_at,
"interval": target.interval_seconds, "kept": target.last_kept,
"dropped": target.last_dropped, "capped": target.last_capped, "cap": target.series_cap,
"keep": list(target.keep or []), "drop": list(target.drop or []),
"scrape_ms": own.get("scrape_ms")}
@prometheus_bp.get("/")
@require_role(UserRole.viewer)
async def index():
async with current_app.db_sessionmaker() as db:
targets = await _all_targets(db)
own = {}
if targets:
rows = (await db.execute(select(PluginMetricLatest).where(
PluginMetricLatest.source_module == SOURCE,
PluginMetricLatest.resource_name.in_([t.name for t in targets]),
PluginMetricLatest.metric_name == "scrape_ms"))).scalars()
own = {r.resource_name: {"scrape_ms": r.value} for r in rows}
now = datetime.now(timezone.utc)
shown = [view(t, own.get(t.name, {}), now) for t in targets]
shown.sort(key=lambda v: (_REACH_ORDER[v["reach"]], v["name"].lower()))
return await render_template("prometheus/index.html", targets=shown)
# ── The editor ────────────────────────────────────────────────────────────────
async def _render_form(target: PrometheusTarget | None, values: dict, error: str | None = None,
status: int = 200):
return await render_template("prometheus/form.html", target=target, v=values, error=error), status
def _typed(form, target: PrometheusTarget | None = None) -> dict:
"""What was typed, to show the form again with an error. The token is not
echoed back."""
return {"name": form.get("name", ""), "url": form.get("url", ""),
"interval_seconds": form.get("interval_seconds", ""), "series_cap": form.get("series_cap", ""),
"keep": form.get("keep", ""), "drop": form.get("drop", ""),
"verify_tls": bool(form.get("verify_tls")), "has_token": bool(target and target.token)}
async def _name_taken(db, name: str, other_than: str | None = None) -> bool:
q = select(PrometheusTarget.id).where(PrometheusTarget.name == name)
if other_than:
q = q.where(PrometheusTarget.id != other_than)
return (await db.execute(q)).first() is not None
async def _target_or_404(db, target_id: str) -> PrometheusTarget:
target = await db.get(PrometheusTarget, target_id)
if target is None:
abort(404)
return target
@prometheus_bp.get("/targets/new")
@require_role(UserRole.admin)
async def new_target():
return await _render_form(None, form_values(None))
@prometheus_bp.post("/targets/new")
@require_role(UserRole.admin)
async def create_target():
form = await request.form
try:
values = parse_form(form)
except FormError as exc:
return await _render_form(None, _typed(form), str(exc), 422)
async with current_app.db_sessionmaker() as db:
async with db.begin():
if await _name_taken(db, values["name"]):
return await _render_form(None, _typed(form),
f"A target is already named “{values['name']}”.", 422)
target = PrometheusTarget(**values)
db.add(target)
await db.flush()
target_id = target.id
await log_audit(current_app, *_actor(), "prometheus.target_created", entity_type="prometheus_target",
entity_id=target_id, detail={"name": values["name"], "url": values["url"]})
return redirect(f"/plugins/prometheus/targets/{target_id}")
@prometheus_bp.get("/targets/<target_id>/edit")
@require_role(UserRole.admin)
async def edit_target(target_id: str):
async with current_app.db_sessionmaker() as db:
target = await _target_or_404(db, target_id)
return await _render_form(target, form_values(target))
@prometheus_bp.post("/targets/<target_id>/edit")
@require_role(UserRole.admin)
async def save_target(target_id: str):
form = await request.form
async with current_app.db_sessionmaker() as db:
async with db.begin():
target = await _target_or_404(db, target_id)
try:
values = parse_form(form, target)
except FormError as exc:
return await _render_form(target, _typed(form, target), str(exc), 422)
if await _name_taken(db, values["name"], other_than=target.id):
return await _render_form(target, _typed(form, target),
f"A target is already named “{values['name']}”.", 422)
old_name = target.name
for column, value in values.items():
setattr(target, column, value)
# Where it is or what is kept may have changed: the last outcome no
# longer describes it, so the page says "not scraped yet" until the next scrape.
target.last_scraped_at = target.last_ok_at = target.last_error = None
target.refused = False
target.last_kept = target.last_dropped = target.last_capped = 0
if old_name != target.name:
# Its history is stored under its name, and its labelled series
# under "<name>:…": move them with the rename.
for model in (PluginMetric, PluginMetricLatest):
await db.execute(update(model).where(
model.source_module == SOURCE, model.resource_name == old_name
).values(resource_name=target.name))
# "<old>:labels" keeps its ":labels" (substr is 1-based).
await db.execute(update(model).where(
model.source_module == SOURCE,
model.resource_name.startswith(old_name + ":", autoescape=True),
).values(resource_name=target.name + func.substr(model.resource_name, len(old_name) + 1)))
from . import scheduler
scheduler.RATES.forget(target_id)
detail = {"name": values["name"]}
if old_name != values["name"]:
detail["old_name"] = old_name
await log_audit(current_app, *_actor(), "prometheus.target_updated", entity_type="prometheus_target",
entity_id=target_id, detail=detail)
return redirect(f"/plugins/prometheus/targets/{target_id}")
@prometheus_bp.post("/targets/<target_id>/toggle")
@require_role(UserRole.admin)
async def toggle_target(target_id: str):
"""Pause a target, or resume a paused one, from its page or the list."""
async with current_app.db_sessionmaker() as db:
async with db.begin():
target = await _target_or_404(db, target_id)
target.enabled = not target.enabled
if target.enabled:
# Resumed: it reads "not scraped yet" until the next scrape,
# rather than stale from before the pause.
target.last_scraped_at = target.last_error = None
target.refused = False
enabled, name = target.enabled, target.name
# A rate needs two readings in a row; one from before the pause isn't one.
from . import scheduler
scheduler.RATES.forget(target_id)
await log_audit(current_app, *_actor(), "prometheus.target_resumed" if enabled else "prometheus.target_paused",
entity_type="prometheus_target", entity_id=target_id, detail={"name": name})
return redirect(request.referrer or f"/plugins/prometheus/targets/{target_id}")
@prometheus_bp.post("/targets/<target_id>/delete")
@require_role(UserRole.admin)
async def delete_target(target_id: str):
async with current_app.db_sessionmaker() as db:
async with db.begin():
target = await _target_or_404(db, target_id)
name = target.name
await db.delete(target)
# Its newest values go with it, so nothing reads it as a target
# that stopped answering. Its history ages out with retention.
await db.execute(delete(PluginMetricLatest).where(
PluginMetricLatest.source_module == SOURCE,
(PluginMetricLatest.resource_name == name)
| PluginMetricLatest.resource_name.startswith(name + ":", autoescape=True)))
from . import scheduler
scheduler.RATES.forget(target_id)
await log_audit(current_app, *_actor(), "prometheus.target_deleted", entity_type="prometheus_target",
entity_id=target_id, detail={"name": name})
return redirect("/plugins/prometheus/")
TEST_FAMILIES_SHOWN = 40
@prometheus_bp.post("/targets/test")
@require_role(UserRole.admin)
async def test_target():
"""Scrape the (unsaved) form's address once and say what would be stored.
An admin's explicit action, so it may contact the endpoint."""
form = await request.form
existing = None
if form.get("target_id"):
async with current_app.db_sessionmaker() as db:
existing = await db.get(PrometheusTarget, form["target_id"])
url = (form.get("url") or "").strip()
token = (form.get("token") or "").strip()
if not token and existing is not None and existing.token and not form.get("clear_token"):
token = decrypt_secret(existing.token, context=f"prometheus target {existing.name} token")
found: dict = {}
if not url.startswith(("http://", "https://")):
found["error"] = "Enter the address first: http://… or https://…"
else:
try:
families = await scrape(url, token, bool(form.get("verify_tls")))
except ScrapeError as exc:
found["error"] = str(exc)
else:
try:
cap = max(1, int(form.get("series_cap") or 1000))
except ValueError:
cap = 1000
keep, drop = patterns(form.get("keep") or ""), patterns(form.get("drop") or "")
mapped = map_families("t", families, keep, drop, cap)
kept_names = {}
for s in mapped.series:
kept_names[s.metric] = kept_names.get(s.metric, 0) + 1
found = {"families": len(families), "kept": len(mapped.series), "dropped": mapped.dropped,
"capped": mapped.capped,
"names": sorted(kept_names.items())[:TEST_FAMILIES_SHOWN],
"more": max(0, len(kept_names) - TEST_FAMILIES_SHOWN)}
return await render_template("prometheus/_test.html", found=found, url=url)
# ── The target page and one series ───────────────────────────────────────────
def group_series(target_name: str, latest: list) -> list[dict]:
"""The target's stored series grouped by metric, each with its rows:
the labels that tell them apart, and the newest value. A state set is
folded to one row per thing it describes, naming the current state
(`states` set on the group)."""
groups: dict[str, list] = {}
for row in latest:
if row.resource_name == target_name and row.metric_name in OWN_METRICS:
continue
labels = row.resource_name[len(target_name) + 1:] if row.resource_name != target_name else ""
groups.setdefault(row.metric_name, []).append({
"labels": labels, "resource": row.resource_name, "at": row.recorded_at,
"value": row.value, **shown_value(row.metric_name, row.value)})
out = []
for metric, rows in sorted(groups.items()):
folded = fold_states(metric, rows)
if folded is not None:
out.append({"metric": metric, "states": True, "rows": [
{"labels": f["labels"], "resource": f["resource"], "moment": None,
"text": f["state"] or "none", "states": f["states"]} for f in folded]})
else:
out.append({"metric": metric, "states": False,
"rows": sorted(rows, key=lambda r: r["labels"])})
return out
@prometheus_bp.get("/targets/<target_id>")
@require_role(UserRole.viewer)
async def target_detail(target_id: str):
q = (request.args.get("q") or "").strip()
async with current_app.db_sessionmaker() as db:
target = await _target_or_404(db, target_id)
latest = await latest_values(db, SOURCE, target.name, include_children=True)
own = {r.metric_name: r.value for r in latest if r.resource_name == target.name
and r.metric_name in OWN_METRICS}
groups = group_series(target.name, latest)
total = sum(len(g["rows"]) for g in groups)
if q:
groups = [g for g in groups if q.lower() in g["metric"].lower()]
now = datetime.now(timezone.utc)
return await render_template("prometheus/target.html", target=view(target, own, now), groups=groups,
total=total, q=q)
RANGES = (1, 6, 24, 168)
@prometheus_bp.get("/targets/<target_id>/series")
@require_role(UserRole.viewer)
async def series_detail(target_id: str):
resource = request.args.get("resource") or ""
metric = request.args.get("metric") or ""
hours = request.args.get("hours", type=int, default=24)
if hours not in RANGES:
hours = 24
async with current_app.db_sessionmaker() as db:
target = await _target_or_404(db, target_id)
if resource != target.name and not resource.startswith(target.name + ":"):
abort(404)
now_row = await db.get(PluginMetricLatest, (SOURCE, resource, metric))
series = await points(db, SOURCE, resource, metric,
datetime.now(timezone.utc) - timedelta(hours=hours))
labels = resource[len(target.name) + 1:] if resource != target.name else ""
stamp_axis = is_moment(metric)
return await render_template(
"prometheus/series.html", target=target, resource=resource, metric=metric, labels=labels,
now=shown_value(metric, now_row.value if now_row else None), hours=hours, ranges=RANGES,
points_json=json.dumps([(t.isoformat(), v) for t, v in series]), stamp_axis=stamp_axis)
# ── The dashboard widget ─────────────────────────────────────────────────────
SPARK_HOURS = 24
def spark_heights(values: list[float]) -> list[float]:
"""Bar heights in % of the series' own peak over the window (System
154's ruling on small history bars); a flat zero draws flat."""
peak = max((abs(v) for v in values), default=0.0)
return [round(100.0 * abs(v) / peak, 1) if peak else 0.0 for v in values]
async def draw(db, template, target: PrometheusTarget, now: datetime):
"""`template` drawn over `target`'s stored values, with the last day's bars
for the items that ask for them. The dashboard widget and the template
editor's preview both draw through here."""
from .template_view import render
latest = await latest_values(db, SOURCE, target.name, include_children=True)
shown = render(template, target.name, latest, now)
if shown.wants:
since = now - timedelta(hours=SPARK_HOURS)
history = {}
for resource, metric in shown.wants:
got = await points(db, SOURCE, resource, metric, since, target=SPARK_HOURS)
history[(resource, metric)] = spark_heights([v for _, v in got])
shown = render(template, target.name, latest, now, history=history)
return shown
@prometheus_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
"""The Prometheus dashboard widget: one target drawn by one template
(System 181's ruling), in widget shape. Reads stored values only."""
target_id = request.args.get("target") or ""
key = request.args.get("template") or ""
now = datetime.now(timezone.utc)
async with current_app.db_sessionmaker() as db:
target = await db.get(PrometheusTarget, target_id) if target_id else None
entry = await library.find(db, key) if key else None
problem = ("This widget has no target; remove it and add it again with one." if not target_id
else "The target this widget showed has been deleted." if target is None
else "This widget has no template; remove it and add it again with one." if not key
else "The template this widget used is gone." if entry is None
else f"The template “{entry.name}” has a problem: {entry.problems[0]}"
if entry.template is None else "")
if problem:
return await render_template("prometheus/widget.html", problem=problem, target=None,
view=None)
shown = await draw(db, entry.template, target, now)
return await render_template("prometheus/widget.html", problem="", view=shown,
target=view_target(target, now), spark_hours=SPARK_HOURS)
def view_target(target: PrometheusTarget, now: datetime) -> dict:
state = target_state(target, now)
return {"id": target.id, "name": target.name, "reach": state.reach, "says": state.says,
"ok_at": target.last_ok_at, "scraped_at": target.last_scraped_at}
# ── The template library (#5692) ─────────────────────────────────────────────
# Bundled templates are read-only: they can be looked at, exported and
# duplicated. An admin writes, pastes or uploads their own, edits and deletes
# them. Nothing is fetched from anywhere (System 181's ruling).
def _shown_entry(entry, used: int) -> dict:
return {"key": entry.key, "id": entry.id, "name": entry.name, "bundled": entry.bundled,
"version": entry.template.version if entry.template else None,
"description": entry.template.description if entry.template else "",
"problem": str(entry.problems[0]) if entry.problems else "",
"updated_at": entry.updated_at, "used": used}
@prometheus_bp.get("/templates/")
@require_role(UserRole.viewer)
async def templates_index():
async with current_app.db_sessionmaker() as db:
entries = await library.entries(db)
used = await library.used_by(db)
shown = [_shown_entry(e, used.get(e.key, 0)) for e in entries]
# One that no longer passes the checker first: its widgets say so too.
shown.sort(key=lambda t: (not t["problem"], t["name"].lower()))
return await render_template("prometheus/templates.html", templates=shown)
async def _editor(entry=None, text: str = "", problems=(), note: str = "", status: int = 200,
saving: bool = False):
"""The template page: an admin's editor, or a read-only view of a
bundled template (or of any template, to a viewer). Both preview it."""
async with current_app.db_sessionmaker() as db:
targets = await library.target_options(db)
used = (await library.used_by(db)).get(entry.key, 0) if entry is not None else 0
editable = session.get("user_role") == "admin" and (entry is None or not entry.bundled)
return await render_template(
"prometheus/template.html", entry=entry, title=entry.name if entry else "New template",
text=text if entry is None or text else entry.text,
problems=[str(p) for p in problems], note=note, targets=targets, editable=editable,
used=used, saving=saving), status
async def _entry_or_404(db, key: str):
entry = await library.find(db, key)
if entry is None:
abort(404)
return entry
@prometheus_bp.get("/templates/new")
@require_role(UserRole.admin)
async def new_template():
"""A blank editor, or a copy of another template (?from=<key>) under a
name nobody has."""
source = request.args.get("from") or ""
if not source:
return await _editor()
async with current_app.db_sessionmaker() as db:
entry = await _entry_or_404(db, source)
taken = {e.name for e in await library.entries(db)}
return await _editor(text=library.renamed(entry.text, library.copy_name(entry.name, taken)),
note=f"A copy of “{entry.name}”. Nothing is saved until you press Save.")
async def _checked(db, text: str, other_than: str | None = None):
"""The template `text` describes and its problems: the checker's, and a
name another template already has."""
from .template_format import Problem, TemplateError, check
try:
template = check(text)
except TemplateError as exc:
return None, exc.problems
if (await library.name_taken(db, template.name, other_than)
or any(e.name == template.name for e in library.bundled())):
line = next((i for i, ln in enumerate(text.splitlines(), 1) if ln.startswith("name:")), 0)
return None, [Problem(line, f"A template is already named “{template.name}”; give this one its own name.")]
return template, []
def _typed_text(form) -> str:
return (form.get("text") or "").replace("\r\n", "\n")
@prometheus_bp.post("/templates/new")
@require_role(UserRole.admin)
async def create_template():
form = await request.form
text = _typed_text(form)
async with current_app.db_sessionmaker() as db:
async with db.begin():
template, problems = await _checked(db, text)
if problems:
return await _editor(text=text, problems=problems, status=422, saving=True)
row = PrometheusWidgetTemplate(name=template.name, text=text)
db.add(row)
await db.flush()
row_id = row.id
await log_audit(current_app, *_actor(), "prometheus.template_created", entity_type="prometheus_template",
entity_id=row_id, detail={"name": template.name, "version": template.version})
return redirect(f"/plugins/prometheus/templates/custom:{row_id}")
UPLOAD_LIMIT = 64 * 1024
@prometheus_bp.post("/templates/upload")
@require_role(UserRole.admin)
async def upload_template():
"""A .yaml file read into a new editor. Nothing is saved until Save."""
files = await request.files
upload = files.get("file")
if upload is None or not upload.filename:
return await _editor(problems=["Choose a .yaml file to upload."], status=422)
raw = upload.read(UPLOAD_LIMIT + 1)
if len(raw) > UPLOAD_LIMIT:
return await _editor(problems=[f"A template is at most {UPLOAD_LIMIT // 1024} KB."], status=422)
try:
text = raw.decode("utf-8-sig").replace("\r\n", "\n")
except UnicodeDecodeError:
return await _editor(problems=[f"{upload.filename} isn't a text file (UTF-8)."], status=422)
return await _editor(text=text, note=f"Read from {upload.filename}. Nothing is saved until you press Save.")
@prometheus_bp.get("/templates/<key>")
@require_role(UserRole.viewer)
async def template_detail(key: str):
async with current_app.db_sessionmaker() as db:
entry = await _entry_or_404(db, key)
return await _editor(entry, problems=entry.problems)
async def _custom_or_404(db, template_id: str) -> PrometheusWidgetTemplate:
row = await db.get(PrometheusWidgetTemplate, template_id)
if row is None:
abort(404)
return row
@prometheus_bp.post("/templates/custom:<template_id>/edit")
@require_role(UserRole.admin)
async def save_template(template_id: str):
form = await request.form
text = _typed_text(form)
async with current_app.db_sessionmaker() as db:
async with db.begin():
row = await _custom_or_404(db, template_id)
template, problems = await _checked(db, text, other_than=row.id)
if problems:
entry = library.custom_entry(row)
return await _editor(entry, text=text, problems=problems, status=422, saving=True)
old_name = row.name
row.name, row.text = template.name, text
row.updated_at = datetime.now(timezone.utc)
detail = {"name": template.name, "version": template.version}
if old_name != template.name:
detail["old_name"] = old_name
await log_audit(current_app, *_actor(), "prometheus.template_updated", entity_type="prometheus_template",
entity_id=template_id, detail=detail)
return redirect(f"/plugins/prometheus/templates/custom:{template_id}")
@prometheus_bp.post("/templates/custom:<template_id>/delete")
@require_role(UserRole.admin)
async def delete_template(template_id: str):
async with current_app.db_sessionmaker() as db:
async with db.begin():
row = await _custom_or_404(db, template_id)
name = row.name
await db.delete(row)
await log_audit(current_app, *_actor(), "prometheus.template_deleted", entity_type="prometheus_template",
entity_id=template_id, detail={"name": name})
return redirect("/plugins/prometheus/templates/")
@prometheus_bp.get("/templates/<key>/export")
@require_role(UserRole.viewer)
async def export_template(key: str):
"""The template's YAML as a file, to share or keep."""
async with current_app.db_sessionmaker() as db:
entry = await _entry_or_404(db, key)
return Response(entry.text, mimetype="application/yaml", headers={
"Content-Disposition": f'attachment; filename="{library.file_name(entry.name)}"'})
@prometheus_bp.post("/templates/preview")
@require_role(UserRole.viewer)
async def preview_template():
"""The editor's live preview: what's typed, checked, and drawn over a
target's stored values as the widget would draw it. Saves nothing."""
from .template_format import TemplateError, check
form = await request.form
text = _typed_text(form)
now = datetime.now(timezone.utc)
try:
template = check(text) if text.strip() else None
except TemplateError as exc:
return await render_template("prometheus/_preview.html", problems=[str(p) for p in exc.problems],
view=None, target=None)
if template is None:
return await render_template("prometheus/_preview.html", problems=[], view=None, target=None,
empty=True)
async with current_app.db_sessionmaker() as db:
target = await db.get(PrometheusTarget, form.get("target") or "") if form.get("target") else None
if target is None:
return await render_template("prometheus/_preview.html", problems=[], view=None, target=None,
template=template)
shown = await draw(db, template, target, now)
return await render_template("prometheus/_preview.html", problems=[], view=shown, template=template,
target=view_target(target, now), spark_hours=SPARK_HOURS)