"""The Prometheus pages. Every GET reads Steward's own records: the target rows the editor writes and the scraper annotates, and the series it stores. Only Test, an admin's explicit action, asks an endpoint anything (System 178's ruling).""" from __future__ import annotations import json from datetime import datetime, timedelta, timezone from quart import Blueprint, abort, current_app, redirect, render_template, request, session from sqlalchemy import delete, func, select, update from steward.auth.middleware import require_role from steward.core.audit import log_audit from steward.core.crypto import decrypt_secret from steward.core.formatting import metric_value from steward.core.metric_series import latest_values, points from steward.models.metrics import PluginMetric, PluginMetricLatest from steward.models.users import UserRole from .models import PrometheusTarget from .scraper import ScrapeError, scrape from .service import (OWN_METRICS, SOURCE, FormError, form_values, map_families, parse_form, patterns, target_state) prometheus_bp = Blueprint("prometheus", __name__, template_folder="templates") _REACH_ORDER = {"down": 0, "stale": 1, "pending": 2, "up": 3, "off": 4} def _actor() -> tuple[str | None, str]: return session.get("user_id"), session.get("username", "") async def _all_targets(db) -> list[PrometheusTarget]: return list((await db.execute(select(PrometheusTarget).order_by(PrometheusTarget.name))).scalars().all()) def view(target: PrometheusTarget, own: dict, now: datetime) -> dict: """What a page says about a target: its state in words and the numbers from its newest scrape.""" state = target_state(target, now) return {"id": target.id, "name": target.name, "url": target.url, "reach": state.reach, "says": state.says, "scraped_at": target.last_scraped_at, "ok_at": target.last_ok_at, "interval": target.interval_seconds, "kept": target.last_kept, "dropped": target.last_dropped, "capped": target.last_capped, "cap": target.series_cap, "keep": list(target.keep or []), "drop": list(target.drop or []), "scrape_ms": own.get("scrape_ms")} def shown_value(metric: str, value: float | None) -> dict: """A series' value in its unit, from the Prometheus naming conventions. A *_timestamp_seconds is a moment: it comes back as `moment` for the page to show as "3m ago" in the reader's zone; 0 there means "never".""" if value is None: return {"text": "—", "moment": None} if metric.endswith("_timestamp_seconds"): if value <= 0: return {"text": "never", "moment": None} return {"text": "", "moment": datetime.fromtimestamp(value, timezone.utc)} return {"text": metric_value(metric, value), "moment": None} @prometheus_bp.get("/") @require_role(UserRole.viewer) async def index(): async with current_app.db_sessionmaker() as db: targets = await _all_targets(db) own = {} if targets: rows = (await db.execute(select(PluginMetricLatest).where( PluginMetricLatest.source_module == SOURCE, PluginMetricLatest.resource_name.in_([t.name for t in targets]), PluginMetricLatest.metric_name == "scrape_ms"))).scalars() own = {r.resource_name: {"scrape_ms": r.value} for r in rows} now = datetime.now(timezone.utc) shown = [view(t, own.get(t.name, {}), now) for t in targets] shown.sort(key=lambda v: (_REACH_ORDER[v["reach"]], v["name"].lower())) return await render_template("prometheus/index.html", targets=shown) # ── The editor ──────────────────────────────────────────────────────────────── async def _render_form(target: PrometheusTarget | None, values: dict, error: str | None = None, status: int = 200): return await render_template("prometheus/form.html", target=target, v=values, error=error), status def _typed(form, target: PrometheusTarget | None = None) -> dict: """What was typed, to show the form again with an error. The token is not echoed back.""" return {"name": form.get("name", ""), "url": form.get("url", ""), "interval_seconds": form.get("interval_seconds", ""), "series_cap": form.get("series_cap", ""), "keep": form.get("keep", ""), "drop": form.get("drop", ""), "verify_tls": bool(form.get("verify_tls")), "enabled": bool(form.get("enabled")), "has_token": bool(target and target.token)} async def _name_taken(db, name: str, other_than: str | None = None) -> bool: q = select(PrometheusTarget.id).where(PrometheusTarget.name == name) if other_than: q = q.where(PrometheusTarget.id != other_than) return (await db.execute(q)).first() is not None async def _target_or_404(db, target_id: str) -> PrometheusTarget: target = await db.get(PrometheusTarget, target_id) if target is None: abort(404) return target @prometheus_bp.get("/targets/new") @require_role(UserRole.admin) async def new_target(): return await _render_form(None, form_values(None)) @prometheus_bp.post("/targets/new") @require_role(UserRole.admin) async def create_target(): form = await request.form try: values = parse_form(form) except FormError as exc: return await _render_form(None, _typed(form), str(exc), 422) async with current_app.db_sessionmaker() as db: async with db.begin(): if await _name_taken(db, values["name"]): return await _render_form(None, _typed(form), f"A target is already named “{values['name']}”.", 422) target = PrometheusTarget(**values) db.add(target) await db.flush() target_id = target.id await log_audit(current_app, *_actor(), "prometheus.target_created", entity_type="prometheus_target", entity_id=target_id, detail={"name": values["name"], "url": values["url"]}) return redirect(f"/plugins/prometheus/targets/{target_id}") @prometheus_bp.get("/targets//edit") @require_role(UserRole.admin) async def edit_target(target_id: str): async with current_app.db_sessionmaker() as db: target = await _target_or_404(db, target_id) return await _render_form(target, form_values(target)) @prometheus_bp.post("/targets//edit") @require_role(UserRole.admin) async def save_target(target_id: str): form = await request.form async with current_app.db_sessionmaker() as db: async with db.begin(): target = await _target_or_404(db, target_id) try: values = parse_form(form, target) except FormError as exc: return await _render_form(target, _typed(form, target), str(exc), 422) if await _name_taken(db, values["name"], other_than=target.id): return await _render_form(target, _typed(form, target), f"A target is already named “{values['name']}”.", 422) old_name = target.name for column, value in values.items(): setattr(target, column, value) # Where it is or what is kept may have changed: the last outcome no # longer describes it, so the page says "not scraped yet" until the next scrape. target.last_scraped_at = target.last_ok_at = target.last_error = None target.last_kept = target.last_dropped = target.last_capped = 0 if old_name != target.name: # Its history is stored under its name, and its labelled series # under ":…": move them with the rename. for model in (PluginMetric, PluginMetricLatest): await db.execute(update(model).where( model.source_module == SOURCE, model.resource_name == old_name ).values(resource_name=target.name)) # ":labels" keeps its ":labels" (substr is 1-based). await db.execute(update(model).where( model.source_module == SOURCE, model.resource_name.startswith(old_name + ":", autoescape=True), ).values(resource_name=target.name + func.substr(model.resource_name, len(old_name) + 1))) from . import scheduler scheduler.RATES.forget(target_id) detail = {"name": values["name"]} if old_name != values["name"]: detail["old_name"] = old_name await log_audit(current_app, *_actor(), "prometheus.target_updated", entity_type="prometheus_target", entity_id=target_id, detail=detail) return redirect(f"/plugins/prometheus/targets/{target_id}") @prometheus_bp.post("/targets//delete") @require_role(UserRole.admin) async def delete_target(target_id: str): async with current_app.db_sessionmaker() as db: async with db.begin(): target = await _target_or_404(db, target_id) name = target.name await db.delete(target) # Its newest values go with it, so nothing reads it as a target # that stopped answering. Its history ages out with retention. await db.execute(delete(PluginMetricLatest).where( PluginMetricLatest.source_module == SOURCE, (PluginMetricLatest.resource_name == name) | PluginMetricLatest.resource_name.startswith(name + ":", autoescape=True))) from . import scheduler scheduler.RATES.forget(target_id) await log_audit(current_app, *_actor(), "prometheus.target_deleted", entity_type="prometheus_target", entity_id=target_id, detail={"name": name}) return redirect("/plugins/prometheus/") TEST_FAMILIES_SHOWN = 40 @prometheus_bp.post("/targets/test") @require_role(UserRole.admin) async def test_target(): """Scrape the (unsaved) form's address once and say what would be stored. An admin's explicit action, so it may contact the endpoint.""" form = await request.form existing = None if form.get("target_id"): async with current_app.db_sessionmaker() as db: existing = await db.get(PrometheusTarget, form["target_id"]) url = (form.get("url") or "").strip() token = (form.get("token") or "").strip() if not token and existing is not None and existing.token and not form.get("clear_token"): token = decrypt_secret(existing.token, context=f"prometheus target {existing.name} token") found: dict = {} if not url.startswith(("http://", "https://")): found["error"] = "Enter the address first: http://… or https://…" else: try: families = await scrape(url, token, bool(form.get("verify_tls"))) except ScrapeError as exc: found["error"] = str(exc) else: try: cap = max(1, int(form.get("series_cap") or 1000)) except ValueError: cap = 1000 keep, drop = patterns(form.get("keep") or ""), patterns(form.get("drop") or "") mapped = map_families("t", families, keep, drop, cap) kept_names = {} for s in mapped.series: kept_names[s.metric] = kept_names.get(s.metric, 0) + 1 found = {"families": len(families), "kept": len(mapped.series), "dropped": mapped.dropped, "capped": mapped.capped, "names": sorted(kept_names.items())[:TEST_FAMILIES_SHOWN], "more": max(0, len(kept_names) - TEST_FAMILIES_SHOWN)} return await render_template("prometheus/_test.html", found=found, url=url) # ── The target page and one series ─────────────────────────────────────────── def group_series(target_name: str, latest: list) -> list[dict]: """The target's stored series grouped by metric, each with its rows: the labels that tell them apart, and the newest value.""" groups: dict[str, list] = {} for row in latest: if row.resource_name == target_name and row.metric_name in OWN_METRICS: continue labels = row.resource_name[len(target_name) + 1:] if row.resource_name != target_name else "" groups.setdefault(row.metric_name, []).append({ "labels": labels, "resource": row.resource_name, "at": row.recorded_at, **shown_value(row.metric_name, row.value)}) return [{"metric": m, "rows": sorted(rows, key=lambda r: r["labels"])} for m, rows in sorted(groups.items())] @prometheus_bp.get("/targets/") @require_role(UserRole.viewer) async def target_detail(target_id: str): q = (request.args.get("q") or "").strip() async with current_app.db_sessionmaker() as db: target = await _target_or_404(db, target_id) latest = await latest_values(db, SOURCE, target.name, include_children=True) own = {r.metric_name: r.value for r in latest if r.resource_name == target.name and r.metric_name in OWN_METRICS} groups = group_series(target.name, latest) total = sum(len(g["rows"]) for g in groups) if q: groups = [g for g in groups if q.lower() in g["metric"].lower()] now = datetime.now(timezone.utc) return await render_template("prometheus/target.html", target=view(target, own, now), groups=groups, total=total, q=q) RANGES = (1, 6, 24, 168) @prometheus_bp.get("/targets//series") @require_role(UserRole.viewer) async def series_detail(target_id: str): resource = request.args.get("resource") or "" metric = request.args.get("metric") or "" hours = request.args.get("hours", type=int, default=24) if hours not in RANGES: hours = 24 async with current_app.db_sessionmaker() as db: target = await _target_or_404(db, target_id) if resource != target.name and not resource.startswith(target.name + ":"): abort(404) now_row = await db.get(PluginMetricLatest, (SOURCE, resource, metric)) series = await points(db, SOURCE, resource, metric, datetime.now(timezone.utc) - timedelta(hours=hours)) labels = resource[len(target.name) + 1:] if resource != target.name else "" stamp_axis = metric.endswith("_timestamp_seconds") return await render_template( "prometheus/series.html", target=target, resource=resource, metric=metric, labels=labels, now=shown_value(metric, now_row.value if now_row else None), hours=hours, ranges=RANGES, points_json=json.dumps([(t.isoformat(), v) for t, v in series]), stamp_axis=stamp_axis)