feat(prometheus): the Prometheus dashboard widget, target + template (#5691)
#5691 "Generic Prometheus dashboard widget: pick a target and a template", step 3 of milestone 508 "Prometheus widget templates". - /plugins/prometheus/widget draws one target with one template in widget shape: the verdict (led by "isn't answering" when the target is down), then each block with problems first, figures beside a share bar or the last 24h of bars, and the healthy rest folding to "+N more healthy" past six. A missing target or template, or one that no longer checks, says so. - library.py: bundled templates from plugins/prometheus/widgets/*.yaml, the picker's target and template lists, and the default title. - widgets/fabled_archives.yaml: the first bundled template, against the series Archives reports (refined in #5693). - Dashboard: a widget setting may take `options_from` (read when the picker opens) with an `empty` sentence; `default_title`; a Details link may name settings ("/plugins/prometheus/targets/{target}"). Prometheus is grouped with the integrations. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -54,6 +54,14 @@ Renaming a target moves its stored history to the new name. Deleting one removes
|
||||
|
||||
---
|
||||
|
||||
## The dashboard widget
|
||||
|
||||
Edit a dashboard, open **Add widget**, and pick **Prometheus** under Integrations. Choose a **target** and a **template**; the widget is named after the template unless you give it a title.
|
||||
|
||||
A template says which of the target's series to show and how: backup states, figures with their units, times, a table, the version running. The widget leads with one line naming the worst problem, then each section with its problems first and the healthy rest after. Its **Details** link opens the target's page.
|
||||
|
||||
Steward ships with one template, **Fabled Archives**. A target whose series don't match its template says so in the widget rather than showing blanks.
|
||||
|
||||
## Alerting on a target
|
||||
|
||||
In **Alerts → New rule**, choose **Prometheus targets**, then the series (the resource) and the metric. Two rules worth having for any target:
|
||||
|
||||
@@ -12,6 +12,34 @@ def setup(app: "Quart") -> None:
|
||||
global _app
|
||||
_app = app
|
||||
from .models import PrometheusTarget # noqa: F401 — registers the table
|
||||
register_widgets()
|
||||
|
||||
|
||||
def register_widgets() -> None:
|
||||
"""One dashboard widget for every target and template (System 181's
|
||||
ruling). Its target and template lists are read when the widget picker
|
||||
opens, so a new target or template appears without a restart."""
|
||||
from steward.core.widgets import register_widget
|
||||
|
||||
from . import library
|
||||
register_widget({
|
||||
"key": "prometheus",
|
||||
"label": "Prometheus",
|
||||
"description": "One scrape target drawn by a widget template: Fabled Archives, or one you copy in",
|
||||
"hx_url": "/plugins/prometheus/widget",
|
||||
"detail_url": "/plugins/prometheus/targets/{target}",
|
||||
"plugin": "prometheus",
|
||||
"poll": True,
|
||||
"default_title": library.default_title,
|
||||
"params": [
|
||||
{"key": "target", "label": "Target", "type": "select", "default": "",
|
||||
"options_from": library.target_options,
|
||||
"empty": "No targets yet: add one under Prometheus first."},
|
||||
{"key": "template", "label": "Template", "type": "select", "default": "",
|
||||
"options_from": library.template_options,
|
||||
"empty": "No templates yet."},
|
||||
],
|
||||
})
|
||||
|
||||
|
||||
def get_scheduled_tasks() -> list:
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""The widget templates a Prometheus widget can draw with.
|
||||
|
||||
Bundled templates ship in plugins/prometheus/widgets/*.yaml, read once and
|
||||
read-only; their key is "bundled:<file name>". Templates an admin copies in
|
||||
join them with the template library (#5692). Steward fetches none from
|
||||
anywhere (System 181's ruling).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from .models import PrometheusTarget
|
||||
from .template_format import Problem, Template, TemplateError, check
|
||||
|
||||
BUNDLED = Path(__file__).parent / "widgets"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Entry:
|
||||
key: str
|
||||
name: str
|
||||
bundled: bool
|
||||
text: str
|
||||
template: Template | None = None
|
||||
problems: list[Problem] = field(default_factory=list)
|
||||
|
||||
|
||||
def _entry(key: str, fallback_name: str, text: str, bundled: bool) -> Entry:
|
||||
try:
|
||||
template = check(text)
|
||||
except TemplateError as exc:
|
||||
return Entry(key, fallback_name, bundled, text, None, exc.problems)
|
||||
return Entry(key, template.name, bundled, text, template)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def bundled() -> tuple[Entry, ...]:
|
||||
return tuple(_entry(f"bundled:{p.stem}", p.stem, p.read_text(encoding="utf-8"), True)
|
||||
for p in sorted(BUNDLED.glob("*.yaml")))
|
||||
|
||||
|
||||
async def entries(db) -> list[Entry]:
|
||||
"""Every template, by name."""
|
||||
return sorted(bundled(), key=lambda e: e.name.lower())
|
||||
|
||||
|
||||
async def find(db, key: str) -> Entry | None:
|
||||
return next((e for e in await entries(db) if e.key == key), None)
|
||||
|
||||
|
||||
# ── The widget's settings ────────────────────────────────────────────────────
|
||||
|
||||
async def target_options(db) -> list[dict]:
|
||||
rows = (await db.execute(select(PrometheusTarget.id, PrometheusTarget.name)
|
||||
.order_by(PrometheusTarget.name))).all()
|
||||
return [{"value": tid, "label": name} for tid, name in rows]
|
||||
|
||||
|
||||
async def template_options(db) -> list[dict]:
|
||||
return [{"value": e.key, "label": f"{e.name} (built in)" if e.bundled else e.name}
|
||||
for e in await entries(db) if e.template is not None]
|
||||
|
||||
|
||||
async def default_title(db, config: dict) -> str | None:
|
||||
"""A new widget is named after its template: "Fabled Archives"."""
|
||||
entry = await find(db, str(config.get("template") or ""))
|
||||
return entry.name if entry and entry.template else None
|
||||
@@ -308,3 +308,57 @@ async def series_detail(target_id: str):
|
||||
"prometheus/series.html", target=target, resource=resource, metric=metric, labels=labels,
|
||||
now=shown_value(metric, now_row.value if now_row else None), hours=hours, ranges=RANGES,
|
||||
points_json=json.dumps([(t.isoformat(), v) for t, v in series]), stamp_axis=stamp_axis)
|
||||
|
||||
|
||||
# ── The dashboard widget ─────────────────────────────────────────────────────
|
||||
|
||||
SPARK_HOURS = 24
|
||||
|
||||
|
||||
def spark_heights(values: list[float]) -> list[float]:
|
||||
"""Bar heights in % of the series' own peak over the window (System
|
||||
154's ruling on small history bars); a flat zero draws flat."""
|
||||
peak = max((abs(v) for v in values), default=0.0)
|
||||
return [round(100.0 * abs(v) / peak, 1) if peak else 0.0 for v in values]
|
||||
|
||||
|
||||
@prometheus_bp.get("/widget")
|
||||
@require_role(UserRole.viewer)
|
||||
async def widget():
|
||||
"""The Prometheus dashboard widget: one target drawn by one template
|
||||
(System 181's ruling), in widget shape. Reads stored values only."""
|
||||
from . import library
|
||||
from .template_view import render as draw
|
||||
|
||||
target_id = request.args.get("target") or ""
|
||||
key = request.args.get("template") or ""
|
||||
now = datetime.now(timezone.utc)
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
target = await db.get(PrometheusTarget, target_id) if target_id else None
|
||||
entry = await library.find(db, key) if key else None
|
||||
problem = ("This widget has no target; remove it and add it again with one." if not target_id
|
||||
else "The target this widget showed has been deleted." if target is None
|
||||
else "This widget has no template; remove it and add it again with one." if not key
|
||||
else "The template this widget used is gone." if entry is None
|
||||
else f"The template “{entry.name}” has a problem: {entry.problems[0]}"
|
||||
if entry.template is None else "")
|
||||
if problem:
|
||||
return await render_template("prometheus/widget.html", problem=problem, target=None,
|
||||
view=None)
|
||||
latest = await latest_values(db, SOURCE, target.name, include_children=True)
|
||||
view = draw(entry.template, target.name, latest, now)
|
||||
if view.wants:
|
||||
since = now - timedelta(hours=SPARK_HOURS)
|
||||
history = {}
|
||||
for resource, metric in view.wants:
|
||||
got = await points(db, SOURCE, resource, metric, since, target=SPARK_HOURS)
|
||||
history[(resource, metric)] = spark_heights([v for _, v in got])
|
||||
view = draw(entry.template, target.name, latest, now, history=history)
|
||||
return await render_template("prometheus/widget.html", problem="", view=view,
|
||||
target=view_target(target, now), spark_hours=SPARK_HOURS)
|
||||
|
||||
|
||||
def view_target(target: PrometheusTarget, now: datetime) -> dict:
|
||||
state = target_state(target, now)
|
||||
return {"id": target.id, "name": target.name, "reach": state.reach, "says": state.says,
|
||||
"ok_at": target.last_ok_at, "scraped_at": target.last_scraped_at}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
{# prometheus/widget.html — the Prometheus dashboard widget (#5691): one
|
||||
target drawn by one template (template_view.render). A verdict naming the
|
||||
worst thing, then each block: its problems first with why, figures in their
|
||||
units beside a bar or the last day's bars, and the healthy rest packed after,
|
||||
past REST rows as "+N more healthy". #}
|
||||
{% from "_macros.html" import status_dot %}
|
||||
{% set REST = 6 %}
|
||||
{% set dot = {"crit": "crit", "warn": "warn", "ok": "ok", "dim": "paused"} %}
|
||||
{% set tone = {"crit": "tone-crit", "warn": "tone-warn", "ok": "tone-ok", "dim": "tone-dim"} %}
|
||||
{% macro shown(v) -%}
|
||||
{%- if v.moment %}{{ v.moment|ago }}{% elif v.text %}{{ v.text }}{% else %}—{% endif -%}
|
||||
{%- endmacro %}
|
||||
{% if problem %}
|
||||
<div class="w-verdict tone-warn">{{ problem }}</div>
|
||||
{% else %}
|
||||
{% set down = target.reach in ("down", "stale") %}
|
||||
<div class="w-verdict {{ 'tone-crit' if target.reach == 'down' else tone[view.tone] }}">
|
||||
{%- if target.reach == "down" %}{{ target.name }} isn't answering{% elif down %}{{ target.name }} isn't being scraped{% endif %}
|
||||
{%- if down %}{% if target.ok_at %}; these values are from {{ target.ok_at|ago }}{% endif %}. {% endif -%}
|
||||
{{ view.verdict }}
|
||||
</div>
|
||||
{% for b in view.blocks %}
|
||||
<div class="w-section">{{ b.title }}</div>
|
||||
{% if b.note %}
|
||||
<div class="w-note tone-warn">{{ b.note }}</div>
|
||||
{% elif b.kind == "info" %}
|
||||
<div class="w-note">{% for r in b.rows %}{{ r.name }}{% if not loop.last %}; {% endif %}{% endfor %}</div>
|
||||
{% elif b.kind == "table" %}
|
||||
<table class="w-table">
|
||||
<thead><tr><th></th>{% for c in b.columns %}<th>{{ c }}</th>{% endfor %}</tr></thead>
|
||||
<tbody>
|
||||
{% for r in b.rows %}
|
||||
<tr><td class="w-name" title="{{ r.name }}">{{ r.name }}</td>{% for c in r.cells %}<td>{{ shown(c) }}</td>{% endfor %}</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% else %}
|
||||
{% set problems = b.rows | selectattr("tone", "in", ["crit", "warn"]) | list %}
|
||||
{% set rest = b.rows | rejectattr("tone", "in", ["crit", "warn"]) | list %}
|
||||
{% set listed = problems + rest[:REST] %}
|
||||
<table class="w-table">
|
||||
{% if b.columns %}<thead><tr><th></th><th>Now</th>{% for c in b.columns %}<th class="w-wide-only">{{ c }}</th>{% endfor %}</tr></thead>{% endif %}
|
||||
<tbody>
|
||||
{% for r in listed %}
|
||||
<tr>
|
||||
<td class="w-name" title="{{ r.name }}">
|
||||
{{- status_dot(dot[r.tone]) -}}
|
||||
{{ r.name }}
|
||||
{%- if r.why %}<span class="w-sub {{ tone[r.tone] }}">{{ r.why }}</span>{% endif %}
|
||||
</td>
|
||||
<td class="{{ tone[r.tone] if r.tone in ('crit', 'warn') }}">
|
||||
{%- if r.history %}<span class="w-bars w-wide-only" aria-hidden="true" title="The last {{ spark_hours }}h, scaled to its own peak">
|
||||
{%- for h in r.history %}<i style="height:{{ [h, 4]|max }}%"></i>{% endfor -%}
|
||||
</span>{% endif -%}
|
||||
{{ shown(r) }}
|
||||
{%- if r.bar is not none %}<div class="meter w-wide-only" title="{{ (r.bar * 100)|round|int }}%"><div class="meter-fill meter-{{ r.tone if r.tone in ('crit', 'warn') else 'ok' }}" style="width:{{ (r.bar * 100)|round(1) }}%"></div></div>{% endif -%}
|
||||
</td>
|
||||
{% for c in r.cells %}<td class="w-wide-only">{{ shown(c) }}</td>{% endfor %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% if rest|length > REST %}<div class="w-more">+{{ rest|length - REST }} more healthy</div>{% endif %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
<div class="w-note">
|
||||
{%- if session.user_id %}<a href="/plugins/prometheus/targets/{{ target.id }}">{{ target.name }}</a>{% else %}{{ target.name }}{% endif %}
|
||||
{%- if target.scraped_at %}, scraped {{ target.scraped_at|ago }}{% endif %}.
|
||||
</div>
|
||||
{% endif %}
|
||||
@@ -0,0 +1,55 @@
|
||||
# Fabled Archives, from its /metrics on the LAN-only backup listener (:8081).
|
||||
# Written against what Archives reports; see docs/plugins/prometheus.md.
|
||||
name: Fabled Archives
|
||||
description: Backup targets, replication jobs, storage, integrity and the version running.
|
||||
version: 1
|
||||
expects: fabledarchives_
|
||||
blocks:
|
||||
- states:
|
||||
title: Backup targets
|
||||
metric: fabledarchives_backup_target_state
|
||||
name: target
|
||||
healthy: [fresh]
|
||||
quiet: [disabled, waiting]
|
||||
warning: [late]
|
||||
columns:
|
||||
- {label: Last backup, metric: fabledarchives_backup_target_last_backup_timestamp_seconds}
|
||||
- {label: Stored, metric: fabledarchives_backup_target_stored_bytes}
|
||||
- states:
|
||||
title: Replication
|
||||
metric: fabledarchives_replication_job_state
|
||||
name: job
|
||||
healthy: [ok, running]
|
||||
warning: [late, warning, never]
|
||||
columns:
|
||||
- {label: Next run, metric: fabledarchives_replication_job_next_run_timestamp_seconds}
|
||||
- values:
|
||||
title: Storage and integrity
|
||||
items:
|
||||
- label: Disk free
|
||||
metric: fabledarchives_data_disk_free_bytes
|
||||
of: fabledarchives_data_disk_size_bytes
|
||||
warn_below: 10%
|
||||
crit_below: 5%
|
||||
sparkline: true
|
||||
- label: Files checked
|
||||
metric: fabledarchives_files_checked
|
||||
of: fabledarchives_files
|
||||
unit: count
|
||||
- label: Malware held
|
||||
metric: fabledarchives_malware_held_files
|
||||
unit: count
|
||||
warn_above: 0
|
||||
- label: Notification channels failing
|
||||
metric: fabledarchives_notification_channels_failing
|
||||
unit: count
|
||||
warn_above: 0
|
||||
- table:
|
||||
title: Stored per person
|
||||
metric: fabledarchives_person_stored_bytes
|
||||
rows: person
|
||||
columns: kind
|
||||
- info:
|
||||
title: Running
|
||||
metric: fabledarchives_build_info
|
||||
labels: [version, channel, commit]
|
||||
@@ -6,7 +6,9 @@
|
||||
# "params" is a list of configurable parameters for the widget. Each param:
|
||||
# key - form field name (stored in config_json)
|
||||
# label - human-readable label
|
||||
# type - "select" (only type currently supported)
|
||||
# type - "select", or "host" for a live list of hosts
|
||||
# options_from - async (db) -> options, read when the picker opens (a plugin's
|
||||
# targets, its templates); "empty" is said when it returns none
|
||||
# default - default value (int or str determines how it's parsed from the form)
|
||||
# options - list of {value, label} dicts
|
||||
#
|
||||
@@ -353,9 +355,9 @@ def recommended_layout(plugins_cfg: dict) -> list[dict]:
|
||||
|
||||
# Group each widget for the picker (Core monitors / Monitoring capabilities /
|
||||
# Integrations), mirroring the Settings → Plugins taxonomy. Capability plugins
|
||||
# are host/monitoring facets; traefik/unifi are discrete vendor integrations.
|
||||
# are host/monitoring facets; traefik/unifi/prometheus are discrete integrations.
|
||||
_CORE_WIDGETS = {"monitors", "status_overview", "alerts", "hosts_overview"}
|
||||
_INTEGRATION_PLUGINS = {"traefik", "unifi"}
|
||||
_INTEGRATION_PLUGINS = {"traefik", "unifi", "prometheus"}
|
||||
GROUP_LABELS = {
|
||||
"core": "Core monitors",
|
||||
"capability": "Monitoring capabilities",
|
||||
|
||||
@@ -3,7 +3,7 @@ import hashlib
|
||||
import json
|
||||
import secrets
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from urllib.parse import urlencode
|
||||
from urllib.parse import quote, urlencode
|
||||
from quart import Blueprint, render_template, current_app, abort, redirect, request, session
|
||||
from sqlalchemy import select, func, update
|
||||
from steward.auth.middleware import public_route, require_role, current_user_id
|
||||
@@ -127,13 +127,41 @@ def _resolve_widgets(widgets: list[DashboardWidget]) -> list[dict]:
|
||||
hx_url = defn["hx_url"] + "?" + urlencode(url_params)
|
||||
out.append({
|
||||
"db": w,
|
||||
"defn": defn,
|
||||
"defn": {**defn, "detail_url": detail_url(defn, config)},
|
||||
"hx_url": hx_url,
|
||||
"title": w.title or defn["label"],
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def detail_url(defn: dict, config: dict) -> str:
|
||||
"""The widget's Details link. A plugin widget may name its settings in
|
||||
it, "/plugins/prometheus/targets/{target}"; when one isn't set, the link
|
||||
goes to the section above, "/plugins/prometheus/"."""
|
||||
url = defn["detail_url"]
|
||||
if "{" not in url:
|
||||
return url
|
||||
try:
|
||||
return url.format(**{k: quote(str(v), safe="") for k, v in config.items()})
|
||||
except (KeyError, IndexError, ValueError):
|
||||
return url[:url.index("{")].rsplit("/", 2)[0] + "/"
|
||||
|
||||
|
||||
async def with_options(db, widget: dict) -> dict:
|
||||
"""A widget definition for the picker, its `options_from` settings
|
||||
filled in now: a plugin's targets, templates and the like, read when
|
||||
the picker opens rather than fixed in the registry."""
|
||||
params = widget.get("params", [])
|
||||
if not any(p.get("options_from") for p in params):
|
||||
return widget
|
||||
filled = []
|
||||
for p in params:
|
||||
if p.get("options_from"):
|
||||
p = {**p, "options": await p["options_from"](db)}
|
||||
filled.append(p)
|
||||
return {**widget, "params": filled}
|
||||
|
||||
|
||||
# ── Edit panel helpers ────────────────────────────────────────────────────────
|
||||
|
||||
# ── Root redirect ─────────────────────────────────────────────────────────────
|
||||
@@ -173,7 +201,7 @@ async def view(dash_id: int):
|
||||
# viewers/shared-readers don't, so skip the extra queries for them.
|
||||
if can_edit:
|
||||
plugins_cfg = current_app.config.get("PLUGINS", {})
|
||||
addable = get_available_widgets(plugins_cfg)
|
||||
addable = [await with_options(db, w) for w in get_available_widgets(plugins_cfg)]
|
||||
hosts = list((await db.execute(select(Host).order_by(Host.name))).scalars())
|
||||
else:
|
||||
addable, hosts = [], []
|
||||
@@ -339,6 +367,8 @@ async def add_widget(dash_id: int, widget_key: str):
|
||||
config[p["key"]] = val
|
||||
config_json = json.dumps(config) if config else None
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
if title is None and defn.get("default_title"):
|
||||
title = await defn["default_title"](db, config)
|
||||
async with db.begin():
|
||||
result = await db.execute(select(Dashboard).where(Dashboard.id == dash_id))
|
||||
dash = result.scalar_one_or_none()
|
||||
|
||||
@@ -45,8 +45,12 @@
|
||||
{% for p in w.params %}
|
||||
<div>
|
||||
<label style="font-size:0.74rem;color:var(--text-muted);display:block;margin-bottom:0.2rem;">{{ p.label }}</label>
|
||||
{% if p.type == "select" %}
|
||||
<select name="param_{{ p.key }}" class="input">
|
||||
{% if p.type == "select" and p.options_from and not p.options %}
|
||||
{# A list read when the picker opens (a plugin's targets, its templates) that is empty. #}
|
||||
<p class="field-hint" style="margin:0;">{{ p.empty or "Nothing to choose from yet." }}</p>
|
||||
{% elif p.type == "select" %}
|
||||
<select name="param_{{ p.key }}" class="input"{% if p.options_from %} required{% endif %}>
|
||||
{% if p.options_from and p.options|length > 1 %}<option value="">— choose —</option>{% endif %}
|
||||
{% for opt in p.options %}
|
||||
<option value="{{ opt.value }}" {% if opt.value == p.default %}selected{% endif %}>{{ opt.label }}</option>
|
||||
{% endfor %}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Integration: the Prometheus dashboard widget (#5691) draws a target's
|
||||
stored series with the bundled Fabled Archives template, the picker lists the
|
||||
targets and templates, and a widget whose target is gone says so.
|
||||
Requires STEWARD_DATABASE_URL."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
if not os.environ.get("STEWARD_DATABASE_URL"):
|
||||
pytest.skip("needs Postgres")
|
||||
from steward.app import create_app
|
||||
return create_app(testing=False)
|
||||
|
||||
|
||||
def _prom(sub):
|
||||
"""A submodule of the prometheus plugin as the app loaded it (see
|
||||
test_prometheus_scrape)."""
|
||||
from steward.core.plugin_manager import get_loaded_plugin_modules
|
||||
loaded = get_loaded_plugin_modules().get("prometheus")
|
||||
return importlib.import_module(f"{loaded.__name__ if loaded else 'plugins.prometheus'}.{sub}")
|
||||
|
||||
|
||||
def _series(name: str) -> list[tuple[str, str, float]]:
|
||||
"""(resource, metric, value) as a scrape of Archives stores them."""
|
||||
rows = [(name, "fabledarchives_data_disk_free_bytes", 17.8e12),
|
||||
(name, "fabledarchives_data_disk_size_bytes", 46e12),
|
||||
(name, "fabledarchives_files", 94116.0), (name, "fabledarchives_files_checked", 93310.0),
|
||||
(name, "fabledarchives_malware_held_files", 2.0),
|
||||
(name, "fabledarchives_notification_channels_failing", 0.0),
|
||||
(f"{name}:version=2026.10.12.1717,channel=dev,commit=3605d0a", "fabledarchives_build_info", 1.0),
|
||||
(f"{name}:person=Bryan,person_id=9,kind=files", "fabledarchives_person_stored_bytes", 4.7e11)]
|
||||
for state in ("disabled", "fresh", "late", "missing", "waiting"):
|
||||
rows.append((f"{name}:target=Steward,target_id=1,state={state}", "fabledarchives_backup_target_state",
|
||||
float(state == "late")))
|
||||
for state in ("ok", "failing", "late", "running"):
|
||||
rows.append((f"{name}:job=Nightly,job_id=5,state={state}", "fabledarchives_replication_job_state",
|
||||
float(state == "ok")))
|
||||
return rows
|
||||
|
||||
|
||||
def test_the_widget_draws_a_target_with_a_template_and_the_picker_lists_them(app):
|
||||
from steward.core.alerts import record_metrics
|
||||
from steward.core.widgets import WIDGET_REGISTRY
|
||||
from steward.dashboard.routes import with_options
|
||||
from steward.models.metrics import PluginMetric
|
||||
from steward.models.users import User, UserRole
|
||||
|
||||
PrometheusTarget = _prom("models").PrometheusTarget
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
name = f"arch-{tag}"
|
||||
|
||||
async def go():
|
||||
uid = str(uuid.uuid4())
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
s.add(User(id=uid, username=f"pw-{tag}", email=f"pw-{tag}@test", password_hash="x",
|
||||
role=UserRole.viewer, is_active=True))
|
||||
target = PrometheusTarget(name=name, url=f"http://{name}/metrics")
|
||||
s.add(target)
|
||||
await s.flush()
|
||||
target_id = target.id
|
||||
await record_metrics(s, [PluginMetric(source_module="prometheus", resource_name=r,
|
||||
metric_name=m, value=v) for r, m, v in _series(name)])
|
||||
async with app.db_sessionmaker() as db:
|
||||
picker = await with_options(db, WIDGET_REGISTRY["prometheus"])
|
||||
client = app.test_client()
|
||||
async with client.session_transaction() as sess:
|
||||
sess.update(user_id=uid, user_role="viewer", username=f"pw-{tag}", login_at=time.time(), tz="UTC")
|
||||
ok = await client.get(f"/plugins/prometheus/widget?target={target_id}&template=bundled:fabled_archives")
|
||||
page = " ".join((await ok.get_data(as_text=True)).split())
|
||||
gone = await client.get("/plugins/prometheus/widget?target=no-such-target&template=bundled:fabled_archives")
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await s.delete(await s.get(PrometheusTarget, target_id))
|
||||
return target_id, picker, ok.status_code, page, gone.status_code, await gone.get_data(as_text=True)
|
||||
|
||||
target_id, picker, status, page, gone_status, gone = asyncio.run(go())
|
||||
targets, templates = picker["params"]
|
||||
assert {"value": target_id, "label": name} in targets["options"]
|
||||
assert {"value": "bundled:fabled_archives", "label": "Fabled Archives (built in)"} in templates["options"]
|
||||
assert status == 200
|
||||
# The late backup target and the held malware are the problems, the late target first.
|
||||
assert "2 need a look. Backup targets: Steward is late." in page
|
||||
assert page.index("2 need a look") < page.index("Backup targets</div>") < page.index("Replication</div>")
|
||||
assert "16.2 TiB of 41.8 TiB" in page and "93,310 of 94,116" in page
|
||||
assert "version 2026.10.12.1717 · channel dev · commit 3605d0a" in page
|
||||
assert f'<a href="/plugins/prometheus/targets/{target_id}">{name}</a>' in page
|
||||
assert gone_status == 200 and "The target this widget showed has been deleted." in gone
|
||||
@@ -0,0 +1,122 @@
|
||||
"""The Prometheus dashboard widget (#5691): one target drawn by one template,
|
||||
in widget shape, and the dashboard pieces it needs (settings read when the
|
||||
picker opens, a Details link naming the target)."""
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
|
||||
from plugins.prometheus import library
|
||||
from plugins.prometheus.routes import spark_heights
|
||||
from plugins.prometheus.template_format import check
|
||||
from plugins.prometheus.template_view import render
|
||||
from steward.dashboard.routes import detail_url, with_options
|
||||
from tests._templates import render_plugin_template
|
||||
from tests.plugins.prometheus.test_templates import NOW, SERIES, TEMPLATE, T, latest, states
|
||||
from tests.test_style_tokens import hex_colours
|
||||
|
||||
|
||||
def test_every_bundled_template_is_a_template():
|
||||
entries = library.bundled()
|
||||
assert [e.key for e in entries] == ["bundled:fabled_archives"]
|
||||
for e in entries:
|
||||
assert e.template is not None, [str(p) for p in e.problems]
|
||||
assert entries[0].name == "Fabled Archives" and entries[0].template.expects == "fabledarchives_"
|
||||
|
||||
|
||||
def test_settings_read_when_the_picker_opens():
|
||||
async def targets(db):
|
||||
return [{"value": "t1", "label": "Archives"}]
|
||||
|
||||
async def none(db):
|
||||
return []
|
||||
|
||||
widget = {"key": "w", "params": [
|
||||
{"key": "target", "type": "select", "default": "", "options_from": targets},
|
||||
{"key": "template", "type": "select", "default": "", "options_from": none, "empty": "No templates yet."},
|
||||
{"key": "hours", "type": "select", "default": 6, "options": [{"value": 6, "label": "6h"}]}]}
|
||||
filled = asyncio.run(with_options(None, widget))
|
||||
assert filled["params"][0]["options"] == [{"value": "t1", "label": "Archives"}]
|
||||
assert filled["params"][1]["options"] == []
|
||||
assert filled["params"][2] is widget["params"][2]
|
||||
assert "options" not in widget["params"][0] # the registry isn't changed
|
||||
plain = {"key": "x", "params": []}
|
||||
assert asyncio.run(with_options(None, plain)) is plain
|
||||
|
||||
|
||||
def test_details_names_the_target_or_falls_back_to_the_section():
|
||||
defn = {"detail_url": "/plugins/prometheus/targets/{target}"}
|
||||
assert detail_url(defn, {"target": "a b/c", "template": "x"}) == "/plugins/prometheus/targets/a%20b%2Fc"
|
||||
assert detail_url(defn, {}) == "/plugins/prometheus/"
|
||||
assert detail_url({"detail_url": "/monitors/"}, {"limit": 5}) == "/monitors/"
|
||||
|
||||
|
||||
def test_bars_scale_to_their_own_peak():
|
||||
assert spark_heights([1.0, 2.0, 4.0]) == [25.0, 50.0, 100.0]
|
||||
assert spark_heights([0.0, 0.0]) == [0.0, 0.0] and spark_heights([]) == []
|
||||
|
||||
|
||||
def _target(reach="up", ok_minutes=1):
|
||||
at = NOW - timedelta(minutes=ok_minutes)
|
||||
return {"id": "t1", "name": T, "reach": reach, "says": "", "ok_at": at, "scraped_at": NOW}
|
||||
|
||||
|
||||
async def _page(app, user=True, **ctx):
|
||||
from quart import session
|
||||
async with app.test_request_context("/plugins/prometheus/widget"):
|
||||
if user:
|
||||
session["user_id"] = "u1"
|
||||
session["user_role"] = "viewer"
|
||||
ctx.setdefault("spark_hours", 24)
|
||||
page = await render_plugin_template(app, "prometheus", "prometheus/widget.html", **ctx)
|
||||
return " ".join(page.split())
|
||||
|
||||
|
||||
async def test_the_widget_leads_with_the_verdict_then_problems_first(app):
|
||||
view = render(check(TEMPLATE), T, latest(SERIES), NOW,
|
||||
history={(T, "fa_disk_free_bytes"): [50.0, 100.0]})
|
||||
page = await _page(app, problem="", view=view, target=_target())
|
||||
assert page.index("3 need a look. Backup targets: Photos is late.") < page.index("Backup targets</div>")
|
||||
assert 'class="w-verdict tone-warn"' in page
|
||||
# Photos (late) before Steward (fresh) before Old (disabled, quiet).
|
||||
targets = page[page.index("Backup targets</div>"):page.index("Storage</div>")]
|
||||
assert targets.index("Photos") < targets.index("Steward") < targets.index("Old")
|
||||
assert '<span class="w-sub tone-warn">is late</span>' in page
|
||||
assert "<th>Now</th>" in page and '<th class="w-wide-only">Last backup</th>' in page
|
||||
assert "16.2 TiB of 41.8 TiB" in page and 'class="meter-fill meter-ok" style="width:38.7%"' in page
|
||||
assert '<span class="w-bars w-wide-only"' in page and 'style="height:50.0%"' in page
|
||||
assert "<th>files</th><th>trash</th>" in page and "437.7 GiB" in page
|
||||
assert "version 2026.10.12.1717 · commit 3605d0a" in page
|
||||
assert '<a href="/plugins/prometheus/targets/t1">Archives</a>' in page
|
||||
assert not hex_colours(page), hex_colours(page)
|
||||
|
||||
|
||||
async def test_the_healthy_rest_folds_after_six(app):
|
||||
many = {**SERIES}
|
||||
for i in range(9):
|
||||
many.update(states(f"target=T{i},target_id=x{i}", "fresh", ["fresh", "late"]))
|
||||
view = render(check(TEMPLATE), T, latest(many), NOW)
|
||||
page = await _page(app, problem="", view=view, target=_target())
|
||||
# 1 late, 10 fresh, 1 quiet in Backup targets: the late one, six of the eleven others.
|
||||
assert "+5 more healthy" in page
|
||||
|
||||
|
||||
async def test_a_target_not_answering_says_so_first(app):
|
||||
view = render(check(TEMPLATE), T, latest(SERIES), NOW)
|
||||
page = await _page(app, problem="", view=view, target=_target("down", ok_minutes=90))
|
||||
assert '<div class="w-verdict tone-crit">Archives isn\'t answering; these values are from' in page
|
||||
assert page.index("isn't answering") < page.index("3 need a look")
|
||||
|
||||
|
||||
async def test_a_widget_that_cant_draw_says_why(app):
|
||||
page = await _page(app, problem="The target this widget showed has been deleted.", view=None, target=None)
|
||||
assert '<div class="w-verdict tone-warn">The target this widget showed has been deleted.</div>' in page
|
||||
|
||||
|
||||
async def test_a_shared_dashboard_gets_no_links(app):
|
||||
view = render(check(TEMPLATE), T, latest(SERIES), NOW)
|
||||
page = await _page(app, user=False, problem="", view=view, target=_target())
|
||||
assert "/plugins/prometheus/targets/t1" not in page and "Archives, scraped" in page
|
||||
|
||||
|
||||
def test_the_widget_names_itself_after_its_template():
|
||||
assert asyncio.run(library.default_title(None, {"template": "bundled:fabled_archives"})) == "Fabled Archives"
|
||||
assert asyncio.run(library.default_title(None, {"template": "bundled:nope"})) is None
|
||||
Reference in New Issue
Block a user