diff --git a/plugins/prometheus/routes.py b/plugins/prometheus/routes.py index adde9ee..67ee391 100644 --- a/plugins/prometheus/routes.py +++ b/plugins/prometheus/routes.py @@ -13,7 +13,6 @@ from sqlalchemy import delete, func, select, update from steward.auth.middleware import require_role from steward.core.audit import log_audit from steward.core.crypto import decrypt_secret -from steward.core.formatting import metric_value from steward.core.metric_series import latest_values, points from steward.models.metrics import PluginMetric, PluginMetricLatest from steward.models.users import UserRole @@ -21,7 +20,7 @@ from steward.models.users import UserRole from .models import PrometheusTarget from .scraper import ScrapeError, scrape from .service import (OWN_METRICS, SOURCE, FormError, fold_states, form_values, map_families, - parse_form, patterns, target_state) + parse_form, patterns, shown_value, target_state) prometheus_bp = Blueprint("prometheus", __name__, template_folder="templates") @@ -48,19 +47,6 @@ def view(target: PrometheusTarget, own: dict, now: datetime) -> dict: "scrape_ms": own.get("scrape_ms")} -def shown_value(metric: str, value: float | None) -> dict: - """A series' value in its unit, from the Prometheus naming conventions. - A *_timestamp_seconds is a moment: it comes back as `moment` for the page - to show as "3m ago" in the reader's zone; 0 there means "never".""" - if value is None: - return {"text": "—", "moment": None} - if metric.endswith("_timestamp_seconds"): - if value <= 0: - return {"text": "never", "moment": None} - return {"text": "", "moment": datetime.fromtimestamp(value, timezone.utc)} - return {"text": metric_value(metric, value), "moment": None} - - @prometheus_bp.get("/") @require_role(UserRole.viewer) async def index(): diff --git a/plugins/prometheus/service.py b/plugins/prometheus/service.py index afc2c79..5ffd774 100644 --- a/plugins/prometheus/service.py +++ b/plugins/prometheus/service.py @@ -5,11 +5,12 @@ from __future__ import annotations import math from dataclasses import dataclass, field -from datetime import datetime +from datetime import datetime, timezone from fnmatch import fnmatchcase from steward.core import prometheus from steward.core.crypto import encrypt_secret +from steward.core.formatting import metric_value from .models import PrometheusTarget @@ -121,6 +122,19 @@ def fold_states(metric: str, rows: list[dict]) -> list[dict] | None: return folded +def shown_value(metric: str, value: float | None) -> dict: + """A series' value in its unit, from the Prometheus naming conventions. + A *_timestamp_seconds is a moment: it comes back as `moment` for the page + to show as "3m ago" in the reader's zone; 0 there means "never".""" + if value is None: + return {"text": "—", "moment": None} + if metric.endswith("_timestamp_seconds"): + if value <= 0: + return {"text": "never", "moment": None} + return {"text": "", "moment": datetime.fromtimestamp(value, timezone.utc)} + return {"text": metric_value(metric, value), "moment": None} + + @dataclass class Series: resource: str diff --git a/plugins/prometheus/template_format.py b/plugins/prometheus/template_format.py new file mode 100644 index 0000000..b257c85 --- /dev/null +++ b/plugins/prometheus/template_format.py @@ -0,0 +1,451 @@ +"""Prometheus widget templates: the YAML a person writes, read and checked. + +A template says which of a target's series a dashboard widget shows and +how. It is data only: it names series and lays them out in blocks, and +Steward draws them (template_view.py). It can't add a visual, a colour or a +formula; a new block is a Steward release (System 181's ruling). + + name: Fabled Archives + version: 1 + expects: fabledarchives_ + blocks: + - states: + title: Backup targets + metric: fabledarchives_backup_target_state + name: target + healthy: [fresh] + quiet: [disabled, waiting] + warning: [late] + - values: + items: + - {label: Disk free, metric: fabledarchives_data_disk_free_bytes, + of: fabledarchives_data_disk_size_bytes, warn_below: 10%} + +check(text) returns the Template, or every problem found, each with its +line and a sentence saying what to change. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +import yaml + +FORMAT = 1 +MAX_BYTES = 64 * 1024 +MAX_BLOCKS = 20 +MAX_ITEMS = 30 + +_METRIC = re.compile(r"^[a-zA-Z_:][a-zA-Z0-9_:]*$") +_LABEL = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") +_DURATION = re.compile(r"^(\d+(?:\.\d+)?)\s*([smhd]?)$") +_SECONDS = {"": 1, "s": 1, "m": 60, "h": 3600, "d": 86400} + +# What a unit shows a value as; the name is the suffix metric_value reads. +UNITS = {"bytes": "_bytes", "seconds": "_secs", "percent": "_pct", "per_sec": "_per_sec", + "count": "", "timestamp": "_timestamp_seconds"} + + +@dataclass +class Problem: + line: int + text: str + + def __str__(self) -> str: + return f"Line {self.line}: {self.text}" if self.line else self.text + + +@dataclass +class Block: + kind: str # states | values | times | table | info + title: str + line: int + spec: dict = field(default_factory=dict) + + +@dataclass +class Template: + name: str + version: int + description: str = "" + expects: str = "" + blocks: list[Block] = field(default_factory=list) + text: str = "" + + +class TemplateError(ValueError): + def __init__(self, problems: list[Problem]): + self.problems = problems + super().__init__("; ".join(str(p) for p in problems)) + + +# ── YAML with line numbers ─────────────────────────────────────────────────── + +class _Map(dict): + line = 0 + lines: dict + + +class _List(list): + line = 0 + lines: list + + +class _Loader(yaml.SafeLoader): + pass + + +def _construct_map(loader, node): + loader.flatten_mapping(node) + out = _Map() + out.line, out.lines = node.start_mark.line + 1, {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=True) + if not isinstance(key, str): + raise yaml.constructor.ConstructorError( + None, None, "a key here should be a word, like `metric`", key_node.start_mark) + if key in out: + raise yaml.constructor.ConstructorError( + None, None, f"`{key}` is given twice", key_node.start_mark) + out[key] = loader.construct_object(value_node, deep=True) + out.lines[key] = key_node.start_mark.line + 1 + return out + + +def _construct_list(loader, node): + out = _List(loader.construct_object(n, deep=True) for n in node.value) + out.line, out.lines = node.start_mark.line + 1, [n.start_mark.line + 1 for n in node.value] + return out + + +_Loader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_map) +_Loader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, _construct_list) + + +def _load(text: str) -> tuple[object, list[Problem]]: + if len(text.encode()) > MAX_BYTES: + return None, [Problem(0, f"A template is at most {MAX_BYTES // 1024} KB.")] + try: + # An alias can make a small file expand without end; a template has no use for one. + for event in yaml.parse(text, Loader=yaml.SafeLoader): + if isinstance(event, yaml.AliasEvent): + return None, [Problem(event.start_mark.line + 1, + "Anchors and aliases (&name, *name) aren't allowed; write the value out.")] + return yaml.load(text, Loader=_Loader), [] + except yaml.constructor.ConstructorError as exc: # ours: a key twice, or not a word + mark = exc.problem_mark + return None, [Problem(mark.line + 1 if mark else 0, exc.problem[:1].upper() + exc.problem[1:] + ".")] + except yaml.MarkedYAMLError as exc: + mark = exc.problem_mark or exc.context_mark + return None, [Problem(mark.line + 1 if mark else 0, f"This isn't valid YAML: {exc.problem}.")] + except yaml.YAMLError as exc: + return None, [Problem(0, f"This isn't valid YAML: {exc}.")] + + +# ── Values ─────────────────────────────────────────────────────────────────── + +def duration(value) -> float | None: + """Seconds from 90, "90s", "15m", "26h" or "2d"; None if it isn't one.""" + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) if value >= 0 else None + m = _DURATION.match(str(value).strip().lower()) + return float(m.group(1)) * _SECONDS[m.group(2)] if m else None + + +@dataclass +class Limit: + """A threshold: a number, or a share of the item's `of` ("10%").""" + value: float + share: bool = False + + +def limit(value) -> Limit | None: + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return Limit(float(value)) + text = str(value).strip() + if text.endswith("%"): + try: + return Limit(float(text[:-1]), share=True) + except ValueError: + return None + return None + + +# ── The checker ────────────────────────────────────────────────────────────── + +class _Check: + def __init__(self): + self.problems: list[Problem] = [] + + def add(self, line: int, text: str) -> None: + self.problems.append(Problem(line, text)) + + def keys(self, m: _Map, allowed: tuple, required: tuple, where: str) -> bool: + ok = True + for key in m: + if key not in allowed: + self.add(m.lines[key], f"{where} has no `{key}`. It takes: {', '.join(allowed)}.") + ok = False + for key in required: + if key not in m: + self.add(m.line, f"{where} needs `{key}`.") + ok = False + return ok + + def text(self, m: _Map, key: str, where: str, default: str = "") -> str: + if key not in m: + return default + value = m[key] + if isinstance(value, (dict, list)) or value is None: + self.add(m.lines[key], f"`{key}` in {where} should be text.") + return default + return str(value) + + def metric(self, m: _Map, key: str, where: str) -> str: + value = m.get(key) + if not isinstance(value, str) or not _METRIC.match(value): + self.add(m.lines.get(key, m.line), + f"`{key}` in {where} should be a metric name as Steward stores it, like " + "fabledarchives_files (a counter is stored as _per_sec).") + return "" + return value + + def label(self, m: _Map, key: str, where: str) -> str: + if key not in m: + return "" + value = m[key] + if not isinstance(value, str) or not _LABEL.match(value): + self.add(m.lines[key], f"`{key}` in {where} should be a label name, like target.") + return "" + return value + + def words(self, m: _Map, key: str, where: str) -> list[str]: + if key not in m: + return [] + value = m[key] + if not isinstance(value, list) or not all(isinstance(v, (str, int)) for v in value): + self.add(m.lines[key], f"`{key}` in {where} should be a list, like [fresh, waiting].") + return [] + if any(isinstance(v, bool) for v in value): + self.add(m.lines[key], f"`{key}` in {where} has a word YAML reads as true or false " + "(on, off, yes, no); put it in quotes, like 'off'.") + return [] + return [str(v) for v in value] + + def where(self, m: _Map, ctx: str) -> dict[str, str]: + if "where" not in m: + return {} + value = m["where"] + if not isinstance(value, dict) or not all( + isinstance(k, str) and _LABEL.match(k) and isinstance(v, (str, int, float)) + and not isinstance(v, bool) for k, v in value.items()): + self.add(m.lines["where"], f"`where` in {ctx} should map labels to values, like " + "{kind: files}. Quote a value YAML would read as true or false.") + return {} + return {k: str(v) for k, v in value.items()} + + def unit(self, m: _Map, ctx: str) -> str: + if "unit" not in m: + return "" + value = m["unit"] + if value not in UNITS: + self.add(m.lines["unit"], f"`unit` in {ctx} is one of: {', '.join(UNITS)}.") + return "" + return value + + def span(self, m: _Map, key: str, ctx: str) -> float | None: + if key not in m: + return None + value = duration(m[key]) + if value is None: + self.add(m.lines[key], f"`{key}` in {ctx} should be a span, like 26h, 90m or 2d.") + return value + + def items(self, m: _Map, ctx: str) -> list[_Map]: + value = m.get("items") + if not isinstance(value, list) or not value: + self.add(m.lines.get("items", m.line), f"`items` in {ctx} should be a list of one or more items.") + return [] + if len(value) > MAX_ITEMS: + self.add(m.lines["items"], f"{ctx} has {len(value)} items; at most {MAX_ITEMS}.") + out = [] + for i, item in enumerate(value): + if not isinstance(item, dict): + self.add(value.lines[i], f"Each item in {ctx} should be a set of keys, like " + "{label: Disk free, metric: …}.") + else: + out.append(item) + return out + + +# Each block: the keys it takes, those it needs, and its title when none is given. +_BLOCKS = { + "states": (("title", "metric", "where", "name", "healthy", "quiet", "warning", "columns"), + ("metric", "healthy"), "States"), + "values": (("title", "items"), ("items",), "Figures"), + "times": (("title", "items"), ("items",), "Times"), + "table": (("title", "metric", "where", "rows", "columns", "unit"), ("metric", "rows", "columns"), + "Table"), + "info": (("title", "metric", "where", "labels"), ("metric",), "About"), +} +_VALUE_ITEM = ("label", "metric", "where", "name", "of", "unit", "warn_above", "crit_above", + "warn_below", "crit_below", "sparkline") +_TIME_ITEM = ("label", "metric", "where", "name", "warn_after", "crit_after") +_COLUMN = ("label", "metric", "unit") + + +def _states(c: _Check, m: _Map, ctx: str) -> dict: + spec = {"metric": c.metric(m, "metric", ctx), "where": c.where(m, ctx), "name": c.label(m, "name", ctx), + "healthy": c.words(m, "healthy", ctx), "quiet": c.words(m, "quiet", ctx), + "warning": c.words(m, "warning", ctx), "columns": []} + if "healthy" in m and not spec["healthy"] and isinstance(m["healthy"], list): + c.add(m.lines["healthy"], f"`healthy` in {ctx} needs at least one state.") + seen: dict[str, str] = {} + for key in ("healthy", "quiet", "warning"): + for state in spec[key]: + if state in seen: + c.add(m.lines[key], f"“{state}” is in both `{seen[state]}` and `{key}` in {ctx}.") + seen[state] = key + if "columns" in m: + cols = m["columns"] + if not isinstance(cols, list): + c.add(m.lines["columns"], f"`columns` in {ctx} should be a list of {{label, metric}}.") + else: + for i, col in enumerate(cols): + where = f"a column of {ctx}" + if not isinstance(col, dict): + c.add(cols.lines[i], f"Each of `columns` in {ctx} should be {{label, metric}}.") + continue + if c.keys(col, _COLUMN, ("label", "metric"), where): + spec["columns"].append({"label": c.text(col, "label", where), + "metric": c.metric(col, "metric", where), + "unit": c.unit(col, where)}) + return spec + + +def _value_item(c: _Check, item: _Map, ctx: str) -> dict | None: + where = f"an item of {ctx}" + if not c.keys(item, _VALUE_ITEM, ("label", "metric"), where): + return None + spec = {"label": c.text(item, "label", where), "metric": c.metric(item, "metric", where), + "where": c.where(item, where), "name": c.label(item, "name", where), + "of": c.metric(item, "of", where) if "of" in item else "", "unit": c.unit(item, where), + "sparkline": False} + if "sparkline" in item: + if not isinstance(item["sparkline"], bool): + c.add(item.lines["sparkline"], f"`sparkline` in {where} is true or false.") + spec["sparkline"] = item["sparkline"] is True + for key in ("warn_above", "crit_above", "warn_below", "crit_below"): + spec[key] = None + if key in item: + lim = limit(item[key]) + if lim is None: + c.add(item.lines[key], f"`{key}` in {where} should be a number, or a share like 10%.") + elif lim.share and not spec["of"]: + c.add(item.lines[key], f"`{key}` in {where} is a share ({item[key]}), which needs `of`: " + "the metric it is a share of.") + spec[key] = lim + return spec + + +def _time_item(c: _Check, item: _Map, ctx: str) -> dict | None: + where = f"an item of {ctx}" + if not c.keys(item, _TIME_ITEM, ("label", "metric"), where): + return None + spec = {"label": c.text(item, "label", where), "metric": c.metric(item, "metric", where), + "where": c.where(item, where), "name": c.label(item, "name", where), + "warn_after": c.span(item, "warn_after", where), "crit_after": c.span(item, "crit_after", where)} + if spec["warn_after"] and spec["crit_after"] and spec["crit_after"] < spec["warn_after"]: + c.add(item.lines["crit_after"], f"`crit_after` in {where} should be later than `warn_after`.") + return spec + + +def _block(c: _Check, raw, line: int, n: int) -> Block | None: + if not isinstance(raw, dict) or len(raw) != 1: + c.add(line, f"Block {n} should be one kind with its settings under it, like " + f"`- states: {{metric: …}}`. The kinds are: {', '.join(_BLOCKS)}.") + return None + kind, body = next(iter(raw.items())) + if kind not in _BLOCKS: + c.add(line, f"Block {n} is “{kind}”; the kinds are: {', '.join(_BLOCKS)}.") + return None + if not isinstance(body, dict): + c.add(line, f"Block {n} ({kind}) should have its settings under it, like `metric: …`.") + return None + allowed, required, default_title = _BLOCKS[kind] + ctx = f"block {n} ({kind})" + if not c.keys(body, allowed, required, ctx.capitalize()): + return None + title = c.text(body, "title", ctx, default_title) + if kind == "states": + spec = _states(c, body, ctx) + elif kind in ("values", "times"): + reader = _value_item if kind == "values" else _time_item + spec = {"items": [s for s in (reader(c, i, ctx) for i in c.items(body, ctx)) if s]} + elif kind == "table": + spec = {"metric": c.metric(body, "metric", ctx), "where": c.where(body, ctx), + "rows": c.label(body, "rows", ctx), "columns": c.label(body, "columns", ctx), + "unit": c.unit(body, ctx)} + else: + spec = {"metric": c.metric(body, "metric", ctx), "where": c.where(body, ctx), + "labels": c.words(body, "labels", ctx)} + return Block(kind, title, line, spec) + + +_TOP = ("name", "description", "version", "format", "expects", "blocks") + + +def check(text: str) -> Template: + """The template `text` describes. Raises TemplateError with every problem + found, each with its line, in the order they appear.""" + raw, problems = _load(text) + if problems: + raise TemplateError(problems) + c = _Check() + if not isinstance(raw, dict): + raise TemplateError([Problem(1, "A template starts with its keys: name, version and blocks.")]) + c.keys(raw, _TOP, ("name", "version", "blocks"), "A template") + name = c.text(raw, "name", "the template").strip() + if "name" in raw and not name: + c.add(raw.lines["name"], "Give the template a name.") + if len(name) > 80: + c.add(raw.lines["name"], "A name is at most 80 characters.") + version = raw.get("version") + if "version" in raw and (isinstance(version, bool) or not isinstance(version, int) or version < 1): + c.add(raw.lines["version"], "`version` is the template's own version: a whole number from 1. " + "Raise it when you change a template you have shared.") + if "format" in raw and raw["format"] != FORMAT: + c.add(raw.lines["format"], f"This Steward reads template format {FORMAT}.") + expects = c.text(raw, "expects", "the template") + if expects and not re.match(r"^[a-zA-Z_:][a-zA-Z0-9_:]*$", expects): + c.add(raw.lines["expects"], "`expects` should be the start of the metric names it reads, " + "like fabledarchives_.") + blocks: list[Block] = [] + raw_blocks = raw.get("blocks") + if "blocks" in raw: + if not isinstance(raw_blocks, list) or not raw_blocks: + c.add(raw.lines["blocks"], "`blocks` should be a list of one or more blocks.") + else: + if len(raw_blocks) > MAX_BLOCKS: + c.add(raw.lines["blocks"], f"A template has at most {MAX_BLOCKS} blocks.") + for i, b in enumerate(raw_blocks): + block = _block(c, b, raw_blocks.lines[i], i + 1) + if block: + blocks.append(block) + if c.problems: + raise TemplateError(sorted(c.problems, key=lambda p: p.line)) + return Template(name=name, version=version, description=c.text(raw, "description", "the template"), + expects=expects, blocks=blocks, text=text) + + +def problems(text: str) -> list[Problem]: + """Every problem with `text`; empty when it is a template.""" + try: + check(text) + except TemplateError as exc: + return exc.problems + return [] diff --git a/plugins/prometheus/template_view.py b/plugins/prometheus/template_view.py new file mode 100644 index 0000000..1644d54 --- /dev/null +++ b/plugins/prometheus/template_view.py @@ -0,0 +1,302 @@ +"""A widget template drawn from a target's newest values: what the +Prometheus dashboard widget shows. Rows in, a view out; nothing is fetched +here, so the widget route reads the database and a test passes rows. + +The view follows the widget shape (System 154): a verdict naming the worst +thing, then each block with its problems first and the healthy rest after. +Every row carries a tone: crit, warn, ok, or dim for a state the template +calls quiet. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime + +from steward.core.formatting import human_duration + +from .service import fold_states, label_pairs, shown_value +from .template_format import UNITS, Block, Limit, Template + +TONE_ORDER = {"crit": 0, "warn": 1, "ok": 2, "dim": 3} + + +@dataclass +class Sample: + resource: str + metric: str + labels: str # as stored: "target=S,state=late" + pairs: dict[str, str] + value: float + at: datetime | None = None + + +@dataclass +class Row: + name: str + tone: str = "ok" + why: str = "" # what is wrong, in words: "is late", "below 10%" + text: str = "" # the value as shown, "—" when there is none + moment: datetime | None = None + bar: float | None = None # a share of `of`, 0 to 1 + cells: list[dict] = field(default_factory=list) + resource: str = "" + metric: str = "" + history: list[float] | None = None + + +@dataclass +class BlockView: + kind: str + title: str + tone: str = "ok" + rows: list[Row] = field(default_factory=list) + columns: list[str] = field(default_factory=list) + note: str = "" # the block couldn't be drawn: no series, or not a state set + + +@dataclass +class View: + tone: str + verdict: str + blocks: list[BlockView] + wants: list[tuple[str, str]] = field(default_factory=list) # (resource, metric) for sparklines + note: str = "" + + +def samples(target: str, latest) -> dict[str, list[Sample]]: + """A target's newest values by metric. `latest` is PluginMetricLatest + rows (resource_name, metric_name, value, recorded_at) for the target and + its ":" children.""" + out: dict[str, list[Sample]] = {} + for row in latest: + if row.resource_name == target: + labels = "" + elif row.resource_name.startswith(target + ":"): + labels = row.resource_name[len(target) + 1:] + else: + continue + out.setdefault(row.metric_name, []).append(Sample( + row.resource_name, row.metric_name, labels, dict(label_pairs(labels)), row.value, + getattr(row, "recorded_at", None))) + return out + + +def pick(index: dict[str, list[Sample]], metric: str, where: dict[str, str]) -> list[Sample]: + return [s for s in index.get(metric, []) if all(s.pairs.get(k) == v for k, v in where.items())] + + +def shown(metric: str, unit: str, value: float | None) -> dict: + """{text, moment} for a value, in `unit` or else as its name says. A + count reads with thousands separated: 93,310.""" + if unit == "count" and value is not None: + return {"text": f"{value:,.0f}" if value == int(value) else f"{value:,.2f}", "moment": None} + return shown_value(f"x{UNITS[unit]}" if unit else metric, value) + + +def _worst(tones) -> str: + tones = [t for t in tones if t != "dim"] + return min(tones, key=TONE_ORDER.__getitem__) if tones else "ok" + + +def _ordered(rows: list[Row]) -> list[Row]: + return sorted(rows, key=lambda r: (TONE_ORDER[r.tone], r.name.lower())) + + +# ── Blocks ─────────────────────────────────────────────────────────────────── + +def _states(block: Block, index, now) -> BlockView: + spec = block.spec + view = BlockView("states", block.title, columns=[c["label"] for c in spec["columns"]]) + found = pick(index, spec["metric"], spec["where"]) + folded = fold_states(spec["metric"], [{"labels": s.labels, "value": s.value, "resource": s.resource} + for s in found]) + if not found: + view.note = f"Nothing reported for {spec['metric']}." + elif folded is None: + view.note = (f"{spec['metric']} isn't a state set: it should have one series per state, " + "the current one 1 and the rest 0.") + for f in folded or []: + pairs = dict(label_pairs(f["labels"])) + name = pairs.get(spec["name"], "") if spec["name"] else "" + current = [s for s in f["state"].split(", ") if s] + tones = [] + for state in current: + if state in spec["healthy"]: + tones.append("ok") + elif state in spec["quiet"]: + tones.append("dim") + elif state in spec["warning"]: + tones.append("warn") + else: + tones.append("crit") + tone = (min(tones, key=TONE_ORDER.__getitem__) if tones else "warn") + row = Row(name=name or f["labels"] or block.title, tone=tone, text=f["state"] or "—", + why="" if tone in ("ok", "dim") else (f"is {f['state']}" if current else "reports no state"), + resource=f["resource"], metric=spec["metric"]) + for col in spec["columns"]: + match = [s for s in index.get(col["metric"], []) if s.labels == f["labels"]] + if not match and name: + match = [s for s in index.get(col["metric"], []) if s.pairs.get(spec["name"]) == name] + row.cells.append(shown(col["metric"], col["unit"], match[0].value if len(match) == 1 else None)) + view.rows.append(row) + view.rows = _ordered(view.rows) + return view + + +def _limit_text(lim: Limit, metric: str, unit: str) -> str: + return f"{lim.value:g}%" if lim.share else (shown(metric, unit, lim.value)["text"] or f"{lim.value:g}") + + +def _judge(item: dict, value: float, of: float | None) -> tuple[str, str]: + for level in ("crit", "warn"): + for side in ("above", "below"): + lim = item[f"{level}_{side}"] + if lim is None: + continue + if lim.share: + if not of: + continue + x = value / of * 100 + else: + x = value + if (x > lim.value) if side == "above" else (x < lim.value): + return level, f"{side} {_limit_text(lim, item['metric'], item['unit'])}" + return "ok", "" + + +def _rows_for(item: dict, index) -> list[tuple[str, Sample | None, str]]: + """The series an item shows: one per value of its `name` label, or the one + series it names. (name, sample, problem).""" + found = pick(index, item["metric"], item["where"]) + if item["name"]: + named = [(s.pairs.get(item["name"], "") or item["label"], s, "") for s in found] + return named or [(item["label"], None, "not reported")] + if len(found) == 1: + return [(item["label"], found[0], "")] + if not found: + return [(item["label"], None, "not reported")] + return [(item["label"], None, f"{len(found)} series match; narrow it with `where` or `name`")] + + +def _values(block: Block, index, now) -> tuple[BlockView, list]: + view, wants = BlockView("values", block.title), [] + for item in block.spec["items"]: + for name, s, problem in _rows_for(item, index): + if s is None: + view.rows.append(Row(name=name, tone="warn", why=problem, text="—", metric=item["metric"])) + continue + row = Row(name=name, resource=s.resource, metric=s.metric, **shown(s.metric, item["unit"], s.value)) + of = None + if item["of"]: + whole = [o for o in index.get(item["of"], []) if o.labels == s.labels] + if len(whole) == 1 and whole[0].value: + of = whole[0].value + row.bar = max(0.0, min(1.0, s.value / of)) + row.text = f"{row.text} of {shown(item['of'], item['unit'], of)['text']}" + row.tone, row.why = _judge(item, s.value, of) + if item["sparkline"]: + wants.append((s.resource, s.metric)) + view.rows.append(row) + view.rows = _ordered(view.rows) + return view, wants + + +def _times(block: Block, index, now) -> BlockView: + view = BlockView("times", block.title) + for item in block.spec["items"]: + for name, s, problem in _rows_for(item, index): + if s is None: + view.rows.append(Row(name=name, tone="warn", why=problem, text="—", metric=item["metric"])) + continue + got = shown_value("x_timestamp_seconds", s.value) + row = Row(name=name, resource=s.resource, metric=s.metric, **got) + if row.moment is None: + if item["warn_after"] or item["crit_after"]: + row.tone, row.why = "warn", "never" + else: + age = (now - row.moment).total_seconds() + for level in ("crit", "warn"): + after = item[f"{level}_after"] + if after and age > after: + row.tone, row.why = level, f"over {human_duration(after)} ago" + break + view.rows.append(row) + view.rows = _ordered(view.rows) + return view + + +def _table(block: Block, index, now) -> BlockView: + spec = block.spec + found = pick(index, spec["metric"], spec["where"]) + rows: dict[str, dict[str, float]] = {} + columns: list[str] = [] + for s in found: + r, c = s.pairs.get(spec["rows"]), s.pairs.get(spec["columns"]) + if r is None or c is None: + continue + rows.setdefault(r, {})[c] = s.value + if c not in columns: + columns.append(c) + columns.sort() + view = BlockView("table", block.title, columns=columns) + if not rows: + view.note = (f"Nothing reported for {spec['metric']} with labels " + f"{spec['rows']} and {spec['columns']}.") + for name in sorted(rows, key=str.lower): + view.rows.append(Row(name=name, cells=[shown(spec["metric"], spec["unit"], rows[name].get(c)) + for c in columns])) + return view + + +def _info(block: Block, index, now) -> BlockView: + spec = block.spec + view = BlockView("info", block.title) + found = sorted(pick(index, spec["metric"], spec["where"]), key=lambda s: s.labels) + if not found: + view.note = f"Nothing reported for {spec['metric']}." + for s in found: + keys = spec["labels"] or list(s.pairs) + view.rows.append(Row(name=" · ".join(f"{k} {s.pairs[k]}" for k in keys if s.pairs.get(k)), + resource=s.resource, metric=s.metric)) + return view + + +_DRAW = {"states": _states, "times": _times, "table": _table, "info": _info} + + +def render(template: Template, target: str, latest, now: datetime, + history: dict[tuple[str, str], list[float]] | None = None) -> View: + """The view of `template` over a target's newest values. `history` gives + the sparkline points the first pass asked for in `wants`.""" + index = samples(target, latest) + blocks, wants = [], [] + for block in template.blocks: + if block.kind == "values": + view, want = _values(block, index, now) + wants += want + else: + view = _DRAW[block.kind](block, index, now) + if history: + for row in view.rows: + row.history = history.get((row.resource, row.metric)) + view.tone = "warn" if view.note else _worst(r.tone for r in view.rows) + blocks.append(view) + + note = "" + if template.expects and not any(m.startswith(template.expects) for m in index): + note = (f"{target} reports no {template.expects}… metrics: is “{template.name}” " + "the template for it, or is the target filtering them out?") + problems = [(b, r) for b in blocks for r in b.rows if r.tone in ("crit", "warn")] + problems.sort(key=lambda p: TONE_ORDER[p[1].tone]) + empty = [b for b in blocks if b.note] + tone = _worst([b.tone for b in blocks] + (["warn"] if note else [])) + if note: + verdict = note + elif len(problems) + len(empty) == 0: + verdict = "All well." + else: + first = (f"{problems[0][0].title}: {problems[0][1].name} {problems[0][1].why}." if problems + else f"{empty[0].title}: {empty[0].note}") + count = len(problems) + len(empty) + verdict = first if count == 1 else f"{count} need a look. {first}" + return View(tone=tone, verdict=verdict, blocks=blocks, wants=wants, note=note) diff --git a/tests/plugins/prometheus/test_templates.py b/tests/plugins/prometheus/test_templates.py new file mode 100644 index 0000000..dc891fd --- /dev/null +++ b/tests/plugins/prometheus/test_templates.py @@ -0,0 +1,238 @@ +"""Prometheus widget templates (#5690): the checker says what is wrong and +where, and the renderer turns a template and a target's newest values into a +verdict, then blocks with their problems first.""" +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace as NS + +import pytest + +from plugins.prometheus.template_format import TemplateError, check, duration, problems +from plugins.prometheus.template_view import render + +NOW = datetime(2026, 10, 12, 19, 0, tzinfo=timezone.utc) +T = "Archives" + + +def before(**kw) -> float: + return (NOW - timedelta(**kw)).timestamp() + + +def latest(series: dict) -> list: + """PluginMetricLatest-like rows from {(labels, metric): value}, labels + as stored after the target's name.""" + return [NS(resource_name=f"{T}:{labels}" if labels else T, metric_name=metric, value=value, + recorded_at=NOW) for (labels, metric), value in series.items()] + + +def states(prefix: str, current: str, all_states: list[str]) -> dict: + return {(f"{prefix},state={s}", "fa_target_state"): float(s == current) for s in all_states} + + +TARGET_STATES = ["disabled", "fresh", "late", "missing", "waiting"] + +SERIES = { + **states("target=Steward,target_id=1", "fresh", TARGET_STATES), + **states("target=Photos,target_id=2", "late", TARGET_STATES), + **states("target=Old,target_id=3", "disabled", TARGET_STATES), + ("target=Steward,target_id=1", "fa_target_last_backup_timestamp_seconds"): before(hours=16), + ("target=Photos,target_id=2", "fa_target_last_backup_timestamp_seconds"): before(hours=30), + ("", "fa_disk_free_bytes"): 17.8e12, + ("", "fa_disk_size_bytes"): 46e12, + ("", "fa_files"): 94116.0, + ("", "fa_files_checked"): 93310.0, + ("", "fa_malware_held_files"): 2.0, + ("person=Bryan,person_id=9,kind=files", "fa_person_stored_bytes"): 4.7e11, + ("person=Bryan,person_id=9,kind=trash", "fa_person_stored_bytes"): 1.2e9, + ("person=Ana,person_id=8,kind=files", "fa_person_stored_bytes"): 3.0e10, + ("job=Nightly,job_id=5", "fa_job_next_run_timestamp_seconds"): (NOW + timedelta(hours=8)).timestamp(), + ("version=2026.10.12.1717,channel=dev,commit=3605d0a", "fa_build_info"): 1.0, +} + +TEMPLATE = """\ +name: Test Archives +version: 1 +expects: fa_ +blocks: + - states: + title: Backup targets + metric: fa_target_state + name: target + healthy: [fresh] + quiet: [disabled, waiting] + warning: [late] + columns: + - {label: Last backup, metric: fa_target_last_backup_timestamp_seconds} + - values: + title: Storage + items: + - {label: Disk free, metric: fa_disk_free_bytes, of: fa_disk_size_bytes, warn_below: 10%, + sparkline: true} + - {label: Files checked, metric: fa_files_checked, of: fa_files, unit: count} + - {label: Malware held, metric: fa_malware_held_files, unit: count, warn_above: 0} + - times: + items: + - {label: Next run, metric: fa_job_next_run_timestamp_seconds, name: job} + - {label: Last backup, metric: fa_target_last_backup_timestamp_seconds, name: target, + warn_after: 26h, crit_after: 3d} + - table: + title: Stored per person + metric: fa_person_stored_bytes + rows: person + columns: kind + - info: + metric: fa_build_info + labels: [version, commit] +""" + + +# ── The checker ────────────────────────────────────────────────────────────── + +def test_a_template_reads_into_its_blocks(): + t = check(TEMPLATE) + assert (t.name, t.version, t.expects) == ("Test Archives", 1, "fa_") + assert [b.kind for b in t.blocks] == ["states", "values", "times", "table", "info"] + assert t.blocks[2].title == "Times" # a block's default title + disk = t.blocks[1].spec["items"][0] + assert disk["of"] == "fa_disk_size_bytes" and disk["warn_below"].share and disk["sparkline"] + assert t.blocks[2].spec["items"][1]["warn_after"] == 26 * 3600 + + +def _said(text: str) -> list[str]: + return [str(p) for p in problems(text)] + + +def test_every_problem_is_said_with_its_line(): + said = _said("""\ +name: X +version: 0 +blocks: + - states: + metric: fa_state + healthy: [ok] + warning: [ok] + colour: red + - gauge: {metric: x} + - values: + items: + - {label: Free, metric: fa free, warn_below: 10%} +""") + assert said == [ + "Line 2: `version` is the template's own version: a whole number from 1. " + "Raise it when you change a template you have shared.", + "Line 8: Block 1 (states) has no `colour`. It takes: title, metric, where, name, healthy, " + "quiet, warning, columns.", + "Line 9: Block 2 is “gauge”; the kinds are: states, values, times, table, info.", + "Line 12: `metric` in an item of block 3 (values) should be a metric name as Steward stores it, " + "like fabledarchives_files (a counter is stored as _per_sec).", + "Line 12: `warn_below` in an item of block 3 (values) is a share (10%), which needs `of`: " + "the metric it is a share of.", + ] + + +def test_a_state_in_two_lists_and_a_yaml_boolean_are_caught(): + said = _said("name: X\nversion: 1\nblocks:\n - states: {metric: s_state, healthy: [ok], warning: [ok]}\n") + assert said == ["Line 4: “ok” is in both `healthy` and `warning` in block 1 (states)."] + said = _said("name: X\nversion: 1\nblocks:\n - states: {metric: s_state, healthy: [on]}\n") + assert "put it in quotes, like 'off'" in said[0] + + +def test_what_isnt_a_template_says_so(): + broken = _said("name: [unclosed\n") + assert broken[0].startswith("Line ") and "This isn't valid YAML" in broken[0] + assert _said("just text") == ["Line 1: A template starts with its keys: name, version and blocks."] + assert _said("name: X\nname: Y\n") == ["Line 2: `name` is given twice."] + assert _said("name: X\n") == ["Line 1: A template needs `version`.", "Line 1: A template needs `blocks`."] + assert "Anchors and aliases" in _said("a: &x 1\nb: *x\n")[0] + assert "at most 64 KB" in _said("x" * (65 * 1024))[0] + with pytest.raises(TemplateError): + check("name: X\nversion: 1\nblocks: []\n") + + +def test_spans(): + assert [duration(v) for v in (90, "90s", "15m", "26h", "2d", "1.5h")] == [90, 90, 900, 93600, 172800, 5400] + assert duration("soon") is None and duration(-1) is None and duration(True) is None + + +# ── The renderer ───────────────────────────────────────────────────────────── + +def _view(series=SERIES, text=TEMPLATE, history=None): + return render(check(text), T, latest(series), NOW, history=history) + + +def test_the_verdict_names_the_worst_and_each_block_leads_with_its_problems(): + view = _view() + # Photos is late, its last backup is past 26h, and malware is held: three warnings. + assert view.tone == "warn" + assert view.verdict == "3 need a look. Backup targets: Photos is late." + targets = view.blocks[0] + assert [(r.name, r.tone, r.text) for r in targets.rows] == [ + ("Photos", "warn", "late"), ("Steward", "ok", "fresh"), ("Old", "dim", "disabled")] + assert targets.rows[0].why == "is late" and targets.columns == ["Last backup"] + assert targets.rows[1].cells[0]["moment"] == datetime.fromtimestamp(before(hours=16), timezone.utc) + assert targets.tone == "warn" + + +def test_one_problem_is_the_verdict_itself(): + view = _view({**SERIES, ("", "fa_malware_held_files"): 0.0, + ("target=Photos,target_id=2", "fa_target_last_backup_timestamp_seconds"): before(hours=2)}) + assert view.verdict == "Backup targets: Photos is late." + + +def test_values_show_shares_bars_and_thresholds(): + storage = _view().blocks[1] + rows = {r.name: r for r in storage.rows} + assert rows["Disk free"].text == "16.2 TiB of 41.8 TiB" and round(rows["Disk free"].bar, 2) == 0.39 + assert rows["Disk free"].tone == "ok" + assert rows["Files checked"].text == "93,310 of 94,116" + assert (rows["Malware held"].tone, rows["Malware held"].why) == ("warn", "above 0") + assert storage.rows[0].name == "Malware held" # problems first + low = _view({**SERIES, ("", "fa_disk_free_bytes"): 2e12}).blocks[1] + assert {r.name: (r.tone, r.why) for r in low.rows}["Disk free"] == ("warn", "below 10%") + + +def test_a_sparkline_is_asked_for_then_drawn(): + view = _view() + assert view.wants == [(T, "fa_disk_free_bytes")] + drawn = _view(history={(T, "fa_disk_free_bytes"): [1.0, 2.0]}) + assert {r.name: r.history for r in drawn.blocks[1].rows}["Disk free"] == [1.0, 2.0] + + +def test_times_read_past_and_future_and_flag_the_overdue(): + times = _view().blocks[2] + rows = {(r.name, r.metric): r for r in times.rows} + photos = rows[("Photos", "fa_target_last_backup_timestamp_seconds")] + assert (photos.tone, photos.why) == ("warn", "over 1d 2h ago") + nightly = rows[("Nightly", "fa_job_next_run_timestamp_seconds")] + assert nightly.tone == "ok" and nightly.moment > NOW # "in 8h" + stale = _view({**SERIES, ("target=Photos,target_id=2", "fa_target_last_backup_timestamp_seconds"): + before(days=4)}).blocks[2] + assert stale.rows[0].tone == "crit" and stale.rows[0].why == "over 3d ago" + + +def test_a_table_pivots_two_labels_and_info_shows_its_labels(): + table, info = _view().blocks[3:] + assert table.columns == ["files", "trash"] + assert [(r.name, [c["text"] for c in r.cells]) for r in table.rows] == [ + ("Ana", ["27.9 GiB", "—"]), ("Bryan", ["437.7 GiB", "1.1 GiB"])] + assert [r.name for r in info.rows] == ["version 2026.10.12.1717 · commit 3605d0a"] + + +def test_missing_series_are_said_not_hidden(): + bare = {k: v for k, v in SERIES.items() if k[1] != "fa_malware_held_files" and k[1] != "fa_build_info"} + view = _view(bare) + malware = {r.name: r for r in view.blocks[1].rows}["Malware held"] + assert (malware.text, malware.tone, malware.why) == ("—", "warn", "not reported") + assert view.blocks[4].note == "Nothing reported for fa_build_info." and view.blocks[4].tone == "warn" + + +def test_a_template_for_another_target_says_so(): + view = _view({("", "node_load1"): 0.5}) + assert view.tone == "warn" and "reports no fa_… metrics" in view.verdict + + +def test_all_well(): + healthy = {**SERIES, **states("target=Photos,target_id=2", "fresh", TARGET_STATES), + ("target=Photos,target_id=2", "fa_target_last_backup_timestamp_seconds"): before(hours=2), + ("", "fa_malware_held_files"): 0.0} + view = _view(healthy) + assert (view.tone, view.verdict) == ("ok", "All well.")