feat(prometheus): widget template format, checker and renderer (#5690)

#5690 "Widget template format: YAML schema, validator and renderer", step 2
of milestone 508 "Prometheus widget templates".

- template_format.check(text) -> Template, or TemplateError with every
  problem, each with its line and what to change. Blocks: states, values
  (of -> share and bar, thresholds as numbers or shares, sparkline),
  times (warn_after / crit_after), table (two labels pivoted), info.
  Anchors/aliases refused, 64 KB cap, YAML's on/off/yes/no flagged.
- template_view.render(template, target, latest, now, history) -> View:
  a verdict naming the worst problem, then each block with its problems
  first (crit, warn, ok, then quiet states). Missing series show as "—"
  and say so; a template whose `expects` prefix the target lacks says so.
- shown_value moves to service.py so the renderer can use it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/prometheus/routes.py

@@ -13,7 +13,6 @@ from sqlalchemy import delete, func, select, update
 from steward.auth.middleware import require_role
 from steward.core.audit import log_audit
 from steward.core.crypto import decrypt_secret
-from steward.core.formatting import metric_value
 from steward.core.metric_series import latest_values, points
 from steward.models.metrics import PluginMetric, PluginMetricLatest
 from steward.models.users import UserRole
@@ -21,7 +20,7 @@ from steward.models.users import UserRole
 from .models import PrometheusTarget
 from .scraper import ScrapeError, scrape
 from .service import (OWN_METRICS, SOURCE, FormError, fold_states, form_values, map_families,
-                      parse_form, patterns, target_state)
+                      parse_form, patterns, shown_value, target_state)
 
 prometheus_bp = Blueprint("prometheus", __name__, template_folder="templates")
 
@@ -48,19 +47,6 @@ def view(target: PrometheusTarget, own: dict, now: datetime) -> dict:
             "scrape_ms": own.get("scrape_ms")}
 
 
-def shown_value(metric: str, value: float | None) -> dict:
-    """A series' value in its unit, from the Prometheus naming conventions.
-    A *_timestamp_seconds is a moment: it comes back as `moment` for the page
-    to show as "3m ago" in the reader's zone; 0 there means "never"."""
-    if value is None:
-        return {"text": "—", "moment": None}
-    if metric.endswith("_timestamp_seconds"):
-        if value <= 0:
-            return {"text": "never", "moment": None}
-        return {"text": "", "moment": datetime.fromtimestamp(value, timezone.utc)}
-    return {"text": metric_value(metric, value), "moment": None}
-
-
 @prometheus_bp.get("/")
 @require_role(UserRole.viewer)
 async def index():

plugins/prometheus/service.py

@@ -5,11 +5,12 @@ from __future__ import annotations
 
 import math
 from dataclasses import dataclass, field
-from datetime import datetime
+from datetime import datetime, timezone
 from fnmatch import fnmatchcase
 
 from steward.core import prometheus
 from steward.core.crypto import encrypt_secret
+from steward.core.formatting import metric_value
 
 from .models import PrometheusTarget
 
@@ -121,6 +122,19 @@ def fold_states(metric: str, rows: list[dict]) -> list[dict] | None:
     return folded
 
 
+def shown_value(metric: str, value: float | None) -> dict:
+    """A series' value in its unit, from the Prometheus naming conventions.
+    A *_timestamp_seconds is a moment: it comes back as `moment` for the page
+    to show as "3m ago" in the reader's zone; 0 there means "never"."""
+    if value is None:
+        return {"text": "—", "moment": None}
+    if metric.endswith("_timestamp_seconds"):
+        if value <= 0:
+            return {"text": "never", "moment": None}
+        return {"text": "", "moment": datetime.fromtimestamp(value, timezone.utc)}
+    return {"text": metric_value(metric, value), "moment": None}
+
+
 @dataclass
 class Series:
     resource: str

plugins/prometheus/template_format.py

@@ -0,0 +1,451 @@
+"""Prometheus widget templates: the YAML a person writes, read and checked.
+
+A template says which of a target's series a dashboard widget shows and
+how. It is data only: it names series and lays them out in blocks, and
+Steward draws them (template_view.py). It can't add a visual, a colour or a
+formula; a new block is a Steward release (System 181's ruling).
+
+    name: Fabled Archives
+    version: 1
+    expects: fabledarchives_
+    blocks:
+      - states:
+          title: Backup targets
+          metric: fabledarchives_backup_target_state
+          name: target
+          healthy: [fresh]
+          quiet: [disabled, waiting]
+          warning: [late]
+      - values:
+          items:
+            - {label: Disk free, metric: fabledarchives_data_disk_free_bytes,
+               of: fabledarchives_data_disk_size_bytes, warn_below: 10%}
+
+check(text) returns the Template, or every problem found, each with its
+line and a sentence saying what to change.
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+import yaml
+
+FORMAT = 1
+MAX_BYTES = 64 * 1024
+MAX_BLOCKS = 20
+MAX_ITEMS = 30
+
+_METRIC = re.compile(r"^[a-zA-Z_:][a-zA-Z0-9_:]*$")
+_LABEL = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
+_DURATION = re.compile(r"^(\d+(?:\.\d+)?)\s*([smhd]?)$")
+_SECONDS = {"": 1, "s": 1, "m": 60, "h": 3600, "d": 86400}
+
+# What a unit shows a value as; the name is the suffix metric_value reads.
+UNITS = {"bytes": "_bytes", "seconds": "_secs", "percent": "_pct", "per_sec": "_per_sec",
+         "count": "", "timestamp": "_timestamp_seconds"}
+
+
+@dataclass
+class Problem:
+    line: int
+    text: str
+
+    def __str__(self) -> str:
+        return f"Line {self.line}: {self.text}" if self.line else self.text
+
+
+@dataclass
+class Block:
+    kind: str        # states | values | times | table | info
+    title: str
+    line: int
+    spec: dict = field(default_factory=dict)
+
+
+@dataclass
+class Template:
+    name: str
+    version: int
+    description: str = ""
+    expects: str = ""
+    blocks: list[Block] = field(default_factory=list)
+    text: str = ""
+
+
+class TemplateError(ValueError):
+    def __init__(self, problems: list[Problem]):
+        self.problems = problems
+        super().__init__("; ".join(str(p) for p in problems))
+
+
+# ── YAML with line numbers ───────────────────────────────────────────────────
+
+class _Map(dict):
+    line = 0
+    lines: dict
+
+
+class _List(list):
+    line = 0
+    lines: list
+
+
+class _Loader(yaml.SafeLoader):
+    pass
+
+
+def _construct_map(loader, node):
+    loader.flatten_mapping(node)
+    out = _Map()
+    out.line, out.lines = node.start_mark.line + 1, {}
+    for key_node, value_node in node.value:
+        key = loader.construct_object(key_node, deep=True)
+        if not isinstance(key, str):
+            raise yaml.constructor.ConstructorError(
+                None, None, "a key here should be a word, like `metric`", key_node.start_mark)
+        if key in out:
+            raise yaml.constructor.ConstructorError(
+                None, None, f"`{key}` is given twice", key_node.start_mark)
+        out[key] = loader.construct_object(value_node, deep=True)
+        out.lines[key] = key_node.start_mark.line + 1
+    return out
+
+
+def _construct_list(loader, node):
+    out = _List(loader.construct_object(n, deep=True) for n in node.value)
+    out.line, out.lines = node.start_mark.line + 1, [n.start_mark.line + 1 for n in node.value]
+    return out
+
+
+_Loader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_map)
+_Loader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, _construct_list)
+
+
+def _load(text: str) -> tuple[object, list[Problem]]:
+    if len(text.encode()) > MAX_BYTES:
+        return None, [Problem(0, f"A template is at most {MAX_BYTES // 1024} KB.")]
+    try:
+        # An alias can make a small file expand without end; a template has no use for one.
+        for event in yaml.parse(text, Loader=yaml.SafeLoader):
+            if isinstance(event, yaml.AliasEvent):
+                return None, [Problem(event.start_mark.line + 1,
+                                      "Anchors and aliases (&name, *name) aren't allowed; write the value out.")]
+        return yaml.load(text, Loader=_Loader), []
+    except yaml.constructor.ConstructorError as exc:      # ours: a key twice, or not a word
+        mark = exc.problem_mark
+        return None, [Problem(mark.line + 1 if mark else 0, exc.problem[:1].upper() + exc.problem[1:] + ".")]
+    except yaml.MarkedYAMLError as exc:
+        mark = exc.problem_mark or exc.context_mark
+        return None, [Problem(mark.line + 1 if mark else 0, f"This isn't valid YAML: {exc.problem}.")]
+    except yaml.YAMLError as exc:
+        return None, [Problem(0, f"This isn't valid YAML: {exc}.")]
+
+
+# ── Values ───────────────────────────────────────────────────────────────────
+
+def duration(value) -> float | None:
+    """Seconds from 90, "90s", "15m", "26h" or "2d"; None if it isn't one."""
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, (int, float)):
+        return float(value) if value >= 0 else None
+    m = _DURATION.match(str(value).strip().lower())
+    return float(m.group(1)) * _SECONDS[m.group(2)] if m else None
+
+
+@dataclass
+class Limit:
+    """A threshold: a number, or a share of the item's `of` ("10%")."""
+    value: float
+    share: bool = False
+
+
+def limit(value) -> Limit | None:
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, (int, float)):
+        return Limit(float(value))
+    text = str(value).strip()
+    if text.endswith("%"):
+        try:
+            return Limit(float(text[:-1]), share=True)
+        except ValueError:
+            return None
+    return None
+
+
+# ── The checker ──────────────────────────────────────────────────────────────
+
+class _Check:
+    def __init__(self):
+        self.problems: list[Problem] = []
+
+    def add(self, line: int, text: str) -> None:
+        self.problems.append(Problem(line, text))
+
+    def keys(self, m: _Map, allowed: tuple, required: tuple, where: str) -> bool:
+        ok = True
+        for key in m:
+            if key not in allowed:
+                self.add(m.lines[key], f"{where} has no `{key}`. It takes: {', '.join(allowed)}.")
+                ok = False
+        for key in required:
+            if key not in m:
+                self.add(m.line, f"{where} needs `{key}`.")
+                ok = False
+        return ok
+
+    def text(self, m: _Map, key: str, where: str, default: str = "") -> str:
+        if key not in m:
+            return default
+        value = m[key]
+        if isinstance(value, (dict, list)) or value is None:
+            self.add(m.lines[key], f"`{key}` in {where} should be text.")
+            return default
+        return str(value)
+
+    def metric(self, m: _Map, key: str, where: str) -> str:
+        value = m.get(key)
+        if not isinstance(value, str) or not _METRIC.match(value):
+            self.add(m.lines.get(key, m.line),
+                     f"`{key}` in {where} should be a metric name as Steward stores it, like "
+                     "fabledarchives_files (a counter is stored as <name>_per_sec).")
+            return ""
+        return value
+
+    def label(self, m: _Map, key: str, where: str) -> str:
+        if key not in m:
+            return ""
+        value = m[key]
+        if not isinstance(value, str) or not _LABEL.match(value):
+            self.add(m.lines[key], f"`{key}` in {where} should be a label name, like target.")
+            return ""
+        return value
+
+    def words(self, m: _Map, key: str, where: str) -> list[str]:
+        if key not in m:
+            return []
+        value = m[key]
+        if not isinstance(value, list) or not all(isinstance(v, (str, int)) for v in value):
+            self.add(m.lines[key], f"`{key}` in {where} should be a list, like [fresh, waiting].")
+            return []
+        if any(isinstance(v, bool) for v in value):
+            self.add(m.lines[key], f"`{key}` in {where} has a word YAML reads as true or false "
+                                   "(on, off, yes, no); put it in quotes, like 'off'.")
+            return []
+        return [str(v) for v in value]
+
+    def where(self, m: _Map, ctx: str) -> dict[str, str]:
+        if "where" not in m:
+            return {}
+        value = m["where"]
+        if not isinstance(value, dict) or not all(
+                isinstance(k, str) and _LABEL.match(k) and isinstance(v, (str, int, float))
+                and not isinstance(v, bool) for k, v in value.items()):
+            self.add(m.lines["where"], f"`where` in {ctx} should map labels to values, like "
+                                       "{kind: files}. Quote a value YAML would read as true or false.")
+            return {}
+        return {k: str(v) for k, v in value.items()}
+
+    def unit(self, m: _Map, ctx: str) -> str:
+        if "unit" not in m:
+            return ""
+        value = m["unit"]
+        if value not in UNITS:
+            self.add(m.lines["unit"], f"`unit` in {ctx} is one of: {', '.join(UNITS)}.")
+            return ""
+        return value
+
+    def span(self, m: _Map, key: str, ctx: str) -> float | None:
+        if key not in m:
+            return None
+        value = duration(m[key])
+        if value is None:
+            self.add(m.lines[key], f"`{key}` in {ctx} should be a span, like 26h, 90m or 2d.")
+        return value
+
+    def items(self, m: _Map, ctx: str) -> list[_Map]:
+        value = m.get("items")
+        if not isinstance(value, list) or not value:
+            self.add(m.lines.get("items", m.line), f"`items` in {ctx} should be a list of one or more items.")
+            return []
+        if len(value) > MAX_ITEMS:
+            self.add(m.lines["items"], f"{ctx} has {len(value)} items; at most {MAX_ITEMS}.")
+        out = []
+        for i, item in enumerate(value):
+            if not isinstance(item, dict):
+                self.add(value.lines[i], f"Each item in {ctx} should be a set of keys, like "
+                                         "{label: Disk free, metric: …}.")
+            else:
+                out.append(item)
+        return out
+
+
+# Each block: the keys it takes, those it needs, and its title when none is given.
+_BLOCKS = {
+    "states": (("title", "metric", "where", "name", "healthy", "quiet", "warning", "columns"),
+               ("metric", "healthy"), "States"),
+    "values": (("title", "items"), ("items",), "Figures"),
+    "times": (("title", "items"), ("items",), "Times"),
+    "table": (("title", "metric", "where", "rows", "columns", "unit"), ("metric", "rows", "columns"),
+              "Table"),
+    "info": (("title", "metric", "where", "labels"), ("metric",), "About"),
+}
+_VALUE_ITEM = ("label", "metric", "where", "name", "of", "unit", "warn_above", "crit_above",
+               "warn_below", "crit_below", "sparkline")
+_TIME_ITEM = ("label", "metric", "where", "name", "warn_after", "crit_after")
+_COLUMN = ("label", "metric", "unit")
+
+
+def _states(c: _Check, m: _Map, ctx: str) -> dict:
+    spec = {"metric": c.metric(m, "metric", ctx), "where": c.where(m, ctx), "name": c.label(m, "name", ctx),
+            "healthy": c.words(m, "healthy", ctx), "quiet": c.words(m, "quiet", ctx),
+            "warning": c.words(m, "warning", ctx), "columns": []}
+    if "healthy" in m and not spec["healthy"] and isinstance(m["healthy"], list):
+        c.add(m.lines["healthy"], f"`healthy` in {ctx} needs at least one state.")
+    seen: dict[str, str] = {}
+    for key in ("healthy", "quiet", "warning"):
+        for state in spec[key]:
+            if state in seen:
+                c.add(m.lines[key], f"“{state}” is in both `{seen[state]}` and `{key}` in {ctx}.")
+            seen[state] = key
+    if "columns" in m:
+        cols = m["columns"]
+        if not isinstance(cols, list):
+            c.add(m.lines["columns"], f"`columns` in {ctx} should be a list of {{label, metric}}.")
+        else:
+            for i, col in enumerate(cols):
+                where = f"a column of {ctx}"
+                if not isinstance(col, dict):
+                    c.add(cols.lines[i], f"Each of `columns` in {ctx} should be {{label, metric}}.")
+                    continue
+                if c.keys(col, _COLUMN, ("label", "metric"), where):
+                    spec["columns"].append({"label": c.text(col, "label", where),
+                                            "metric": c.metric(col, "metric", where),
+                                            "unit": c.unit(col, where)})
+    return spec
+
+
+def _value_item(c: _Check, item: _Map, ctx: str) -> dict | None:
+    where = f"an item of {ctx}"
+    if not c.keys(item, _VALUE_ITEM, ("label", "metric"), where):
+        return None
+    spec = {"label": c.text(item, "label", where), "metric": c.metric(item, "metric", where),
+            "where": c.where(item, where), "name": c.label(item, "name", where),
+            "of": c.metric(item, "of", where) if "of" in item else "", "unit": c.unit(item, where),
+            "sparkline": False}
+    if "sparkline" in item:
+        if not isinstance(item["sparkline"], bool):
+            c.add(item.lines["sparkline"], f"`sparkline` in {where} is true or false.")
+        spec["sparkline"] = item["sparkline"] is True
+    for key in ("warn_above", "crit_above", "warn_below", "crit_below"):
+        spec[key] = None
+        if key in item:
+            lim = limit(item[key])
+            if lim is None:
+                c.add(item.lines[key], f"`{key}` in {where} should be a number, or a share like 10%.")
+            elif lim.share and not spec["of"]:
+                c.add(item.lines[key], f"`{key}` in {where} is a share ({item[key]}), which needs `of`: "
+                                       "the metric it is a share of.")
+            spec[key] = lim
+    return spec
+
+
+def _time_item(c: _Check, item: _Map, ctx: str) -> dict | None:
+    where = f"an item of {ctx}"
+    if not c.keys(item, _TIME_ITEM, ("label", "metric"), where):
+        return None
+    spec = {"label": c.text(item, "label", where), "metric": c.metric(item, "metric", where),
+            "where": c.where(item, where), "name": c.label(item, "name", where),
+            "warn_after": c.span(item, "warn_after", where), "crit_after": c.span(item, "crit_after", where)}
+    if spec["warn_after"] and spec["crit_after"] and spec["crit_after"] < spec["warn_after"]:
+        c.add(item.lines["crit_after"], f"`crit_after` in {where} should be later than `warn_after`.")
+    return spec
+
+
+def _block(c: _Check, raw, line: int, n: int) -> Block | None:
+    if not isinstance(raw, dict) or len(raw) != 1:
+        c.add(line, f"Block {n} should be one kind with its settings under it, like "
+                    f"`- states: {{metric: …}}`. The kinds are: {', '.join(_BLOCKS)}.")
+        return None
+    kind, body = next(iter(raw.items()))
+    if kind not in _BLOCKS:
+        c.add(line, f"Block {n} is “{kind}”; the kinds are: {', '.join(_BLOCKS)}.")
+        return None
+    if not isinstance(body, dict):
+        c.add(line, f"Block {n} ({kind}) should have its settings under it, like `metric: …`.")
+        return None
+    allowed, required, default_title = _BLOCKS[kind]
+    ctx = f"block {n} ({kind})"
+    if not c.keys(body, allowed, required, ctx.capitalize()):
+        return None
+    title = c.text(body, "title", ctx, default_title)
+    if kind == "states":
+        spec = _states(c, body, ctx)
+    elif kind in ("values", "times"):
+        reader = _value_item if kind == "values" else _time_item
+        spec = {"items": [s for s in (reader(c, i, ctx) for i in c.items(body, ctx)) if s]}
+    elif kind == "table":
+        spec = {"metric": c.metric(body, "metric", ctx), "where": c.where(body, ctx),
+                "rows": c.label(body, "rows", ctx), "columns": c.label(body, "columns", ctx),
+                "unit": c.unit(body, ctx)}
+    else:
+        spec = {"metric": c.metric(body, "metric", ctx), "where": c.where(body, ctx),
+                "labels": c.words(body, "labels", ctx)}
+    return Block(kind, title, line, spec)
+
+
+_TOP = ("name", "description", "version", "format", "expects", "blocks")
+
+
+def check(text: str) -> Template:
+    """The template `text` describes. Raises TemplateError with every problem
+    found, each with its line, in the order they appear."""
+    raw, problems = _load(text)
+    if problems:
+        raise TemplateError(problems)
+    c = _Check()
+    if not isinstance(raw, dict):
+        raise TemplateError([Problem(1, "A template starts with its keys: name, version and blocks.")])
+    c.keys(raw, _TOP, ("name", "version", "blocks"), "A template")
+    name = c.text(raw, "name", "the template").strip()
+    if "name" in raw and not name:
+        c.add(raw.lines["name"], "Give the template a name.")
+    if len(name) > 80:
+        c.add(raw.lines["name"], "A name is at most 80 characters.")
+    version = raw.get("version")
+    if "version" in raw and (isinstance(version, bool) or not isinstance(version, int) or version < 1):
+        c.add(raw.lines["version"], "`version` is the template's own version: a whole number from 1. "
+                                    "Raise it when you change a template you have shared.")
+    if "format" in raw and raw["format"] != FORMAT:
+        c.add(raw.lines["format"], f"This Steward reads template format {FORMAT}.")
+    expects = c.text(raw, "expects", "the template")
+    if expects and not re.match(r"^[a-zA-Z_:][a-zA-Z0-9_:]*$", expects):
+        c.add(raw.lines["expects"], "`expects` should be the start of the metric names it reads, "
+                                    "like fabledarchives_.")
+    blocks: list[Block] = []
+    raw_blocks = raw.get("blocks")
+    if "blocks" in raw:
+        if not isinstance(raw_blocks, list) or not raw_blocks:
+            c.add(raw.lines["blocks"], "`blocks` should be a list of one or more blocks.")
+        else:
+            if len(raw_blocks) > MAX_BLOCKS:
+                c.add(raw.lines["blocks"], f"A template has at most {MAX_BLOCKS} blocks.")
+            for i, b in enumerate(raw_blocks):
+                block = _block(c, b, raw_blocks.lines[i], i + 1)
+                if block:
+                    blocks.append(block)
+    if c.problems:
+        raise TemplateError(sorted(c.problems, key=lambda p: p.line))
+    return Template(name=name, version=version, description=c.text(raw, "description", "the template"),
+                    expects=expects, blocks=blocks, text=text)
+
+
+def problems(text: str) -> list[Problem]:
+    """Every problem with `text`; empty when it is a template."""
+    try:
+        check(text)
+    except TemplateError as exc:
+        return exc.problems
+    return []

plugins/prometheus/template_view.py

@@ -0,0 +1,302 @@
+"""A widget template drawn from a target's newest values: what the
+Prometheus dashboard widget shows. Rows in, a view out; nothing is fetched
+here, so the widget route reads the database and a test passes rows.
+
+The view follows the widget shape (System 154): a verdict naming the worst
+thing, then each block with its problems first and the healthy rest after.
+Every row carries a tone: crit, warn, ok, or dim for a state the template
+calls quiet.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+
+from steward.core.formatting import human_duration
+
+from .service import fold_states, label_pairs, shown_value
+from .template_format import UNITS, Block, Limit, Template
+
+TONE_ORDER = {"crit": 0, "warn": 1, "ok": 2, "dim": 3}
+
+
+@dataclass
+class Sample:
+    resource: str
+    metric: str
+    labels: str              # as stored: "target=S,state=late"
+    pairs: dict[str, str]
+    value: float
+    at: datetime | None = None
+
+
+@dataclass
+class Row:
+    name: str
+    tone: str = "ok"
+    why: str = ""            # what is wrong, in words: "is late", "below 10%"
+    text: str = ""           # the value as shown, "—" when there is none
+    moment: datetime | None = None
+    bar: float | None = None  # a share of `of`, 0 to 1
+    cells: list[dict] = field(default_factory=list)
+    resource: str = ""
+    metric: str = ""
+    history: list[float] | None = None
+
+
+@dataclass
+class BlockView:
+    kind: str
+    title: str
+    tone: str = "ok"
+    rows: list[Row] = field(default_factory=list)
+    columns: list[str] = field(default_factory=list)
+    note: str = ""           # the block couldn't be drawn: no series, or not a state set
+
+
+@dataclass
+class View:
+    tone: str
+    verdict: str
+    blocks: list[BlockView]
+    wants: list[tuple[str, str]] = field(default_factory=list)   # (resource, metric) for sparklines
+    note: str = ""
+
+
+def samples(target: str, latest) -> dict[str, list[Sample]]:
+    """A target's newest values by metric. `latest` is PluginMetricLatest
+    rows (resource_name, metric_name, value, recorded_at) for the target and
+    its "<target>:<labels>" children."""
+    out: dict[str, list[Sample]] = {}
+    for row in latest:
+        if row.resource_name == target:
+            labels = ""
+        elif row.resource_name.startswith(target + ":"):
+            labels = row.resource_name[len(target) + 1:]
+        else:
+            continue
+        out.setdefault(row.metric_name, []).append(Sample(
+            row.resource_name, row.metric_name, labels, dict(label_pairs(labels)), row.value,
+            getattr(row, "recorded_at", None)))
+    return out
+
+
+def pick(index: dict[str, list[Sample]], metric: str, where: dict[str, str]) -> list[Sample]:
+    return [s for s in index.get(metric, []) if all(s.pairs.get(k) == v for k, v in where.items())]
+
+
+def shown(metric: str, unit: str, value: float | None) -> dict:
+    """{text, moment} for a value, in `unit` or else as its name says. A
+    count reads with thousands separated: 93,310."""
+    if unit == "count" and value is not None:
+        return {"text": f"{value:,.0f}" if value == int(value) else f"{value:,.2f}", "moment": None}
+    return shown_value(f"x{UNITS[unit]}" if unit else metric, value)
+
+
+def _worst(tones) -> str:
+    tones = [t for t in tones if t != "dim"]
+    return min(tones, key=TONE_ORDER.__getitem__) if tones else "ok"
+
+
+def _ordered(rows: list[Row]) -> list[Row]:
+    return sorted(rows, key=lambda r: (TONE_ORDER[r.tone], r.name.lower()))
+
+
+# ── Blocks ───────────────────────────────────────────────────────────────────
+
+def _states(block: Block, index, now) -> BlockView:
+    spec = block.spec
+    view = BlockView("states", block.title, columns=[c["label"] for c in spec["columns"]])
+    found = pick(index, spec["metric"], spec["where"])
+    folded = fold_states(spec["metric"], [{"labels": s.labels, "value": s.value, "resource": s.resource}
+                                          for s in found])
+    if not found:
+        view.note = f"Nothing reported for {spec['metric']}."
+    elif folded is None:
+        view.note = (f"{spec['metric']} isn't a state set: it should have one series per state, "
+                     "the current one 1 and the rest 0.")
+    for f in folded or []:
+        pairs = dict(label_pairs(f["labels"]))
+        name = pairs.get(spec["name"], "") if spec["name"] else ""
+        current = [s for s in f["state"].split(", ") if s]
+        tones = []
+        for state in current:
+            if state in spec["healthy"]:
+                tones.append("ok")
+            elif state in spec["quiet"]:
+                tones.append("dim")
+            elif state in spec["warning"]:
+                tones.append("warn")
+            else:
+                tones.append("crit")
+        tone = (min(tones, key=TONE_ORDER.__getitem__) if tones else "warn")
+        row = Row(name=name or f["labels"] or block.title, tone=tone, text=f["state"] or "—",
+                  why="" if tone in ("ok", "dim") else (f"is {f['state']}" if current else "reports no state"),
+                  resource=f["resource"], metric=spec["metric"])
+        for col in spec["columns"]:
+            match = [s for s in index.get(col["metric"], []) if s.labels == f["labels"]]
+            if not match and name:
+                match = [s for s in index.get(col["metric"], []) if s.pairs.get(spec["name"]) == name]
+            row.cells.append(shown(col["metric"], col["unit"], match[0].value if len(match) == 1 else None))
+        view.rows.append(row)
+    view.rows = _ordered(view.rows)
+    return view
+
+
+def _limit_text(lim: Limit, metric: str, unit: str) -> str:
+    return f"{lim.value:g}%" if lim.share else (shown(metric, unit, lim.value)["text"] or f"{lim.value:g}")
+
+
+def _judge(item: dict, value: float, of: float | None) -> tuple[str, str]:
+    for level in ("crit", "warn"):
+        for side in ("above", "below"):
+            lim = item[f"{level}_{side}"]
+            if lim is None:
+                continue
+            if lim.share:
+                if not of:
+                    continue
+                x = value / of * 100
+            else:
+                x = value
+            if (x > lim.value) if side == "above" else (x < lim.value):
+                return level, f"{side} {_limit_text(lim, item['metric'], item['unit'])}"
+    return "ok", ""
+
+
+def _rows_for(item: dict, index) -> list[tuple[str, Sample | None, str]]:
+    """The series an item shows: one per value of its `name` label, or the one
+    series it names. (name, sample, problem)."""
+    found = pick(index, item["metric"], item["where"])
+    if item["name"]:
+        named = [(s.pairs.get(item["name"], "") or item["label"], s, "") for s in found]
+        return named or [(item["label"], None, "not reported")]
+    if len(found) == 1:
+        return [(item["label"], found[0], "")]
+    if not found:
+        return [(item["label"], None, "not reported")]
+    return [(item["label"], None, f"{len(found)} series match; narrow it with `where` or `name`")]
+
+
+def _values(block: Block, index, now) -> tuple[BlockView, list]:
+    view, wants = BlockView("values", block.title), []
+    for item in block.spec["items"]:
+        for name, s, problem in _rows_for(item, index):
+            if s is None:
+                view.rows.append(Row(name=name, tone="warn", why=problem, text="—", metric=item["metric"]))
+                continue
+            row = Row(name=name, resource=s.resource, metric=s.metric, **shown(s.metric, item["unit"], s.value))
+            of = None
+            if item["of"]:
+                whole = [o for o in index.get(item["of"], []) if o.labels == s.labels]
+                if len(whole) == 1 and whole[0].value:
+                    of = whole[0].value
+                    row.bar = max(0.0, min(1.0, s.value / of))
+                    row.text = f"{row.text} of {shown(item['of'], item['unit'], of)['text']}"
+            row.tone, row.why = _judge(item, s.value, of)
+            if item["sparkline"]:
+                wants.append((s.resource, s.metric))
+            view.rows.append(row)
+    view.rows = _ordered(view.rows)
+    return view, wants
+
+
+def _times(block: Block, index, now) -> BlockView:
+    view = BlockView("times", block.title)
+    for item in block.spec["items"]:
+        for name, s, problem in _rows_for(item, index):
+            if s is None:
+                view.rows.append(Row(name=name, tone="warn", why=problem, text="—", metric=item["metric"]))
+                continue
+            got = shown_value("x_timestamp_seconds", s.value)
+            row = Row(name=name, resource=s.resource, metric=s.metric, **got)
+            if row.moment is None:
+                if item["warn_after"] or item["crit_after"]:
+                    row.tone, row.why = "warn", "never"
+            else:
+                age = (now - row.moment).total_seconds()
+                for level in ("crit", "warn"):
+                    after = item[f"{level}_after"]
+                    if after and age > after:
+                        row.tone, row.why = level, f"over {human_duration(after)} ago"
+                        break
+            view.rows.append(row)
+    view.rows = _ordered(view.rows)
+    return view
+
+
+def _table(block: Block, index, now) -> BlockView:
+    spec = block.spec
+    found = pick(index, spec["metric"], spec["where"])
+    rows: dict[str, dict[str, float]] = {}
+    columns: list[str] = []
+    for s in found:
+        r, c = s.pairs.get(spec["rows"]), s.pairs.get(spec["columns"])
+        if r is None or c is None:
+            continue
+        rows.setdefault(r, {})[c] = s.value
+        if c not in columns:
+            columns.append(c)
+    columns.sort()
+    view = BlockView("table", block.title, columns=columns)
+    if not rows:
+        view.note = (f"Nothing reported for {spec['metric']} with labels "
+                     f"{spec['rows']} and {spec['columns']}.")
+    for name in sorted(rows, key=str.lower):
+        view.rows.append(Row(name=name, cells=[shown(spec["metric"], spec["unit"], rows[name].get(c))
+                                               for c in columns]))
+    return view
+
+
+def _info(block: Block, index, now) -> BlockView:
+    spec = block.spec
+    view = BlockView("info", block.title)
+    found = sorted(pick(index, spec["metric"], spec["where"]), key=lambda s: s.labels)
+    if not found:
+        view.note = f"Nothing reported for {spec['metric']}."
+    for s in found:
+        keys = spec["labels"] or list(s.pairs)
+        view.rows.append(Row(name=" · ".join(f"{k} {s.pairs[k]}" for k in keys if s.pairs.get(k)),
+                             resource=s.resource, metric=s.metric))
+    return view
+
+
+_DRAW = {"states": _states, "times": _times, "table": _table, "info": _info}
+
+
+def render(template: Template, target: str, latest, now: datetime,
+           history: dict[tuple[str, str], list[float]] | None = None) -> View:
+    """The view of `template` over a target's newest values. `history` gives
+    the sparkline points the first pass asked for in `wants`."""
+    index = samples(target, latest)
+    blocks, wants = [], []
+    for block in template.blocks:
+        if block.kind == "values":
+            view, want = _values(block, index, now)
+            wants += want
+        else:
+            view = _DRAW[block.kind](block, index, now)
+        if history:
+            for row in view.rows:
+                row.history = history.get((row.resource, row.metric))
+        view.tone = "warn" if view.note else _worst(r.tone for r in view.rows)
+        blocks.append(view)
+
+    note = ""
+    if template.expects and not any(m.startswith(template.expects) for m in index):
+        note = (f"{target} reports no {template.expects}… metrics: is “{template.name}” "
+                "the template for it, or is the target filtering them out?")
+    problems = [(b, r) for b in blocks for r in b.rows if r.tone in ("crit", "warn")]
+    problems.sort(key=lambda p: TONE_ORDER[p[1].tone])
+    empty = [b for b in blocks if b.note]
+    tone = _worst([b.tone for b in blocks] + (["warn"] if note else []))
+    if note:
+        verdict = note
+    elif len(problems) + len(empty) == 0:
+        verdict = "All well."
+    else:
+        first = (f"{problems[0][0].title}: {problems[0][1].name} {problems[0][1].why}." if problems
+                 else f"{empty[0].title}: {empty[0].note}")
+        count = len(problems) + len(empty)
+        verdict = first if count == 1 else f"{count} need a look. {first}"
+    return View(tone=tone, verdict=verdict, blocks=blocks, wants=wants, note=note)

tests/plugins/prometheus/test_templates.py

@@ -0,0 +1,238 @@
+"""Prometheus widget templates (#5690): the checker says what is wrong and
+where, and the renderer turns a template and a target's newest values into a
+verdict, then blocks with their problems first."""
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace as NS
+
+import pytest
+
+from plugins.prometheus.template_format import TemplateError, check, duration, problems
+from plugins.prometheus.template_view import render
+
+NOW = datetime(2026, 10, 12, 19, 0, tzinfo=timezone.utc)
+T = "Archives"
+
+
+def before(**kw) -> float:
+    return (NOW - timedelta(**kw)).timestamp()
+
+
+def latest(series: dict) -> list:
+    """PluginMetricLatest-like rows from {(labels, metric): value}, labels
+    as stored after the target's name."""
+    return [NS(resource_name=f"{T}:{labels}" if labels else T, metric_name=metric, value=value,
+               recorded_at=NOW) for (labels, metric), value in series.items()]
+
+
+def states(prefix: str, current: str, all_states: list[str]) -> dict:
+    return {(f"{prefix},state={s}", "fa_target_state"): float(s == current) for s in all_states}
+
+
+TARGET_STATES = ["disabled", "fresh", "late", "missing", "waiting"]
+
+SERIES = {
+    **states("target=Steward,target_id=1", "fresh", TARGET_STATES),
+    **states("target=Photos,target_id=2", "late", TARGET_STATES),
+    **states("target=Old,target_id=3", "disabled", TARGET_STATES),
+    ("target=Steward,target_id=1", "fa_target_last_backup_timestamp_seconds"): before(hours=16),
+    ("target=Photos,target_id=2", "fa_target_last_backup_timestamp_seconds"): before(hours=30),
+    ("", "fa_disk_free_bytes"): 17.8e12,
+    ("", "fa_disk_size_bytes"): 46e12,
+    ("", "fa_files"): 94116.0,
+    ("", "fa_files_checked"): 93310.0,
+    ("", "fa_malware_held_files"): 2.0,
+    ("person=Bryan,person_id=9,kind=files", "fa_person_stored_bytes"): 4.7e11,
+    ("person=Bryan,person_id=9,kind=trash", "fa_person_stored_bytes"): 1.2e9,
+    ("person=Ana,person_id=8,kind=files", "fa_person_stored_bytes"): 3.0e10,
+    ("job=Nightly,job_id=5", "fa_job_next_run_timestamp_seconds"): (NOW + timedelta(hours=8)).timestamp(),
+    ("version=2026.10.12.1717,channel=dev,commit=3605d0a", "fa_build_info"): 1.0,
+}
+
+TEMPLATE = """\
+name: Test Archives
+version: 1
+expects: fa_
+blocks:
+  - states:
+      title: Backup targets
+      metric: fa_target_state
+      name: target
+      healthy: [fresh]
+      quiet: [disabled, waiting]
+      warning: [late]
+      columns:
+        - {label: Last backup, metric: fa_target_last_backup_timestamp_seconds}
+  - values:
+      title: Storage
+      items:
+        - {label: Disk free, metric: fa_disk_free_bytes, of: fa_disk_size_bytes, warn_below: 10%,
+           sparkline: true}
+        - {label: Files checked, metric: fa_files_checked, of: fa_files, unit: count}
+        - {label: Malware held, metric: fa_malware_held_files, unit: count, warn_above: 0}
+  - times:
+      items:
+        - {label: Next run, metric: fa_job_next_run_timestamp_seconds, name: job}
+        - {label: Last backup, metric: fa_target_last_backup_timestamp_seconds, name: target,
+           warn_after: 26h, crit_after: 3d}
+  - table:
+      title: Stored per person
+      metric: fa_person_stored_bytes
+      rows: person
+      columns: kind
+  - info:
+      metric: fa_build_info
+      labels: [version, commit]
+"""
+
+
+# ── The checker ──────────────────────────────────────────────────────────────
+
+def test_a_template_reads_into_its_blocks():
+    t = check(TEMPLATE)
+    assert (t.name, t.version, t.expects) == ("Test Archives", 1, "fa_")
+    assert [b.kind for b in t.blocks] == ["states", "values", "times", "table", "info"]
+    assert t.blocks[2].title == "Times"                       # a block's default title
+    disk = t.blocks[1].spec["items"][0]
+    assert disk["of"] == "fa_disk_size_bytes" and disk["warn_below"].share and disk["sparkline"]
+    assert t.blocks[2].spec["items"][1]["warn_after"] == 26 * 3600
+
+
+def _said(text: str) -> list[str]:
+    return [str(p) for p in problems(text)]
+
+
+def test_every_problem_is_said_with_its_line():
+    said = _said("""\
+name: X
+version: 0
+blocks:
+  - states:
+      metric: fa_state
+      healthy: [ok]
+      warning: [ok]
+      colour: red
+  - gauge: {metric: x}
+  - values:
+      items:
+        - {label: Free, metric: fa free, warn_below: 10%}
+""")
+    assert said == [
+        "Line 2: `version` is the template's own version: a whole number from 1. "
+        "Raise it when you change a template you have shared.",
+        "Line 8: Block 1 (states) has no `colour`. It takes: title, metric, where, name, healthy, "
+        "quiet, warning, columns.",
+        "Line 9: Block 2 is “gauge”; the kinds are: states, values, times, table, info.",
+        "Line 12: `metric` in an item of block 3 (values) should be a metric name as Steward stores it, "
+        "like fabledarchives_files (a counter is stored as <name>_per_sec).",
+        "Line 12: `warn_below` in an item of block 3 (values) is a share (10%), which needs `of`: "
+        "the metric it is a share of.",
+    ]
+
+
+def test_a_state_in_two_lists_and_a_yaml_boolean_are_caught():
+    said = _said("name: X\nversion: 1\nblocks:\n  - states: {metric: s_state, healthy: [ok], warning: [ok]}\n")
+    assert said == ["Line 4: “ok” is in both `healthy` and `warning` in block 1 (states)."]
+    said = _said("name: X\nversion: 1\nblocks:\n  - states: {metric: s_state, healthy: [on]}\n")
+    assert "put it in quotes, like 'off'" in said[0]
+
+
+def test_what_isnt_a_template_says_so():
+    broken = _said("name: [unclosed\n")
+    assert broken[0].startswith("Line ") and "This isn't valid YAML" in broken[0]
+    assert _said("just text") == ["Line 1: A template starts with its keys: name, version and blocks."]
+    assert _said("name: X\nname: Y\n") == ["Line 2: `name` is given twice."]
+    assert _said("name: X\n") == ["Line 1: A template needs `version`.", "Line 1: A template needs `blocks`."]
+    assert "Anchors and aliases" in _said("a: &x 1\nb: *x\n")[0]
+    assert "at most 64 KB" in _said("x" * (65 * 1024))[0]
+    with pytest.raises(TemplateError):
+        check("name: X\nversion: 1\nblocks: []\n")
+
+
+def test_spans():
+    assert [duration(v) for v in (90, "90s", "15m", "26h", "2d", "1.5h")] == [90, 90, 900, 93600, 172800, 5400]
+    assert duration("soon") is None and duration(-1) is None and duration(True) is None
+
+
+# ── The renderer ─────────────────────────────────────────────────────────────
+
+def _view(series=SERIES, text=TEMPLATE, history=None):
+    return render(check(text), T, latest(series), NOW, history=history)
+
+
+def test_the_verdict_names_the_worst_and_each_block_leads_with_its_problems():
+    view = _view()
+    # Photos is late, its last backup is past 26h, and malware is held: three warnings.
+    assert view.tone == "warn"
+    assert view.verdict == "3 need a look. Backup targets: Photos is late."
+    targets = view.blocks[0]
+    assert [(r.name, r.tone, r.text) for r in targets.rows] == [
+        ("Photos", "warn", "late"), ("Steward", "ok", "fresh"), ("Old", "dim", "disabled")]
+    assert targets.rows[0].why == "is late" and targets.columns == ["Last backup"]
+    assert targets.rows[1].cells[0]["moment"] == datetime.fromtimestamp(before(hours=16), timezone.utc)
+    assert targets.tone == "warn"
+
+
+def test_one_problem_is_the_verdict_itself():
+    view = _view({**SERIES, ("", "fa_malware_held_files"): 0.0,
+                  ("target=Photos,target_id=2", "fa_target_last_backup_timestamp_seconds"): before(hours=2)})
+    assert view.verdict == "Backup targets: Photos is late."
+
+
+def test_values_show_shares_bars_and_thresholds():
+    storage = _view().blocks[1]
+    rows = {r.name: r for r in storage.rows}
+    assert rows["Disk free"].text == "16.2 TiB of 41.8 TiB" and round(rows["Disk free"].bar, 2) == 0.39
+    assert rows["Disk free"].tone == "ok"
+    assert rows["Files checked"].text == "93,310 of 94,116"
+    assert (rows["Malware held"].tone, rows["Malware held"].why) == ("warn", "above 0")
+    assert storage.rows[0].name == "Malware held"                       # problems first
+    low = _view({**SERIES, ("", "fa_disk_free_bytes"): 2e12}).blocks[1]
+    assert {r.name: (r.tone, r.why) for r in low.rows}["Disk free"] == ("warn", "below 10%")
+
+
+def test_a_sparkline_is_asked_for_then_drawn():
+    view = _view()
+    assert view.wants == [(T, "fa_disk_free_bytes")]
+    drawn = _view(history={(T, "fa_disk_free_bytes"): [1.0, 2.0]})
+    assert {r.name: r.history for r in drawn.blocks[1].rows}["Disk free"] == [1.0, 2.0]
+
+
+def test_times_read_past_and_future_and_flag_the_overdue():
+    times = _view().blocks[2]
+    rows = {(r.name, r.metric): r for r in times.rows}
+    photos = rows[("Photos", "fa_target_last_backup_timestamp_seconds")]
+    assert (photos.tone, photos.why) == ("warn", "over 1d 2h ago")
+    nightly = rows[("Nightly", "fa_job_next_run_timestamp_seconds")]
+    assert nightly.tone == "ok" and nightly.moment > NOW                # "in 8h"
+    stale = _view({**SERIES, ("target=Photos,target_id=2", "fa_target_last_backup_timestamp_seconds"):
+                   before(days=4)}).blocks[2]
+    assert stale.rows[0].tone == "crit" and stale.rows[0].why == "over 3d ago"
+
+
+def test_a_table_pivots_two_labels_and_info_shows_its_labels():
+    table, info = _view().blocks[3:]
+    assert table.columns == ["files", "trash"]
+    assert [(r.name, [c["text"] for c in r.cells]) for r in table.rows] == [
+        ("Ana", ["27.9 GiB", "—"]), ("Bryan", ["437.7 GiB", "1.1 GiB"])]
+    assert [r.name for r in info.rows] == ["version 2026.10.12.1717 · commit 3605d0a"]
+
+
+def test_missing_series_are_said_not_hidden():
+    bare = {k: v for k, v in SERIES.items() if k[1] != "fa_malware_held_files" and k[1] != "fa_build_info"}
+    view = _view(bare)
+    malware = {r.name: r for r in view.blocks[1].rows}["Malware held"]
+    assert (malware.text, malware.tone, malware.why) == ("—", "warn", "not reported")
+    assert view.blocks[4].note == "Nothing reported for fa_build_info." and view.blocks[4].tone == "warn"
+
+
+def test_a_template_for_another_target_says_so():
+    view = _view({("", "node_load1"): 0.5})
+    assert view.tone == "warn" and "reports no fa_… metrics" in view.verdict
+
+
+def test_all_well():
+    healthy = {**SERIES, **states("target=Photos,target_id=2", "fresh", TARGET_STATES),
+               ("target=Photos,target_id=2", "fa_target_last_backup_timestamp_seconds"): before(hours=2),
+               ("", "fa_malware_held_files"): 0.0}
+    view = _view(healthy)
+    assert (view.tone, view.verdict) == ("ok", "All well.")