bvandeusenandClaude Opus 5.5
4d95c5d0c2
#5690 "Widget template format: YAML schema, validator and renderer", step 2 of milestone 508 "Prometheus widget templates". - template_format.check(text) -> Template, or TemplateError with every problem, each with its line and what to change. Blocks: states, values (of -> share and bar, thresholds as numbers or shares, sparkline), times (warn_after / crit_after), table (two labels pivoted), info. Anchors/aliases refused, 64 KB cap, YAML's on/off/yes/no flagged. - template_view.render(template, target, latest, now, history) -> View: a verdict naming the worst problem, then each block with its problems first (crit, warn, ok, then quiet states). Missing series show as "—" and say so; a template whose `expects` prefix the target lacks says so. - shown_value moves to service.py so the renderer can use it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
270 lines
11 KiB
Python
270 lines
11 KiB
Python
"""Prometheus targets: the form a person edits, the row it becomes, how a
|
|
scrape's series become Steward metrics, and what a page says about the
|
|
newest scrape."""
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from fnmatch import fnmatchcase
|
|
|
|
from steward.core import prometheus
|
|
from steward.core.crypto import encrypt_secret
|
|
from steward.core.formatting import metric_value
|
|
|
|
from .models import PrometheusTarget
|
|
|
|
SOURCE = "prometheus"
|
|
|
|
# What each target records about its own scrape, under its own name.
|
|
IS_UP = "is_up"
|
|
SCRAPE_MS = "scrape_ms"
|
|
SERIES_KEPT = "series_kept"
|
|
SERIES_CAPPED = "series_capped"
|
|
OWN_METRICS = (IS_UP, SCRAPE_MS, SERIES_KEPT, SERIES_CAPPED)
|
|
|
|
# Column widths of plugin_metrics; a longer name can't be stored.
|
|
RESOURCE_MAX = 255
|
|
METRIC_MAX = 128
|
|
|
|
INTERVAL_MIN, INTERVAL_MAX = 10, 3600
|
|
CAP_MIN, CAP_MAX = 1, 10000
|
|
|
|
|
|
class FormError(ValueError):
|
|
"""What is wrong with a submitted target form, said to the person."""
|
|
|
|
|
|
# ── Series → metrics ─────────────────────────────────────────────────────────
|
|
|
|
def wanted(name: str, keep: list[str], drop: list[str]) -> bool:
|
|
"""Whether a family is stored: it matches a keep pattern (or there are
|
|
none) and no drop pattern. Drop wins over keep."""
|
|
if keep and not any(fnmatchcase(name, p) for p in keep):
|
|
return False
|
|
return not any(fnmatchcase(name, p) for p in drop)
|
|
|
|
|
|
def resource_for(target: str, labels: tuple[tuple[str, str], ...]) -> str:
|
|
"""The resource a series is stored under: the target's name, then its
|
|
labels as key=value, so "Archives:target=Steward,state=late". A series
|
|
without labels is the target itself."""
|
|
if not labels:
|
|
return target
|
|
return target + ":" + ",".join(f"{k}={v}" for k, v in labels)
|
|
|
|
|
|
def rate_name(sample_name: str) -> str:
|
|
"""A counter is stored as a per-second rate: requests_total →
|
|
requests_per_sec, job_seconds_sum → job_seconds_sum_per_sec."""
|
|
base = sample_name[: -len("_total")] if sample_name.endswith("_total") else sample_name
|
|
return base + "_per_sec"
|
|
|
|
|
|
def label_pairs(labels: str) -> list[tuple[str, str]]:
|
|
"""resource_for's labels read back: "target=S,state=late" →
|
|
[("target", "S"), ("state", "late")]. A piece without "=" was part of the
|
|
value before it, a value that held a comma."""
|
|
pairs: list[tuple[str, str]] = []
|
|
for piece in labels.split(",") if labels else []:
|
|
key, eq, value = piece.partition("=")
|
|
if eq and key:
|
|
pairs.append((key, value))
|
|
elif pairs:
|
|
pairs[-1] = (pairs[-1][0], pairs[-1][1] + "," + piece)
|
|
return pairs
|
|
|
|
|
|
def state_label(metric: str, pairs: list[tuple[str, str]]) -> str | None:
|
|
"""Which label names a state set's state: `state`, or the metric's own
|
|
name as OpenMetrics writes a stateset (foo{foo="a"} 1)."""
|
|
keys = {k for k, _ in pairs}
|
|
for key in ("state", metric):
|
|
if key in keys:
|
|
return key
|
|
return None
|
|
|
|
|
|
def fold_states(metric: str, rows: list[dict]) -> list[dict] | None:
|
|
"""A state set's series folded to one row per thing it describes.
|
|
|
|
A state set is one series per possible state, the current one 1 and the
|
|
rest 0: Archives' fabledarchives_backup_target_state{target, state}. Each
|
|
row in `rows` has `labels` (resource_for's form), `value` and `resource`.
|
|
A metric is folded when its name ends in _state (or its state label is
|
|
its own name) and every series carries the state label and reads 0 or 1;
|
|
anything else is None, and shown series by series. The name matters: a
|
|
count split by a `state` label (fabledarchives_jobs{state="waiting"})
|
|
often reads 0 or 1 too, and is not a state set.
|
|
|
|
Each folded row: `labels` without the state, `state` (the one set to 1;
|
|
several are joined, none is ""), `states` (every state, in order) and
|
|
`resource` (the current state's series, else the first)."""
|
|
if not rows:
|
|
return None
|
|
sets: dict[str, list[tuple[str, dict]]] = {}
|
|
for row in rows:
|
|
pairs = label_pairs(row["labels"])
|
|
key = state_label(metric, pairs)
|
|
if key is None or row["value"] not in (0, 1):
|
|
return None
|
|
if key == "state" and not metric.endswith("_state"):
|
|
return None
|
|
rest = ",".join(f"{k}={v}" for k, v in pairs if k != key)
|
|
sets.setdefault(rest, []).append((dict(pairs)[key], row))
|
|
folded = []
|
|
for rest, members in sorted(sets.items()):
|
|
members.sort(key=lambda m: m[0])
|
|
current = [(s, r) for s, r in members if r["value"] == 1]
|
|
folded.append({"labels": rest, "state": ", ".join(s for s, _ in current),
|
|
"states": [s for s, _ in members],
|
|
"resource": (current or members)[0][1]["resource"]})
|
|
return folded
|
|
|
|
|
|
def shown_value(metric: str, value: float | None) -> dict:
|
|
"""A series' value in its unit, from the Prometheus naming conventions.
|
|
A *_timestamp_seconds is a moment: it comes back as `moment` for the page
|
|
to show as "3m ago" in the reader's zone; 0 there means "never"."""
|
|
if value is None:
|
|
return {"text": "—", "moment": None}
|
|
if metric.endswith("_timestamp_seconds"):
|
|
if value <= 0:
|
|
return {"text": "never", "moment": None}
|
|
return {"text": "", "moment": datetime.fromtimestamp(value, timezone.utc)}
|
|
return {"text": metric_value(metric, value), "moment": None}
|
|
|
|
|
|
@dataclass
|
|
class Series:
|
|
resource: str
|
|
metric: str
|
|
value: float
|
|
counter: bool
|
|
|
|
|
|
@dataclass
|
|
class Mapped:
|
|
series: list[Series] = field(default_factory=list)
|
|
dropped: int = 0 # left out by the keep/drop patterns
|
|
capped: int = 0 # past the series cap, or a name too long to store
|
|
|
|
|
|
def map_families(target: str, families: dict[str, prometheus.Family], keep: list[str],
|
|
drop: list[str], cap: int) -> Mapped:
|
|
"""The series of one scrape that Steward stores, in the endpoint's order.
|
|
|
|
Gauges and untyped samples are stored as they are; counters (and a
|
|
histogram's or summary's _sum and _count) are marked to become rates.
|
|
Histogram buckets and summary quantiles are skipped: Steward stores one
|
|
number per series, not a distribution. NaN and infinite values are
|
|
skipped as unmeasured."""
|
|
out = Mapped()
|
|
for family in families.values():
|
|
if not wanted(family.name, keep, drop):
|
|
out.dropped += len(family.samples)
|
|
continue
|
|
for sample in family.samples:
|
|
counter = family.type == "counter"
|
|
if family.type in ("histogram", "summary"):
|
|
if not sample.name.endswith(("_sum", "_count")):
|
|
continue
|
|
counter = True
|
|
if math.isnan(sample.value) or math.isinf(sample.value):
|
|
continue
|
|
metric = rate_name(sample.name) if counter else sample.name
|
|
resource = resource_for(target, sample.labels)
|
|
if len(out.series) >= cap or len(metric) > METRIC_MAX or len(resource) > RESOURCE_MAX:
|
|
out.capped += 1
|
|
continue
|
|
out.series.append(Series(resource, metric, sample.value, counter))
|
|
return out
|
|
|
|
|
|
# ── The form ─────────────────────────────────────────────────────────────────
|
|
|
|
def patterns(text: str) -> list[str]:
|
|
"""Name patterns from a textarea: one per line or comma-separated."""
|
|
out: list[str] = []
|
|
for line in (text or "").replace(",", "\n").splitlines():
|
|
p = line.strip()
|
|
if p and p not in out:
|
|
out.append(p)
|
|
return out
|
|
|
|
|
|
def form_values(target: PrometheusTarget | None) -> dict:
|
|
"""The form's fields for an existing target. The token is never sent back
|
|
to the browser; a blank token field on save keeps the stored one."""
|
|
if target is None:
|
|
return {"interval_seconds": 60, "series_cap": 1000, "verify_tls": True, "enabled": True,
|
|
"keep": "", "drop": ""}
|
|
return {"name": target.name, "url": target.url, "interval_seconds": target.interval_seconds,
|
|
"series_cap": target.series_cap, "verify_tls": target.verify_tls,
|
|
"enabled": target.enabled, "keep": "\n".join(target.keep or []),
|
|
"drop": "\n".join(target.drop or []), "has_token": bool(target.token)}
|
|
|
|
|
|
def _bounded(form, key: str, default: int, low: int, high: int, says: str) -> int:
|
|
try:
|
|
value = int(form.get(key) or default)
|
|
except ValueError:
|
|
raise FormError(says) from None
|
|
if not low <= value <= high:
|
|
raise FormError(says)
|
|
return value
|
|
|
|
|
|
def parse_form(form, existing: PrometheusTarget | None = None) -> dict:
|
|
"""The submitted form as the columns to store, the token encrypted.
|
|
Raises FormError saying what to fix."""
|
|
name = (form.get("name") or "").strip()
|
|
url = (form.get("url") or "").strip()
|
|
if not name:
|
|
raise FormError("Give the target a name.")
|
|
if len(name) > 100 or any(c in name for c in "/:,"):
|
|
raise FormError("A name is up to 100 characters, without / : or a comma.")
|
|
if not url.startswith(("http://", "https://")) or len(url) > 500:
|
|
raise FormError("The address starts with http:// or https://, e.g. http://archives:8081/metrics.")
|
|
interval = _bounded(form, "interval_seconds", 60, INTERVAL_MIN, INTERVAL_MAX,
|
|
f"Scrape every {INTERVAL_MIN} to {INTERVAL_MAX} seconds.")
|
|
cap = _bounded(form, "series_cap", 1000, CAP_MIN, CAP_MAX,
|
|
f"The series cap is between {CAP_MIN} and {CAP_MAX}.")
|
|
remove_token = bool(form.get("clear_token"))
|
|
typed = (form.get("token") or "").strip()
|
|
if typed:
|
|
token = encrypt_secret(typed)
|
|
elif existing is not None and not remove_token:
|
|
token = existing.token
|
|
else:
|
|
token = ""
|
|
return {"name": name, "url": url, "token": token,
|
|
"verify_tls": bool(form.get("verify_tls")), "interval_seconds": interval,
|
|
"keep": patterns(form.get("keep") or ""), "drop": patterns(form.get("drop") or ""),
|
|
"series_cap": cap, "enabled": bool(form.get("enabled"))}
|
|
|
|
|
|
# ── What the newest scrape says ──────────────────────────────────────────────
|
|
|
|
# A target not scraped for this many intervals has stopped being scraped
|
|
# (plugin turned off, scheduler stuck), whatever its last outcome said.
|
|
STALE_SCRAPES = 3
|
|
|
|
|
|
@dataclass
|
|
class TargetState:
|
|
reach: str # up | down | stale | pending | off
|
|
says: str # one line, in words
|
|
|
|
|
|
def target_state(target: PrometheusTarget, now: datetime) -> TargetState:
|
|
if not target.enabled:
|
|
return TargetState("off", "Not scraped: turned off.")
|
|
if target.last_scraped_at is None:
|
|
return TargetState("pending", "Not scraped yet.")
|
|
if (now - target.last_scraped_at).total_seconds() > STALE_SCRAPES * max(target.interval_seconds, 1):
|
|
return TargetState("stale", "Not scraped recently: the Prometheus plugin may be turned off.")
|
|
if target.last_error:
|
|
return TargetState("down", target.last_error)
|
|
return TargetState("up", "Answering.")
|