"""Prometheus targets: the form a person edits, the row it becomes, how a scrape's series become Steward metrics, and what a page says about the newest scrape.""" from __future__ import annotations import math from dataclasses import dataclass, field from datetime import datetime, timezone from fnmatch import fnmatchcase from steward.core import prometheus from steward.core.crypto import encrypt_secret from steward.core.formatting import metric_value from .models import PrometheusTarget SOURCE = "prometheus" # What each target records about its own scrape, under its own name. IS_UP = "is_up" SCRAPE_MS = "scrape_ms" SERIES_KEPT = "series_kept" SERIES_CAPPED = "series_capped" OWN_METRICS = (IS_UP, SCRAPE_MS, SERIES_KEPT, SERIES_CAPPED) # Column widths of plugin_metrics; a longer name can't be stored. RESOURCE_MAX = 255 METRIC_MAX = 128 INTERVAL_MIN, INTERVAL_MAX = 10, 3600 CAP_MIN, CAP_MAX = 1, 10000 class FormError(ValueError): """What is wrong with a submitted target form, said to the person.""" # ── Series → metrics ───────────────────────────────────────────────────────── def wanted(name: str, keep: list[str], drop: list[str]) -> bool: """Whether a family is stored: it matches a keep pattern (or there are none) and no drop pattern. Drop wins over keep.""" if keep and not any(fnmatchcase(name, p) for p in keep): return False return not any(fnmatchcase(name, p) for p in drop) def resource_for(target: str, labels: tuple[tuple[str, str], ...]) -> str: """The resource a series is stored under: the target's name, then its labels as key=value, so "Archives:target=Steward,state=late". A series without labels is the target itself.""" if not labels: return target return target + ":" + ",".join(f"{k}={v}" for k, v in labels) def rate_name(sample_name: str) -> str: """A counter is stored as a per-second rate: requests_total → requests_per_sec, job_seconds_sum → job_seconds_sum_per_sec.""" base = sample_name[: -len("_total")] if sample_name.endswith("_total") else sample_name return base + "_per_sec" def label_pairs(labels: str) -> list[tuple[str, str]]: """resource_for's labels read back: "target=S,state=late" → [("target", "S"), ("state", "late")]. A piece without "=" was part of the value before it, a value that held a comma.""" pairs: list[tuple[str, str]] = [] for piece in labels.split(",") if labels else []: key, eq, value = piece.partition("=") if eq and key: pairs.append((key, value)) elif pairs: pairs[-1] = (pairs[-1][0], pairs[-1][1] + "," + piece) return pairs def state_label(metric: str, pairs: list[tuple[str, str]]) -> str | None: """Which label names a state set's state: `state`, or the metric's own name as OpenMetrics writes a stateset (foo{foo="a"} 1).""" keys = {k for k, _ in pairs} for key in ("state", metric): if key in keys: return key return None def fold_states(metric: str, rows: list[dict]) -> list[dict] | None: """A state set's series folded to one row per thing it describes. A state set is one series per possible state, the current one 1 and the rest 0: Archives' fabledarchives_backup_target_state{target, state}. Each row in `rows` has `labels` (resource_for's form), `value` and `resource`. A metric is folded when its name ends in _state (or its state label is its own name) and every series carries the state label and reads 0 or 1; anything else is None, and shown series by series. The name matters: a count split by a `state` label (fabledarchives_jobs{state="waiting"}) often reads 0 or 1 too, and is not a state set. Each folded row: `labels` without the state, `state` (the one set to 1; several are joined, none is ""), `states` (every state, in order) and `resource` (the current state's series, else the first).""" if not rows: return None sets: dict[str, list[tuple[str, dict]]] = {} for row in rows: pairs = label_pairs(row["labels"]) key = state_label(metric, pairs) if key is None or row["value"] not in (0, 1): return None if key == "state" and not metric.endswith("_state"): return None rest = ",".join(f"{k}={v}" for k, v in pairs if k != key) sets.setdefault(rest, []).append((dict(pairs)[key], row)) folded = [] for rest, members in sorted(sets.items()): members.sort(key=lambda m: m[0]) current = [(s, r) for s, r in members if r["value"] == 1] folded.append({"labels": rest, "state": ", ".join(s for s, _ in current), "states": [s for s, _ in members], "resource": (current or members)[0][1]["resource"]}) return folded def shown_value(metric: str, value: float | None) -> dict: """A series' value in its unit, from the Prometheus naming conventions. A *_timestamp_seconds is a moment: it comes back as `moment` for the page to show as "3m ago" in the reader's zone; 0 there means "never".""" if value is None: return {"text": "—", "moment": None} if metric.endswith("_timestamp_seconds"): if value <= 0: return {"text": "never", "moment": None} return {"text": "", "moment": datetime.fromtimestamp(value, timezone.utc)} return {"text": metric_value(metric, value), "moment": None} @dataclass class Series: resource: str metric: str value: float counter: bool @dataclass class Mapped: series: list[Series] = field(default_factory=list) dropped: int = 0 # left out by the keep/drop patterns capped: int = 0 # past the series cap, or a name too long to store def map_families(target: str, families: dict[str, prometheus.Family], keep: list[str], drop: list[str], cap: int) -> Mapped: """The series of one scrape that Steward stores, in the endpoint's order. Gauges and untyped samples are stored as they are; counters (and a histogram's or summary's _sum and _count) are marked to become rates. Histogram buckets and summary quantiles are skipped: Steward stores one number per series, not a distribution. NaN and infinite values are skipped as unmeasured.""" out = Mapped() for family in families.values(): if not wanted(family.name, keep, drop): out.dropped += len(family.samples) continue for sample in family.samples: counter = family.type == "counter" if family.type in ("histogram", "summary"): if not sample.name.endswith(("_sum", "_count")): continue counter = True if math.isnan(sample.value) or math.isinf(sample.value): continue metric = rate_name(sample.name) if counter else sample.name resource = resource_for(target, sample.labels) if len(out.series) >= cap or len(metric) > METRIC_MAX or len(resource) > RESOURCE_MAX: out.capped += 1 continue out.series.append(Series(resource, metric, sample.value, counter)) return out # ── The form ───────────────────────────────────────────────────────────────── def patterns(text: str) -> list[str]: """Name patterns from a textarea: one per line or comma-separated.""" out: list[str] = [] for line in (text or "").replace(",", "\n").splitlines(): p = line.strip() if p and p not in out: out.append(p) return out def form_values(target: PrometheusTarget | None) -> dict: """The form's fields for an existing target. The token is never sent back to the browser; a blank token field on save keeps the stored one.""" if target is None: return {"interval_seconds": 60, "series_cap": 1000, "verify_tls": True, "enabled": True, "keep": "", "drop": ""} return {"name": target.name, "url": target.url, "interval_seconds": target.interval_seconds, "series_cap": target.series_cap, "verify_tls": target.verify_tls, "enabled": target.enabled, "keep": "\n".join(target.keep or []), "drop": "\n".join(target.drop or []), "has_token": bool(target.token)} def _bounded(form, key: str, default: int, low: int, high: int, says: str) -> int: try: value = int(form.get(key) or default) except ValueError: raise FormError(says) from None if not low <= value <= high: raise FormError(says) return value def parse_form(form, existing: PrometheusTarget | None = None) -> dict: """The submitted form as the columns to store, the token encrypted. Raises FormError saying what to fix.""" name = (form.get("name") or "").strip() url = (form.get("url") or "").strip() if not name: raise FormError("Give the target a name.") if len(name) > 100 or any(c in name for c in "/:,"): raise FormError("A name is up to 100 characters, without / : or a comma.") if not url.startswith(("http://", "https://")) or len(url) > 500: raise FormError("The address starts with http:// or https://, e.g. http://archives:8081/metrics.") interval = _bounded(form, "interval_seconds", 60, INTERVAL_MIN, INTERVAL_MAX, f"Scrape every {INTERVAL_MIN} to {INTERVAL_MAX} seconds.") cap = _bounded(form, "series_cap", 1000, CAP_MIN, CAP_MAX, f"The series cap is between {CAP_MIN} and {CAP_MAX}.") remove_token = bool(form.get("clear_token")) typed = (form.get("token") or "").strip() if typed: token = encrypt_secret(typed) elif existing is not None and not remove_token: token = existing.token else: token = "" return {"name": name, "url": url, "token": token, "verify_tls": bool(form.get("verify_tls")), "interval_seconds": interval, "keep": patterns(form.get("keep") or ""), "drop": patterns(form.get("drop") or ""), "series_cap": cap, "enabled": bool(form.get("enabled"))} # ── What the newest scrape says ────────────────────────────────────────────── # A target not scraped for this many intervals has stopped being scraped # (plugin turned off, scheduler stuck), whatever its last outcome said. STALE_SCRAPES = 3 @dataclass class TargetState: reach: str # up | down | stale | pending | off says: str # one line, in words def target_state(target: PrometheusTarget, now: datetime) -> TargetState: if not target.enabled: return TargetState("off", "Not scraped: turned off.") if target.last_scraped_at is None: return TargetState("pending", "Not scraped yet.") if (now - target.last_scraped_at).total_seconds() > STALE_SCRAPES * max(target.interval_seconds, 1): return TargetState("stale", "Not scraped recently: the Prometheus plugin may be turned off.") if target.last_error: return TargetState("down", target.last_error) return TargetState("up", "Answering.")