bvandeusenandClaude Opus 5.5
f7a856cb90
The connection's job is keeping Home Assistant running. Each poll now reads open
repairs, Supervisor health, disk, recovery mode and add-ons that should be running
over its websocket, for the page, the overview and the problems_open, disk_free_pct
and update_failed metrics. That needs a websocket client, so websockets joins the
dependencies.
A weekly slot, set on the page or through MCP and off until then, installs add-ons,
then Core, then the OS, each after its own confirmed backup, ending at the first
failure. The slot arithmetic moves from the host agent's cadence to steward.core.weekly
so both share it.
The device scrape goes: the page's Metrics section and a target's token_from from
4c9ed02 are removed, and a migration deletes targets using it, as they'd otherwise ask
with no token. A refused token still stops a scrape.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
284 lines
12 KiB
Python
284 lines
12 KiB
Python
"""Prometheus targets: the form a person edits, the row it becomes, how a
|
|
scrape's series become Steward metrics, and what a page says about the
|
|
newest scrape."""
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from fnmatch import fnmatchcase
|
|
|
|
from steward.core import prometheus
|
|
from steward.core.crypto import encrypt_secret
|
|
from steward.core.formatting import metric_value
|
|
|
|
from .models import PrometheusTarget
|
|
|
|
SOURCE = "prometheus"
|
|
|
|
# What each target records about its own scrape, under its own name.
|
|
IS_UP = "is_up"
|
|
SCRAPE_MS = "scrape_ms"
|
|
SERIES_KEPT = "series_kept"
|
|
SERIES_CAPPED = "series_capped"
|
|
OWN_METRICS = (IS_UP, SCRAPE_MS, SERIES_KEPT, SERIES_CAPPED)
|
|
|
|
# Column widths of plugin_metrics; a longer name can't be stored.
|
|
RESOURCE_MAX = 255
|
|
METRIC_MAX = 128
|
|
|
|
INTERVAL_MIN, INTERVAL_MAX = 10, 3600
|
|
CAP_MIN, CAP_MAX = 1, 10000
|
|
|
|
|
|
class FormError(ValueError):
|
|
"""What is wrong with a submitted target form, said to the person."""
|
|
|
|
|
|
# ── Series → metrics ─────────────────────────────────────────────────────────
|
|
|
|
def wanted(name: str, keep: list[str], drop: list[str]) -> bool:
|
|
"""Whether a family is stored: it matches a keep pattern (or there are
|
|
none) and no drop pattern. Drop wins over keep."""
|
|
if keep and not any(fnmatchcase(name, p) for p in keep):
|
|
return False
|
|
return not any(fnmatchcase(name, p) for p in drop)
|
|
|
|
|
|
def resource_for(target: str, labels: tuple[tuple[str, str], ...]) -> str:
|
|
"""The resource a series is stored under: the target's name, then its
|
|
labels as key=value, so "Archives:target=Steward,state=late". A series
|
|
without labels is the target itself."""
|
|
if not labels:
|
|
return target
|
|
return target + ":" + ",".join(f"{k}={v}" for k, v in labels)
|
|
|
|
|
|
def rate_name(sample_name: str) -> str:
|
|
"""A counter is stored as a per-second rate: requests_total →
|
|
requests_per_sec, job_seconds_sum → job_seconds_sum_per_sec."""
|
|
base = sample_name[: -len("_total")] if sample_name.endswith("_total") else sample_name
|
|
return base + "_per_sec"
|
|
|
|
|
|
def label_pairs(labels: str) -> list[tuple[str, str]]:
|
|
"""resource_for's labels read back: "target=S,state=late" →
|
|
[("target", "S"), ("state", "late")]. A piece without "=" was part of the
|
|
value before it, a value that held a comma."""
|
|
pairs: list[tuple[str, str]] = []
|
|
for piece in labels.split(",") if labels else []:
|
|
key, eq, value = piece.partition("=")
|
|
if eq and key:
|
|
pairs.append((key, value))
|
|
elif pairs:
|
|
pairs[-1] = (pairs[-1][0], pairs[-1][1] + "," + piece)
|
|
return pairs
|
|
|
|
|
|
def state_label(metric: str, pairs: list[tuple[str, str]]) -> str | None:
|
|
"""Which label names a state set's state: `state`, or the metric's own
|
|
name as OpenMetrics writes a stateset (foo{foo="a"} 1)."""
|
|
keys = {k for k, _ in pairs}
|
|
for key in ("state", metric):
|
|
if key in keys:
|
|
return key
|
|
return None
|
|
|
|
|
|
def fold_states(metric: str, rows: list[dict]) -> list[dict] | None:
|
|
"""A state set's series folded to one row per thing it describes.
|
|
|
|
A state set is one series per possible state, the current one 1 and the
|
|
rest 0: Archives' fabledarchives_backup_target_state{target, state}. Each
|
|
row in `rows` has `labels` (resource_for's form), `value` and `resource`.
|
|
A metric is folded when its name ends in _state (or its state label is
|
|
its own name) and every series carries the state label and reads 0 or 1;
|
|
anything else is None, and shown series by series. The name matters: a
|
|
count split by a `state` label (fabledarchives_jobs{state="waiting"})
|
|
often reads 0 or 1 too, and is not a state set.
|
|
|
|
Each folded row: `labels` without the state, `state` (the one set to 1;
|
|
several are joined, none is ""), `states` (every state, in order) and
|
|
`resource` (the current state's series, else the first)."""
|
|
if not rows:
|
|
return None
|
|
sets: dict[str, list[tuple[str, dict]]] = {}
|
|
for row in rows:
|
|
pairs = label_pairs(row["labels"])
|
|
key = state_label(metric, pairs)
|
|
if key is None or row["value"] not in (0, 1):
|
|
return None
|
|
if key == "state" and not metric.endswith("_state"):
|
|
return None
|
|
rest = ",".join(f"{k}={v}" for k, v in pairs if k != key)
|
|
sets.setdefault(rest, []).append((dict(pairs)[key], row))
|
|
folded = []
|
|
for rest, members in sorted(sets.items()):
|
|
members.sort(key=lambda m: m[0])
|
|
current = [(s, r) for s, r in members if r["value"] == 1]
|
|
folded.append({"labels": rest, "state": ", ".join(s for s, _ in current),
|
|
"states": [s for s, _ in members],
|
|
"resource": (current or members)[0][1]["resource"]})
|
|
return folded
|
|
|
|
|
|
# A series named for a moment holds Unix seconds, not a span of time: the
|
|
# convention's *_timestamp_seconds, and the *_time_seconds some exporters use
|
|
# (Home Assistant's homeassistant_last_updated_time_seconds).
|
|
MOMENT_SUFFIXES = ("_timestamp_seconds", "_time_seconds")
|
|
|
|
|
|
def is_moment(metric: str) -> bool:
|
|
return metric.endswith(MOMENT_SUFFIXES)
|
|
|
|
|
|
def shown_value(metric: str, value: float | None) -> dict:
|
|
"""A series' value in its unit, from the Prometheus naming conventions.
|
|
A moment (is_moment) comes back as `moment` for the page to show as
|
|
"3m ago" in the reader's zone; 0 there means "never"."""
|
|
if value is None:
|
|
return {"text": "—", "moment": None}
|
|
if is_moment(metric):
|
|
if value <= 0:
|
|
return {"text": "never", "moment": None}
|
|
return {"text": "", "moment": datetime.fromtimestamp(value, timezone.utc)}
|
|
return {"text": metric_value(metric, value), "moment": None}
|
|
|
|
|
|
@dataclass
|
|
class Series:
|
|
resource: str
|
|
metric: str
|
|
value: float
|
|
counter: bool
|
|
|
|
|
|
@dataclass
|
|
class Mapped:
|
|
series: list[Series] = field(default_factory=list)
|
|
dropped: int = 0 # left out by the keep/drop patterns
|
|
capped: int = 0 # past the series cap, or a name too long to store
|
|
|
|
|
|
def map_families(target: str, families: dict[str, prometheus.Family], keep: list[str],
|
|
drop: list[str], cap: int) -> Mapped:
|
|
"""The series of one scrape that Steward stores, in the endpoint's order.
|
|
|
|
Gauges and untyped samples are stored as they are; counters (and a
|
|
histogram's or summary's _sum and _count) are marked to become rates.
|
|
Histogram buckets and summary quantiles are skipped: Steward stores one
|
|
number per series, not a distribution. NaN and infinite values are
|
|
skipped as unmeasured."""
|
|
out = Mapped()
|
|
for family in families.values():
|
|
if not wanted(family.name, keep, drop):
|
|
out.dropped += len(family.samples)
|
|
continue
|
|
for sample in family.samples:
|
|
counter = family.type == "counter"
|
|
if family.type in ("histogram", "summary"):
|
|
if not sample.name.endswith(("_sum", "_count")):
|
|
continue
|
|
counter = True
|
|
if math.isnan(sample.value) or math.isinf(sample.value):
|
|
continue
|
|
metric = rate_name(sample.name) if counter else sample.name
|
|
resource = resource_for(target, sample.labels)
|
|
if len(out.series) >= cap or len(metric) > METRIC_MAX or len(resource) > RESOURCE_MAX:
|
|
out.capped += 1
|
|
continue
|
|
out.series.append(Series(resource, metric, sample.value, counter))
|
|
return out
|
|
|
|
|
|
# ── The form ─────────────────────────────────────────────────────────────────
|
|
|
|
def patterns(text: str) -> list[str]:
|
|
"""Name patterns from a textarea: one per line, or separated by commas or
|
|
spaces. A metric name never holds whitespace, so a paste that turned line
|
|
breaks into spaces still reads as the patterns it was."""
|
|
out: list[str] = []
|
|
for p in (text or "").replace(",", " ").split():
|
|
if p not in out:
|
|
out.append(p)
|
|
return out
|
|
|
|
|
|
def form_values(target: PrometheusTarget | None) -> dict:
|
|
"""The form's fields for an existing target. The token is never sent back
|
|
to the browser; a blank token field on save keeps the stored one."""
|
|
if target is None:
|
|
return {"interval_seconds": 60, "series_cap": 1000, "verify_tls": True, "keep": "", "drop": ""}
|
|
return {"name": target.name, "url": target.url, "interval_seconds": target.interval_seconds,
|
|
"series_cap": target.series_cap, "verify_tls": target.verify_tls,
|
|
"keep": "\n".join(target.keep or []),
|
|
"drop": "\n".join(target.drop or []), "has_token": bool(target.token)}
|
|
|
|
|
|
def _bounded(form, key: str, default: int, low: int, high: int, says: str) -> int:
|
|
try:
|
|
value = int(form.get(key) or default)
|
|
except ValueError:
|
|
raise FormError(says) from None
|
|
if not low <= value <= high:
|
|
raise FormError(says)
|
|
return value
|
|
|
|
|
|
def parse_form(form, existing: PrometheusTarget | None = None) -> dict:
|
|
"""The submitted form as the columns to store, the token encrypted.
|
|
Whether it is paused isn't among them: Pause/Resume sets that, and a new
|
|
target starts scraping. Raises FormError saying what to fix."""
|
|
name = (form.get("name") or "").strip()
|
|
url = (form.get("url") or "").strip()
|
|
if not name:
|
|
raise FormError("Give the target a name.")
|
|
if len(name) > 100 or any(c in name for c in "/:,"):
|
|
raise FormError("A name is up to 100 characters, without / : or a comma.")
|
|
if not url.startswith(("http://", "https://")) or len(url) > 500:
|
|
raise FormError("The address starts with http:// or https://, e.g. http://archives:8081/metrics.")
|
|
interval = _bounded(form, "interval_seconds", 60, INTERVAL_MIN, INTERVAL_MAX,
|
|
f"Scrape every {INTERVAL_MIN} to {INTERVAL_MAX} seconds.")
|
|
cap = _bounded(form, "series_cap", 1000, CAP_MIN, CAP_MAX,
|
|
f"The series cap is between {CAP_MIN} and {CAP_MAX}.")
|
|
remove_token = bool(form.get("clear_token"))
|
|
typed = (form.get("token") or "").strip()
|
|
if typed:
|
|
token = encrypt_secret(typed)
|
|
elif existing is not None and not remove_token:
|
|
token = existing.token
|
|
else:
|
|
token = ""
|
|
return {"name": name, "url": url, "token": token,
|
|
"verify_tls": bool(form.get("verify_tls")), "interval_seconds": interval,
|
|
"keep": patterns(form.get("keep") or ""), "drop": patterns(form.get("drop") or ""),
|
|
"series_cap": cap}
|
|
|
|
|
|
# ── What the newest scrape says ──────────────────────────────────────────────
|
|
|
|
# A target not scraped for this many intervals has stopped being scraped
|
|
# (plugin turned off, scheduler stuck), whatever its last outcome said.
|
|
STALE_SCRAPES = 3
|
|
|
|
|
|
@dataclass
|
|
class TargetState:
|
|
reach: str # up | down | stale | pending | paused
|
|
says: str # one line, in words
|
|
|
|
|
|
def target_state(target: PrometheusTarget, now: datetime) -> TargetState:
|
|
if not target.enabled:
|
|
return TargetState("paused", "Paused: not scraped until it's resumed.")
|
|
if target.refused:
|
|
return TargetState("down", f"{target.last_error} Steward stopped asking, as some services ban an "
|
|
"address after a few refusals. Save a new token to start again.")
|
|
if target.last_scraped_at is None:
|
|
return TargetState("pending", "Not scraped yet.")
|
|
if (now - target.last_scraped_at).total_seconds() > STALE_SCRAPES * max(target.interval_seconds, 1):
|
|
return TargetState("stale", "Not scraped recently: the Prometheus plugin may be turned off.")
|
|
if target.last_error:
|
|
return TargetState("down", target.last_error)
|
|
return TargetState("up", "Answering.")
|