FabledSteward/plugins/prometheus/service.py

"""Prometheus targets: the form a person edits, the row it becomes, how a
scrape's series become Steward metrics, and what a page says about the
newest scrape."""
from __future__ import annotations

import math
from dataclasses import dataclass, field
from datetime import datetime
from fnmatch import fnmatchcase

from steward.core import prometheus
from steward.core.crypto import encrypt_secret

from .models import PrometheusTarget

SOURCE = "prometheus"

# What each target records about its own scrape, under its own name.
IS_UP = "is_up"
SCRAPE_MS = "scrape_ms"
SERIES_KEPT = "series_kept"
SERIES_CAPPED = "series_capped"
OWN_METRICS = (IS_UP, SCRAPE_MS, SERIES_KEPT, SERIES_CAPPED)

# Column widths of plugin_metrics; a longer name can't be stored.
RESOURCE_MAX = 255
METRIC_MAX = 128

INTERVAL_MIN, INTERVAL_MAX = 10, 3600
CAP_MIN, CAP_MAX = 1, 10000


class FormError(ValueError):
    """What is wrong with a submitted target form, said to the person."""


# ── Series → metrics ─────────────────────────────────────────────────────────

def wanted(name: str, keep: list[str], drop: list[str]) -> bool:
    """Whether a family is stored: it matches a keep pattern (or there are
    none) and no drop pattern. Drop wins over keep."""
    if keep and not any(fnmatchcase(name, p) for p in keep):
        return False
    return not any(fnmatchcase(name, p) for p in drop)


def resource_for(target: str, labels: tuple[tuple[str, str], ...]) -> str:
    """The resource a series is stored under: the target's name, then its
    labels as key=value, so "Archives:target=Steward,state=late". A series
    without labels is the target itself."""
    if not labels:
        return target
    return target + ":" + ",".join(f"{k}={v}" for k, v in labels)


def rate_name(sample_name: str) -> str:
    """A counter is stored as a per-second rate: requests_total →
    requests_per_sec, job_seconds_sum → job_seconds_sum_per_sec."""
    base = sample_name[: -len("_total")] if sample_name.endswith("_total") else sample_name
    return base + "_per_sec"


@dataclass
class Series:
    resource: str
    metric: str
    value: float
    counter: bool


@dataclass
class Mapped:
    series: list[Series] = field(default_factory=list)
    dropped: int = 0     # left out by the keep/drop patterns
    capped: int = 0      # past the series cap, or a name too long to store


def map_families(target: str, families: dict[str, prometheus.Family], keep: list[str],
                 drop: list[str], cap: int) -> Mapped:
    """The series of one scrape that Steward stores, in the endpoint's order.

    Gauges and untyped samples are stored as they are; counters (and a
    histogram's or summary's _sum and _count) are marked to become rates.
    Histogram buckets and summary quantiles are skipped: Steward stores one
    number per series, not a distribution. NaN and infinite values are
    skipped as unmeasured."""
    out = Mapped()
    for family in families.values():
        if not wanted(family.name, keep, drop):
            out.dropped += len(family.samples)
            continue
        for sample in family.samples:
            counter = family.type == "counter"
            if family.type in ("histogram", "summary"):
                if not sample.name.endswith(("_sum", "_count")):
                    continue
                counter = True
            if math.isnan(sample.value) or math.isinf(sample.value):
                continue
            metric = rate_name(sample.name) if counter else sample.name
            resource = resource_for(target, sample.labels)
            if len(out.series) >= cap or len(metric) > METRIC_MAX or len(resource) > RESOURCE_MAX:
                out.capped += 1
                continue
            out.series.append(Series(resource, metric, sample.value, counter))
    return out


# ── The form ─────────────────────────────────────────────────────────────────

def patterns(text: str) -> list[str]:
    """Name patterns from a textarea: one per line or comma-separated."""
    out: list[str] = []
    for line in (text or "").replace(",", "\n").splitlines():
        p = line.strip()
        if p and p not in out:
            out.append(p)
    return out


def form_values(target: PrometheusTarget | None) -> dict:
    """The form's fields for an existing target. The token is never sent back
    to the browser; a blank token field on save keeps the stored one."""
    if target is None:
        return {"interval_seconds": 60, "series_cap": 1000, "verify_tls": True, "enabled": True,
                "keep": "", "drop": ""}
    return {"name": target.name, "url": target.url, "interval_seconds": target.interval_seconds,
            "series_cap": target.series_cap, "verify_tls": target.verify_tls,
            "enabled": target.enabled, "keep": "\n".join(target.keep or []),
            "drop": "\n".join(target.drop or []), "has_token": bool(target.token)}


def _bounded(form, key: str, default: int, low: int, high: int, says: str) -> int:
    try:
        value = int(form.get(key) or default)
    except ValueError:
        raise FormError(says) from None
    if not low <= value <= high:
        raise FormError(says)
    return value


def parse_form(form, existing: PrometheusTarget | None = None) -> dict:
    """The submitted form as the columns to store, the token encrypted.
    Raises FormError saying what to fix."""
    name = (form.get("name") or "").strip()
    url = (form.get("url") or "").strip()
    if not name:
        raise FormError("Give the target a name.")
    if len(name) > 100 or any(c in name for c in "/:,"):
        raise FormError("A name is up to 100 characters, without / : or a comma.")
    if not url.startswith(("http://", "https://")) or len(url) > 500:
        raise FormError("The address starts with http:// or https://, e.g. http://archives:8081/metrics.")
    interval = _bounded(form, "interval_seconds", 60, INTERVAL_MIN, INTERVAL_MAX,
                        f"Scrape every {INTERVAL_MIN} to {INTERVAL_MAX} seconds.")
    cap = _bounded(form, "series_cap", 1000, CAP_MIN, CAP_MAX,
                   f"The series cap is between {CAP_MIN} and {CAP_MAX}.")
    remove_token = bool(form.get("clear_token"))
    typed = (form.get("token") or "").strip()
    if typed:
        token = encrypt_secret(typed)
    elif existing is not None and not remove_token:
        token = existing.token
    else:
        token = ""
    return {"name": name, "url": url, "token": token,
            "verify_tls": bool(form.get("verify_tls")), "interval_seconds": interval,
            "keep": patterns(form.get("keep") or ""), "drop": patterns(form.get("drop") or ""),
            "series_cap": cap, "enabled": bool(form.get("enabled"))}


# ── What the newest scrape says ──────────────────────────────────────────────

# A target not scraped for this many intervals has stopped being scraped
# (plugin turned off, scheduler stuck), whatever its last outcome said.
STALE_SCRAPES = 3


@dataclass
class TargetState:
    reach: str                 # up | down | stale | pending | off
    says: str                  # one line, in words


def target_state(target: PrometheusTarget, now: datetime) -> TargetState:
    if not target.enabled:
        return TargetState("off", "Not scraped: turned off.")
    if target.last_scraped_at is None:
        return TargetState("pending", "Not scraped yet.")
    if (now - target.last_scraped_at).total_seconds() > STALE_SCRAPES * max(target.interval_seconds, 1):
        return TargetState("stale", "Not scraped recently: the Prometheus plugin may be turned off.")
    if target.last_error:
        return TargetState("down", target.last_error)
    return TargetState("up", "Answering.")