bvandeusenandClaude Opus 5.5
ec3e5e7e0d
#5689 "Prometheus target page: fold state sets, show epoch times, size bytes", step 1 of milestone 508 "Prometheus widget templates". - A state set (name ending in _state, a state label, every series 0 or 1; or the OpenMetrics form, the label named after the metric) reads as one row per thing it describes, naming the current state, the others on hover. fabledarchives_jobs{state} is a count and stays as it is. - label_pairs / state_label / fold_states in service.py, for the widget templates to reuse. - After a good scrape, series it no longer has lose their newest value (forget_gone): Archives' old build_info row read as current beside the new one. History stays; a failed scrape forgets nothing; alert rules keep their own last-sample time. - Epoch times and _bytes sizes were already read by shown_value / metric_value. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
183 lines
8.7 KiB
Python
183 lines
8.7 KiB
Python
"""The Prometheus plugin's pure parts: which series are stored and under what
|
|
names, the form, and what a page says about the newest scrape (#5668)."""
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from plugins.prometheus.models import PrometheusTarget
|
|
from plugins.prometheus.scheduler import is_due
|
|
from plugins.prometheus.scraper import ScrapeError, scrape
|
|
from plugins.prometheus.service import (FormError, fold_states, label_pairs, map_families, parse_form,
|
|
patterns, rate_name, resource_for, target_state, wanted)
|
|
from steward.core import prometheus
|
|
from steward.core.counter_rates import CounterRates
|
|
|
|
TEXT = """\
|
|
# TYPE fa_backup_target_state gauge
|
|
fa_backup_target_state{target="Steward",state="fresh"} 1
|
|
fa_backup_target_state{target="Steward",state="late"} 0
|
|
# TYPE fa_http_requests_total counter
|
|
fa_http_requests_total{route="GET /",status="200"} 100
|
|
# TYPE fa_job_seconds histogram
|
|
fa_job_seconds_bucket{le="1"} 2
|
|
fa_job_seconds_bucket{le="+Inf"} 3
|
|
fa_job_seconds_sum 4.5
|
|
fa_job_seconds_count 3
|
|
# TYPE fa_rpc summary
|
|
fa_rpc{quantile="0.5"} 0.2
|
|
fa_rpc_count 9
|
|
fa_disk_free_bytes 1e9
|
|
fa_broken NaN
|
|
other_thing 7
|
|
"""
|
|
|
|
|
|
def _mapped(keep=(), drop=(), cap=1000, target="Archives"):
|
|
return map_families(target, prometheus.parse(TEXT), list(keep), list(drop), cap)
|
|
|
|
|
|
def test_gauges_stay_counters_become_rates_and_distributions_keep_sum_and_count():
|
|
got = {(s.resource, s.metric): (s.value, s.counter) for s in _mapped().series}
|
|
assert got == {
|
|
("Archives:target=Steward,state=fresh", "fa_backup_target_state"): (1.0, False),
|
|
("Archives:target=Steward,state=late", "fa_backup_target_state"): (0.0, False),
|
|
("Archives:route=GET /,status=200", "fa_http_requests_per_sec"): (100.0, True),
|
|
("Archives", "fa_job_seconds_sum_per_sec"): (4.5, True),
|
|
("Archives", "fa_job_seconds_count_per_sec"): (3.0, True),
|
|
("Archives", "fa_rpc_count_per_sec"): (9.0, True),
|
|
("Archives", "fa_disk_free_bytes"): (1e9, False),
|
|
("Archives", "other_thing"): (7.0, False),
|
|
}
|
|
|
|
|
|
def test_keep_and_drop_patterns_choose_families_and_drop_wins():
|
|
m = _mapped(keep=["fa_*"], drop=["fa_http_*"])
|
|
assert {s.metric for s in m.series} == {
|
|
"fa_backup_target_state", "fa_job_seconds_sum_per_sec", "fa_job_seconds_count_per_sec",
|
|
"fa_rpc_count_per_sec", "fa_disk_free_bytes"}
|
|
assert m.dropped == 2 # the counter's one sample, other_thing's one
|
|
assert wanted("x", [], []) and not wanted("x", ["y*"], []) and not wanted("y1", ["y*"], ["y1"])
|
|
|
|
|
|
def test_the_series_cap_and_overlong_names_are_counted_not_stored():
|
|
m = _mapped(cap=3)
|
|
assert len(m.series) == 3 and m.capped == 5
|
|
long = map_families("t" * 250, prometheus.parse('m{a="bcdefgh"} 1\n'), [], [], 10)
|
|
assert long.series == [] and long.capped == 1
|
|
|
|
|
|
def test_names():
|
|
assert rate_name("jobs_total") == "jobs_per_sec"
|
|
assert rate_name("job_seconds_sum") == "job_seconds_sum_per_sec"
|
|
assert resource_for("T", ()) == "T"
|
|
assert resource_for("T", (("a", "1"), ("b", "x"))) == "T:a=1,b=x"
|
|
assert patterns("a_*, b_*\n\n a_* \nc") == ["a_*", "b_*", "c"]
|
|
|
|
|
|
def test_labels_read_back_as_stored_even_with_a_comma_in_a_value():
|
|
labels = resource_for("A", (("job", "Nightly"), ("schedule", "0 3 * * 1,3,5"), ("state", "ok")))
|
|
assert label_pairs(labels[2:]) == [("job", "Nightly"), ("schedule", "0 3 * * 1,3,5"), ("state", "ok")]
|
|
assert label_pairs("") == []
|
|
|
|
|
|
def _rows(metric_rows):
|
|
return [{"labels": labels, "value": value, "resource": f"A:{labels}"} for labels, value in metric_rows]
|
|
|
|
|
|
def test_a_state_set_folds_to_its_current_state_per_thing():
|
|
rows = _rows([("target=S,state=late", 0), ("target=S,state=fresh", 1), ("target=S,state=missing", 0),
|
|
("target=T,state=fresh", 0), ("target=T,state=late", 1), ("target=T,state=missing", 1),
|
|
("target=U,state=fresh", 0), ("target=U,state=late", 0)])
|
|
folded = fold_states("fa_backup_target_state", rows)
|
|
assert [(f["labels"], f["state"]) for f in folded] == [
|
|
("target=S", "fresh"), ("target=T", "late, missing"), ("target=U", "")]
|
|
assert folded[0]["states"] == ["fresh", "late", "missing"]
|
|
assert folded[0]["resource"] == "A:target=S,state=fresh" # the current one's chart
|
|
assert folded[2]["resource"] == "A:target=U,state=fresh" # none current: the first
|
|
# OpenMetrics names a stateset's label after the metric itself.
|
|
assert fold_states("fa_mode", _rows([("fa_mode=a", 1), ("fa_mode=b", 0)]))[0]["state"] == "a"
|
|
|
|
|
|
def test_only_a_state_set_is_folded():
|
|
# A count split by a state label reads 0 or 1 often enough; its name says it is a count.
|
|
assert fold_states("fa_jobs", _rows([("kind=scan,state=waiting", 1), ("kind=scan,state=running", 0)])) is None
|
|
assert fold_states("fa_state", _rows([("state=a", 1), ("state=b", 2)])) is None # not 0/1
|
|
assert fold_states("fa_state", _rows([("state=a", 1), ("target=S", 0)])) is None # a row without one
|
|
assert fold_states("fa_state", []) is None
|
|
|
|
|
|
def test_a_counter_becomes_a_rate_and_a_reset_is_skipped():
|
|
rates = CounterRates()
|
|
t0 = datetime(2026, 10, 12, 12, 0, tzinfo=timezone.utc)
|
|
key = ("t1", "Archives", "x_per_sec")
|
|
assert rates.rate(key, 100, t0) is None
|
|
assert rates.rate(key, 160, t0 + timedelta(seconds=60)) == 1.0
|
|
assert rates.rate(key, 5, t0 + timedelta(seconds=120)) is None # the service restarted
|
|
assert rates.rate(key, 35, t0 + timedelta(seconds=150)) == 1.0
|
|
rates.forget("t1")
|
|
assert rates.rate(key, 99, t0 + timedelta(seconds=180)) is None
|
|
|
|
|
|
class _Form(dict):
|
|
def getlist(self, key):
|
|
v = self.get(key)
|
|
return v if isinstance(v, list) else ([v] if v else [])
|
|
|
|
|
|
def test_the_form_checks_what_it_stores_and_keeps_a_blank_token():
|
|
values = parse_form(_Form(name="Archives", url="http://archives:8081/metrics", token="s3cret",
|
|
interval_seconds="30", series_cap="500", keep="fabledarchives_*",
|
|
enabled="on", verify_tls="on"))
|
|
assert values["token"].startswith("enc:") or values["token"] == "s3cret"
|
|
assert (values["interval_seconds"], values["series_cap"], values["keep"], values["enabled"]) == (
|
|
30, 500, ["fabledarchives_*"], True)
|
|
existing = PrometheusTarget(name="Archives", url="http://a/metrics", token="enc:v1:kept")
|
|
again = parse_form(_Form(name="Archives", url="http://a/metrics"), existing)
|
|
assert again["token"] == "enc:v1:kept" and again["enabled"] is False
|
|
cleared = parse_form(_Form(name="Archives", url="http://a/metrics", clear_token="on"), existing)
|
|
assert cleared["token"] == ""
|
|
for bad in (dict(name="", url="http://a"), dict(name="a:b", url="http://a"),
|
|
dict(name="a", url="ftp://a"), dict(name="a", url="http://a", interval_seconds="5"),
|
|
dict(name="a", url="http://a", series_cap="0")):
|
|
with pytest.raises(FormError):
|
|
parse_form(_Form(bad))
|
|
|
|
|
|
def test_state_and_due():
|
|
now = datetime(2026, 10, 12, 12, 0, tzinfo=timezone.utc)
|
|
t = PrometheusTarget(name="a", url="http://a", enabled=True, interval_seconds=60)
|
|
assert target_state(t, now).reach == "pending" and is_due(t, now)
|
|
t.last_scraped_at = now - timedelta(seconds=30)
|
|
assert target_state(t, now).reach == "up" and not is_due(t, now)
|
|
t.last_scraped_at = now - timedelta(seconds=58)
|
|
assert is_due(t, now) # half a tick of slack
|
|
t.last_error = "Couldn't connect: refused."
|
|
t.last_scraped_at = now - timedelta(seconds=30)
|
|
assert (target_state(t, now).reach, target_state(t, now).says) == ("down", "Couldn't connect: refused.")
|
|
t.last_scraped_at = now - timedelta(minutes=10)
|
|
assert target_state(t, now).reach == "stale"
|
|
t.enabled = False
|
|
assert target_state(t, now).reach == "off" and not is_due(t, now)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_a_scrape_sends_the_token_and_says_why_it_failed():
|
|
seen = {}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
seen["auth"] = request.headers.get("Authorization")
|
|
if request.url.path == "/denied":
|
|
return httpx.Response(401)
|
|
if request.url.path == "/html":
|
|
return httpx.Response(200, text="<html>hello</html>")
|
|
return httpx.Response(200, text="up_thing 1\n")
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
fams = await scrape("http://svc/metrics", "tok", transport=transport)
|
|
assert list(fams) == ["up_thing"] and seen["auth"] == "Bearer tok"
|
|
with pytest.raises(ScrapeError, match="token was refused"):
|
|
await scrape("http://svc/denied", "bad", transport=transport)
|
|
with pytest.raises(ScrapeError, match="no metrics"):
|
|
await scrape("http://svc/html", transport=transport)
|