feat(prometheus): fold state sets on the target page; forget series an endpoint drops (#5689)
#5689 "Prometheus target page: fold state sets, show epoch times, size bytes", step 1 of milestone 508 "Prometheus widget templates". - A state set (name ending in _state, a state label, every series 0 or 1; or the OpenMetrics form, the label named after the metric) reads as one row per thing it describes, naming the current state, the others on hover. fabledarchives_jobs{state} is a count and stays as it is. - label_pairs / state_label / fold_states in service.py, for the widget templates to reuse. - After a good scrape, series it no longer has lose their newest value (forget_gone): Archives' old build_info row read as current beside the new one. History stays; a failed scrape forgets nothing; alert rules keep their own last-sample time. - Epoch times and _bytes sizes were already read by shown_value / metric_value. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
"""Integration: a Prometheus scrape writes each target's outcome to its row,
|
||||
its kept series to the metric tables (a counter as a rate), and its own
|
||||
is_up, which an alert rule can watch (#5668). Requires STEWARD_DATABASE_URL."""
|
||||
is_up, which an alert rule can watch (#5668). A series a good scrape no
|
||||
longer reports loses its newest value (#5689); a failed scrape keeps them.
|
||||
Requires STEWARD_DATABASE_URL."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
@@ -14,7 +16,7 @@ pytestmark = pytest.mark.integration
|
||||
|
||||
TEXT = """\
|
||||
# TYPE fa_backup_target_state gauge
|
||||
fa_backup_target_state{target="Steward",state="late"} 0
|
||||
fa_backup_target_state{target="Steward",state="STATE"} 1
|
||||
# TYPE fa_requests_total counter
|
||||
fa_requests_total COUNT
|
||||
# TYPE noise gauge
|
||||
@@ -57,13 +59,14 @@ def test_a_scrape_records_series_rates_and_a_down_target_fires_its_rule(app):
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
up, down = f"arch-{tag}", f"gone-{tag}"
|
||||
counts = iter([1_000, 1_011])
|
||||
states = iter(["late", "fresh"])
|
||||
tokens = []
|
||||
|
||||
async def fake_scrape(url, token, verify_tls):
|
||||
tokens.append(token)
|
||||
if "gone" in url:
|
||||
raise ScrapeError("Couldn't connect: connection refused.")
|
||||
return prometheus.parse(TEXT.replace("COUNT", str(next(counts))))
|
||||
return prometheus.parse(TEXT.replace("COUNT", str(next(counts))).replace("STATE", next(states)))
|
||||
|
||||
async def latest(name_prefix):
|
||||
async with app.db_sessionmaker() as s:
|
||||
@@ -117,7 +120,10 @@ def test_a_scrape_records_series_rates_and_a_down_target_fires_its_rule(app):
|
||||
|
||||
first, second, rows, state, down_latest = asyncio.run(go())
|
||||
assert "scrape-token" in tokens
|
||||
assert first[(f"{up}:target=Steward,state=late", "fa_backup_target_state")] == 0.0
|
||||
assert first[(f"{up}:target=Steward,state=late", "fa_backup_target_state")] == 1.0
|
||||
# The second scrape no longer reports "late": its newest value is forgotten.
|
||||
assert (f"{up}:target=Steward,state=late", "fa_backup_target_state") not in second
|
||||
assert second[(f"{up}:target=Steward,state=fresh", "fa_backup_target_state")] == 1.0
|
||||
assert (up, "fa_requests_per_sec") not in first # a counter needs two scrapes
|
||||
assert first[(up, "is_up")] == 1.0 and first[(up, "series_kept")] == 2.0
|
||||
assert not any(m == "noise" for _, m in first) # not in the keep list
|
||||
|
||||
Reference in New Issue
Block a user