feat(prometheus): scrape any /metrics endpoint into Steward's metrics, with pages (#5668, #5669)
CI / lint (push) Successful in 2s
CI / integration (push) Failing after 1m12s
CI / unit (push) Failing after 1m27s
CI / publish (push) Skipped

A bundled prometheus plugin, on by default like the other protocol plugins.
Targets are rows edited on its page: address, bearer token (encrypted),
interval, keep/leave-out name patterns and a series cap. Steward does the
filtering; the endpoint serves everything (Archives' ruling).

- A 5 s tick scrapes due targets at once, each within a 30 s deadline, and
  writes the outcome to its row and its series through record_metrics, so
  alert rules can watch any of them.
- Gauges are stored as they are; counters (and a histogram's or summary's
  _sum/_count) as <name>_per_sec rates from the second scrape. Buckets and
  quantiles are skipped. A labelled series is stored under
  "<target>:k=v,…". Each target also records is_up, scrape_ms, series_kept
  and series_capped.
- Pages: targets not answering first, with why; a target's series grouped
  by metric with their newest values; a chart per series; the editor with
  Test, which scrapes once and says what would be stored.
- The counter-to-rate keeper moves to steward/core/counter_rates.py; SNMP's
  RateKeeper now scales its rate on top of it.
- metric_value reads Prometheus's _seconds as a span and _per_sec as a rate.
- The catalog guard treats an empty list as a dynamic source (snmp,
  prometheus), as the rule form does.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-12 07:04:01 -04:00
co-authored by Claude Opus 5.5
parent a9998681f9
commit 3cb2a95054
24 changed files with 1598 additions and 27 deletions
@@ -0,0 +1,18 @@
{# What Test found (milestone 506): how many series the address hands over
and which would be stored with the filters typed, or why it got nothing. #}
{% if found.error %}
<p class="tone-crit" role="alert" style="margin:0.75rem 0 0;">{{ found.error }}</p>
{% else %}
<div style="margin-top:0.75rem;">
<p class="tone-ok" style="margin:0 0 0.4rem;">{{ url }} answered with {{ found.families }} metric famil{{ "y" if found.families == 1 else "ies" }}.</p>
<p class="field-hint" style="margin:0 0 0.5rem;">With what's typed here, {{ found.kept }} series would be stored
{%- if found.dropped %}, {{ found.dropped }} left out by the filters{% endif %}
{%- if found.capped %}, and <span class="tone-warn">{{ found.capped }} over the cap</span>{% endif %}.</p>
{% if found.names %}
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(18rem,1fr));gap:0.15rem 0.9rem;font-size:0.82rem;">
{% for name, n in found.names %}<span style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="{{ name }}"><span class="mono">{{ name }}</span>{% if n > 1 %} <span class="tone-dim">×{{ n }}</span>{% endif %}</span>{% endfor %}
</div>
{% if found.more %}<p class="field-hint" style="margin:0.4rem 0 0;">and {{ found.more }} more.</p>{% endif %}
{% endif %}
</div>
{% endif %}
@@ -0,0 +1,111 @@
{# plugins/prometheus/templates/prometheus/form.html — add or edit a scrape
target (milestone 506): where its /metrics is, its token, how often, and
what to keep, with Test to scrape it once before saving. The token is
never sent back to the browser; a blank token field keeps the stored one. #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, page_header %}
{% block title %}{{ ("Edit " ~ target.name) if target else "Add Prometheus target" }} — Steward{% endblock %}
{% block breadcrumb %}
{%- if target -%}
{{ crumbs([("Prometheus", "/plugins/prometheus/"), (target.name, "/plugins/prometheus/targets/" ~ target.id), ("Edit", "")]) }}
{%- else -%}
{{ crumbs([("Prometheus", "/plugins/prometheus/"), ("Add target", "")]) }}
{%- endif -%}
{% endblock %}
{% block content %}
{{ page_header(("Edit " ~ target.name) if target else "Add Prometheus target") }}
<div class="form-page">
{% if error %}<p class="field-error" role="alert">{{ error }}</p>{% endif %}
<form method="post" id="prom-form" action="{{ '/plugins/prometheus/targets/' ~ target.id ~ '/edit' if target else '/plugins/prometheus/targets/new' }}">{{ csrf_field() }}
{% if target %}<input type="hidden" name="target_id" value="{{ target.id }}">{% endif %}
<section class="card form-section" aria-labelledby="prom-where">
<h2 id="prom-where" class="section-title">Endpoint</h2>
<div class="form-pair">
<div class="field">
<label class="field-label" for="prom-name">Name</label>
<input class="input" id="prom-name" name="name" required maxlength="100" value="{{ v.name or '' }}" placeholder="Archives" autocomplete="off">
<span class="field-hint">How its series are named in Steward: alert rules and charts show it.</span>
</div>
<div class="field">
<label class="field-label" for="prom-url">Metrics address</label>
<input class="input mono" id="prom-url" name="url" required maxlength="500" value="{{ v.url or '' }}" placeholder="http://archives:8081/metrics" autocomplete="off">
</div>
<div class="field">
<label class="field-label" for="prom-token">Bearer token</label>
<input class="input" id="prom-token" name="token" type="password" autocomplete="new-password"
placeholder="{{ 'kept; type to change' if v.has_token else 'none' }}">
<span class="field-hint">Sent as "Authorization: Bearer …". Blank {{ "keeps the saved one" if v.has_token else "sends none" }}.</span>
{% if v.has_token %}
<label style="display:flex;gap:0.5rem;align-items:center;margin-top:0.3rem;cursor:pointer;">
<input type="checkbox" name="clear_token" value="on"> <span class="field-hint">Remove the saved token</span>
</label>
{% endif %}
</div>
<div class="field">
<label class="field-label" for="prom-interval">Scrape every (seconds)</label>
<input class="input" id="prom-interval" name="interval_seconds" type="number" min="10" max="3600" value="{{ v.interval_seconds or 60 }}">
</div>
</div>
<div style="display:flex;gap:1.5rem;flex-wrap:wrap;margin-bottom:0.9rem;">
<label style="display:flex;gap:0.5rem;align-items:center;cursor:pointer;">
<input type="checkbox" name="enabled" value="on"{% if v.enabled %} checked{% endif %}> Scrape this target
</label>
<label style="display:flex;gap:0.5rem;align-items:center;cursor:pointer;">
<input type="checkbox" name="verify_tls" value="on"{% if v.verify_tls %} checked{% endif %}> Check its HTTPS certificate
</label>
</div>
<div style="display:flex;align-items:center;gap:0.75rem;flex-wrap:wrap;">
<button type="button" class="btn btn-sm btn-ghost"
hx-post="/plugins/prometheus/targets/test" hx-include="#prom-form" hx-target="#prom-test"
hx-indicator="#prom-testing">Test</button>
<span id="prom-testing" class="field-hint htmx-indicator">Scraping…</span>
<span class="field-hint">Scrapes the address once with what's typed here, filters included, and says what would be stored. Nothing is saved.</span>
</div>
<div id="prom-test" aria-live="polite"></div>
</section>
<section class="card form-section" aria-labelledby="prom-what">
<h2 id="prom-what" class="section-title">What to keep</h2>
<p class="field-hint" style="margin-top:0;">The endpoint hands over everything; Steward stores what you choose. Patterns match the endpoint's metric names, with <span class="mono">*</span> for any run of characters.</p>
<div class="form-pair">
<div class="field">
<label class="field-label" for="prom-keep">Keep</label>
<textarea class="input mono" id="prom-keep" name="keep" rows="4" placeholder="fabledarchives_*">{{ v.keep or '' }}</textarea>
<span class="field-hint">One a line. Empty keeps everything.</span>
</div>
<div class="field">
<label class="field-label" for="prom-drop">Leave out</label>
<textarea class="input mono" id="prom-drop" name="drop" rows="4" placeholder="go_*&#10;process_*">{{ v.drop or '' }}</textarea>
<span class="field-hint">One a line. Wins over Keep.</span>
</div>
<div class="field">
<label class="field-label" for="prom-cap">Series cap</label>
<input class="input" id="prom-cap" name="series_cap" type="number" min="1" max="10000" value="{{ v.series_cap or 1000 }}">
<span class="field-hint">The most series stored from one scrape. Past it, the rest are counted and shown, not stored.</span>
</div>
</div>
<p class="field-hint" style="margin:0;">Gauges are stored as they are. Counters are stored as a rate per second, named <span class="mono">…_per_sec</span>, from the second scrape on. Histogram buckets and summary quantiles aren't stored; their <span class="mono">_sum</span> and <span class="mono">_count</span> are, as rates.</p>
</section>
<div class="form-actions">
<button type="submit" class="btn">{{ "Save" if target else "Add target" }}</button>
<a href="{{ '/plugins/prometheus/targets/' ~ target.id if target else '/plugins/prometheus/' }}" class="btn btn-ghost">Cancel</a>
</div>
</form>
{% if target %}
<section class="card danger-zone" aria-labelledby="prom-delete">
<div>
<h2 id="prom-delete" class="section-title">Delete this target</h2>
<p class="field-hint">Steward stops scraping it. Its past series age out with the rest of the history.</p>
</div>
<form method="post" action="/plugins/prometheus/targets/{{ target.id }}/delete"
data-confirm="Delete Prometheus target {{ target.name }}? Steward stops scraping it.">{{ csrf_field() }}
<button class="btn btn-danger" type="submit">Delete target</button>
</form>
</section>
{% endif %}
</div>
{% endblock %}
@@ -0,0 +1,59 @@
{# plugins/prometheus/templates/prometheus/index.html — every scrape target,
those not answering first, each saying what its newest scrape found: how
many series it stored, or why it got nothing (milestone 506). #}
{% extends "base.html" %}
{% from "_macros.html" import page_header, status_badge, stat_tile, empty_state %}
{% block title %}Prometheus — Steward{% endblock %}
{% block content %}
{% set admin = session.user_role == 'admin' %}
{% call page_header("Prometheus targets", (targets|length ~ " target" ~ ("s" if targets|length != 1 else "")) if targets else none) %}
{% if admin %}<a class="btn btn-sm" href="/plugins/prometheus/targets/new">Add target</a>{% endif %}
{% endcall %}
{% if not targets %}
<div class="card">{{ empty_state("No targets yet. Add a service's /metrics address, and Steward scrapes it on a schedule so its numbers can be charted and alerted on.",
"Add target" if admin else none, "/plugins/prometheus/targets/new") }}</div>
{% endif %}
{% for t in targets %}
<section class="card" aria-labelledby="prom-{{ t.id }}">
<div class="host-health-head">
{% if t.reach == 'up' %}{{ status_badge('up', 'Answering') }}
{% elif t.reach == 'down' %}{{ status_badge('down', 'Not answering') }}
{% elif t.reach == 'stale' %}{{ status_badge('stale', 'Not scraped') }}
{% elif t.reach == 'off' %}{{ status_badge('stale', 'Turned off') }}
{% else %}{{ status_badge('pending', 'Not scraped yet') }}{% endif %}
<h2 id="prom-{{ t.id }}" class="section-title" style="margin:0;"><a href="/plugins/prometheus/targets/{{ t.id }}">{{ t.name }}</a></h2>
<span class="field-hint mono" style="min-width:0;max-width:28rem;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="{{ t.url }}">{{ t.url }}</span>
<span class="page-actions" style="margin-left:auto;">
<a href="/plugins/prometheus/targets/{{ t.id }}" class="btn btn-sm btn-ghost">Series</a>
{% if admin %}<a href="/plugins/prometheus/targets/{{ t.id }}/edit" class="btn btn-sm btn-ghost">Edit</a>{% endif %}
</span>
</div>
{% if t.reach == 'down' %}
<p class="tone-crit" style="margin:0 0 0.4rem;">{{ t.says }}</p>
<p class="field-hint" style="margin:0 0 0.6rem;">
{%- if t.ok_at %}Last answered {{ t.ok_at|ago }}.{% else %}It has never answered.{% endif %}
Tried {{ t.scraped_at|ago }}.</p>
{% elif t.reach in ('stale', 'off') %}
<p class="field-hint" style="margin:0 0 0.6rem;">{{ t.says }}{% if t.scraped_at %} Last scraped {{ t.scraped_at|ago }}.{% endif %}</p>
{% elif t.reach == 'pending' %}
<p class="field-hint" style="margin:0 0 0.6rem;">Not scraped yet: the first scrape runs within seconds of saving.</p>
{% endif %}
{% if t.reach == 'up' %}
<div class="stat-strip" style="margin-bottom:0;">
{{ stat_tile("Series stored", t.kept) }}
{{ stat_tile("Over the cap", t.capped, hint=("cap is " ~ t.cap) if t.capped else none) }}
{{ stat_tile("Left out by filters", t.dropped) }}
{{ stat_tile("Scrape time", (t.scrape_ms|round|int ~ " ms") if t.scrape_ms is not none else "—") }}
{{ stat_tile("Every", t.interval|duration) }}
</div>
{% if t.capped %}
<p class="tone-warn" style="margin:0.6rem 0 0;">{{ t.capped }} series weren't stored: past the cap of {{ t.cap }}, or a name too long to store. {% if admin %}<a href="/plugins/prometheus/targets/{{ t.id }}/edit">Narrow what's kept or raise the cap.</a>{% endif %}</p>
{% endif %}
{% endif %}
</section>
{% endfor %}
{% endblock %}
@@ -0,0 +1,66 @@
{# plugins/prometheus/templates/prometheus/series.html — one stored series
of a target (milestone 506): its newest value and a chart over a chosen
range. A *_timestamp_seconds series charts as moments, not numbers. #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, page_header %}
{% block title %}{{ metric }} — {{ target.name }} — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Prometheus", "/plugins/prometheus/"), (target.name, "/plugins/prometheus/targets/" ~ target.id), (metric, "")]) }}{% endblock %}
{% block content %}
{% call page_header(metric, labels or none) %}
{% for h in ranges %}
<a href="?resource={{ resource|urlencode }}&metric={{ metric|urlencode }}&hours={{ h }}" class="range-btn{% if hours == h %} active{% endif %}"{% if hours == h %} aria-current="true"{% endif %}>{% if h < 168 %}{{ h }}h{% else %}7d{% endif %}</a>
{% endfor %}
{% endcall %}
<p class="page-verdict">Now: <strong>{% if now.moment %}{{ now.moment|ago }}{% else %}{{ now.text }}{% endif %}</strong></p>
<section class="card" aria-label="{{ metric }} over the last {{ hours }} hours">
<div style="position:relative;height:260px;">
<canvas id="prom-chart" role="img" aria-label="{{ metric }} over the last {{ hours }} hours"></canvas>
</div>
</section>
<p class="field-hint">To be told when it crosses a line, add an alert rule on <strong>Prometheus targets</strong> → <span class="mono">{{ resource }}</span> → <span class="mono">{{ metric }}</span>.
<a href="/alerts/rules/new">New alert rule</a></p>
<script>
(function () {
var points = {{ points_json | safe }};
var hours = {{ hours }};
var moments = {{ "true" if stamp_axis else "false" }};
var el = document.getElementById("prom-chart");
if (points.length < 2) {
el.parentElement.innerHTML = '<p class="field-hint" style="margin:0;">Not enough samples yet for a chart.</p>';
return;
}
function stamp(iso) {
var d = new Date(iso);
return hours > 24
? d.toLocaleDateString([], { weekday: "short", day: "numeric" }) + " " + d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false })
: d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false });
}
function moment(v) {
return new Date(v * 1000).toLocaleString([], { day: "numeric", month: "short", hour: "2-digit", minute: "2-digit", hour12: false });
}
new Chart(el.getContext("2d"), {
type: "line",
data: { labels: points.map(function (p) { return p[0]; }),
datasets: [{ data: points.map(function (p) { return p[1]; }), borderColor: stewardColor("--blue"),
backgroundColor: "transparent", borderWidth: 1.5, pointRadius: 0, tension: 0.2 }] },
options: {
responsive: true, maintainAspectRatio: false, animation: false,
interaction: { mode: "index", intersect: false },
plugins: { legend: { display: false },
tooltip: { callbacks: {
title: function (items) { return items.length ? new Date(points[items[0].dataIndex][0]).toLocaleString([], { hour12: false }) : ""; },
label: function (item) { return moments ? moment(item.parsed.y) : String(item.parsed.y); } } } },
scales: {
x: { ticks: { maxTicksLimit: 8, maxRotation: 0, font: { size: 10 },
callback: function (v) { return stamp(points[v][0]); } }, grid: { display: false } },
y: { ticks: { font: { size: 10 }, maxTicksLimit: 5,
callback: function (v) { return moments ? moment(v) : v; } } }
}
}
});
})();
</script>
{% endblock %}
@@ -0,0 +1,69 @@
{# plugins/prometheus/templates/prometheus/target.html — one scrape target
(milestone 506): what its newest scrape found, then every series it
stored, grouped by metric, each with its labels and newest value and a
link to its chart. #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs, page_header, status_badge, stat_tile %}
{% block title %}Prometheus — {{ target.name }} — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Prometheus", "/plugins/prometheus/"), (target.name, "")]) }}{% endblock %}
{% block content %}
{% set admin = session.user_role == 'admin' %}
{% call page_header(target.name, target.url) %}
{% if admin %}<a href="/plugins/prometheus/targets/{{ target.id }}/edit" class="btn btn-sm btn-ghost">Edit</a>{% endif %}
{% endcall %}
<p class="page-verdict">
{% if target.reach == 'up' %}{{ status_badge('up', 'Answering') }} scraped {{ target.scraped_at|ago }}, every {{ target.interval|duration }}
{% elif target.reach == 'down' %}{{ status_badge('down', 'Not answering') }} <span class="tone-crit">{{ target.says }}</span>
{% if target.ok_at %}Last answered {{ target.ok_at|ago }}; the values below are from then.{% else %}It has never answered.{% endif %}
{% elif target.reach == 'stale' %}{{ status_badge('stale', 'Not scraped') }} {{ target.says }}
{% elif target.reach == 'off' %}{{ status_badge('stale', 'Turned off') }} {{ target.says }}
{% else %}{{ status_badge('pending', 'Not scraped yet') }} The first scrape runs within seconds of saving.{% endif %}
</p>
{% if target.capped %}
<p class="tone-warn">{{ target.capped }} series weren't stored on the last scrape: past the cap of {{ target.cap }}, or a name too long to store.
{% if admin %}<a href="/plugins/prometheus/targets/{{ target.id }}/edit">Narrow what's kept or raise the cap.</a>{% endif %}</p>
{% endif %}
{% if target.reach != 'pending' %}
<div class="stat-strip">
{{ stat_tile("Series stored", target.kept) }}
{{ stat_tile("Over the cap", target.capped, state="degraded" if target.capped else none) }}
{{ stat_tile("Left out by filters", target.dropped) }}
{{ stat_tile("Scrape time", (target.scrape_ms|round|int ~ " ms") if target.scrape_ms is not none else "—") }}
</div>
{% endif %}
<p class="field-hint">Keeping {{ target.keep|join(", ") if target.keep else "everything" }}{% if target.drop %}, leaving out {{ target.drop|join(", ") }}{% endif %}.
Alert rules can watch any of these under <strong>Prometheus targets</strong>.</p>
<section class="card" aria-labelledby="prom-series">
<div style="display:flex;align-items:center;gap:0.75rem;flex-wrap:wrap;margin-bottom:0.5rem;">
<h2 id="prom-series" class="section-title" style="margin:0;">Series <span class="tone-muted">{{ total }}</span></h2>
<form method="get" style="margin-left:auto;display:flex;gap:0.5rem;">
<label class="sr-only" for="prom-q">Find a metric</label>
<input class="input" id="prom-q" name="q" value="{{ q }}" placeholder="Find a metric" style="max-width:16rem;">
{% if q %}<a class="btn btn-sm btn-ghost" href="/plugins/prometheus/targets/{{ target.id }}">Clear</a>{% endif %}
</form>
</div>
{% if not groups %}
<p class="field-hint" style="margin:0;">{% if q %}No stored metric matches “{{ q }}”.{% elif target.reach == 'pending' %}Nothing stored yet.{% else %}Nothing stored: the filters leave every series out, or the target hasn't answered.{% endif %}</p>
{% else %}
<table class="table">
<thead><tr><th scope="col">Metric</th><th scope="col">Labels</th><th scope="col" style="text-align:right;">Now</th></tr></thead>
<tbody>
{% for g in groups %}
{% for r in g.rows %}
<tr>
{% if loop.first %}<td class="mono" rowspan="{{ g.rows|length }}" style="vertical-align:top;overflow-wrap:anywhere;">{{ g.metric }}</td>{% endif %}
<td class="mono" style="overflow-wrap:anywhere;font-size:0.82rem;">{{ r.labels or "—" }}</td>
<td style="text-align:right;white-space:nowrap;">
<a href="/plugins/prometheus/targets/{{ target.id }}/series?resource={{ r.resource|urlencode }}&metric={{ g.metric|urlencode }}">{% if r.moment %}{{ r.moment|ago }}{% else %}{{ r.text }}{% endif %}</a>
</td>
</tr>
{% endfor %}
{% endfor %}
</tbody>
</table>
{% endif %}
</section>
{% endblock %}