From 4c9ed0265221e64411b4980e7707a46a3fafd035 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 1 Nov 2026 12:06:55 -0500 Subject: [PATCH] feat(homeassistant): its page offers the metrics scrape under the same sign-in, so one user covers everything (#6744) A Prometheus target's token_from names the plugin that supplies its token at each scrape. Read its metrics makes the target; Use this sign-in moves one at that address across, keeping its name, filters and history. A refusal stops the connection, so one Try again or sign-in starts everything, and the sign-in is only sent to its own address. A 404 says how to turn on Home Assistant's Prometheus integration, as Steward doesn't change Home Assistant. Refused also joins the volatile columns a backup check ignores. Co-Authored-By: Claude Opus 5.5 --- docs/plugins/homeassistant.md | 8 +- docs/plugins/prometheus.md | 2 + plugins/homeassistant/client.py | 6 + plugins/homeassistant/mcp_tools.py | 8 +- plugins/homeassistant/metrics.py | 153 ++++++++++++++++ plugins/homeassistant/routes.py | 30 +++- .../templates/homeassistant/form.html | 4 +- .../templates/homeassistant/index.html | 16 ++ .../versions/prometheus_004_token_from.py | 23 +++ plugins/prometheus/models.py | 4 +- plugins/prometheus/routes.py | 17 +- plugins/prometheus/scheduler.py | 40 ++++- plugins/prometheus/scraper.py | 6 + plugins/prometheus/service.py | 3 +- .../prometheus/templates/prometheus/form.html | 6 +- steward/backup/contents.py | 2 +- steward/core/prometheus.py | 6 + tests/integration/test_homeassistant.py | 168 ++++++++++++++++++ tests/plugins/prometheus/test_service.py | 4 + 19 files changed, 482 insertions(+), 24 deletions(-) create mode 100644 plugins/homeassistant/metrics.py create mode 100644 plugins/prometheus/migrations/versions/prometheus_004_token_from.py diff --git a/docs/plugins/homeassistant.md b/docs/plugins/homeassistant.md index 9e5543a..31ac24b 100644 --- a/docs/plugins/homeassistant.md +++ b/docs/plugins/homeassistant.md @@ -12,7 +12,7 @@ Your browser and Steward both need to reach Home Assistant at that address. Home ## What Steward holds -**Steward stores a credential with the same access as the user who signed in.** Home Assistant has no narrower permission, so an administrator's credential can do anything that administrator can, including controlling every device. Steward only uses it to read status, take backups and install updates. It is stored encrypted. +**Steward stores a credential with the same access as the user who signed in.** Home Assistant has no narrower permission, so an administrator's credential can do anything that administrator can, including controlling every device. Steward only uses it to read status and metrics, take backups and install updates. It is stored encrypted. We recommend a service account, a user made just for Steward, rather than your own. What Steward does then appears under its own name in Home Assistant's logs, so it's easy to tell apart from what people do. @@ -31,4 +31,8 @@ One runs at a time, and it carries on if Steward restarts. Firmware and custom i ## Metrics -Under source `homeassistant`, resource `homeassistant`, read every 5 minutes: `is_up`, `updates_waiting` and `backup_age_hours`. An alert rule on `backup_age_hours` above 26 warns when a daily backup has been missed; a rule's no-data warning covers Home Assistant going quiet. +One sign-in covers everything: the Metrics section of the page reads Home Assistant's entity readings with the same user, through a target on the Prometheus page. **Read its metrics** makes that target, or **Use this sign-in** moves an existing one at that address onto it, keeping its name, filters and history and forgetting its token. Home Assistant only serves these with its Prometheus integration turned on (`prometheus:` in its `configuration.yaml`, then a restart). Steward doesn't change that for you; it says so if they aren't there. + +If Home Assistant refuses that user, everything stops after one refusal, and signing in again or Try again starts everything. To go back to a separate token, type one into the target's form on the Prometheus page. + +The poll itself records these under source `homeassistant`, resource `homeassistant`, read every 5 minutes: `is_up`, `updates_waiting` and `backup_age_hours`. An alert rule on `backup_age_hours` above 26 warns when a daily backup has been missed; a rule's no-data warning covers Home Assistant going quiet. diff --git a/docs/plugins/prometheus.md b/docs/plugins/prometheus.md index 36df618..469f090 100644 --- a/docs/plugins/prometheus.md +++ b/docs/plugins/prometheus.md @@ -48,6 +48,8 @@ A value's unit is read from Prometheus naming: `_bytes` as bytes, `_seconds` as **A refused token stops the scraping.** When an endpoint answers 401 or 403, Steward stops asking and says so on the target's page and in the overview, because asking again won't change the answer and some services, Home Assistant among them, ban an address after a few refusals. Saving the target, usually with a new token, or pausing and resuming it, starts it again. Any other failure is retried at the next scrape. +**A target can use a plugin's sign-in instead of a token.** Home Assistant's page sets this up, so one Home Assistant user covers its metrics too. A token typed into the target's form replaces it. That sign-in is only sent to the address it was made for, and a refusal stops it from the plugin's page, not here. + A **state set** is one series per possible state, the current one 1 and the rest 0, such as `fabledarchives_backup_target_state{target, state}`. The target page shows it as one row per target (or job), naming the current state; the other states are listed on hover. A metric is read as a state set when its name ends in `_state` and every series has a `state` label reading 0 or 1. The OpenMetrics form, where the label is named after the metric (`foo{foo="a"} 1`), counts too. Each state is still stored as its own series, so an alert rule can watch one state. **Only what the endpoint reports now is current.** When a scrape succeeds, any series it no longer has loses its newest value. That happens when the endpoint stopped reporting it, for example an old `build_info` after an upgrade, or when a filter or the cap now leaves it out. Its history stays on the chart until retention removes it. A failed scrape leaves the newest values as they were. diff --git a/plugins/homeassistant/client.py b/plugins/homeassistant/client.py index a1d1017..813dff1 100644 --- a/plugins/homeassistant/client.py +++ b/plugins/homeassistant/client.py @@ -118,6 +118,12 @@ async def redeem_code(url: str, client_id: str, code: str, verify_tls: bool, tra return refresh_token, _remember(refresh_token, found) +def forget_access() -> None: + """Drop access tokens made so far: after a refusal the next ask starts from the + sign-in, which a dead user is refused at without counting toward a ban.""" + _access.clear() + + async def access_token(connection, transport=None) -> str: refresh_token = decrypt_secret(connection.refresh_token, context="Home Assistant sign-in") token, until = _access.get(refresh_token, (None, 0.0)) diff --git a/plugins/homeassistant/mcp_tools.py b/plugins/homeassistant/mcp_tools.py index 8054b74..358ee48 100644 --- a/plugins/homeassistant/mcp_tools.py +++ b/plugins/homeassistant/mcp_tools.py @@ -13,6 +13,7 @@ from steward.mcp.tools._shape import age_s, db_session, iso, now_utc, redact_url from steward.mcp.tools.overview import finding from steward.models.users import UserRole +from . import metrics from .models import HomeAssistantConnection, HomeAssistantUpdate from .routes import connection_says from .service import STALE_BACKUP_HOURS, backup_age_hours, backup_verdict, installable, parse_moment @@ -35,13 +36,16 @@ async def get_home_assistant() -> dict: `installable` is what update_home_assistant takes (Core, OS, add-ons); `others_waiting` is firmware and custom integrations, which Steward leaves to Home Assistant. `can_install` is whether the Home Assistant user who signed - Steward in is an administrator, which installing needs. + Steward in is an administrator, which installing needs. `metrics` is the + Prometheus target that reads its metrics, `uses_sign_in` saying whether that + same user does; null when the Prometheus plugin is off. """ now = now_utc() async with db_session() as db: c = (await db.execute(select(HomeAssistantConnection))).scalars().first() runs = list((await db.execute(select(HomeAssistantUpdate).order_by( HomeAssistantUpdate.started_at.desc()).limit(RECENT_UPDATES))).scalars()) + scrape = await metrics.state(db, c) if c else None if c is None: return {"connected": False, "says": "No connection yet: an admin signs in on the Home Assistant page."} reach, says = connection_says(c, now) @@ -55,7 +59,7 @@ async def get_home_assistant() -> dict: "backup": {"verdict": backup_verdict(backup, now)[1], "last_ok": iso(parse_moment(backup.get("last_ok"))), "age_hours": round(age, 1) if age is not None else None, "stale_after_hours": STALE_BACKUP_HOURS, "next": iso(parse_moment(backup.get("next"))), "last_result": backup.get("last_result")}, - "updates": [_run_view(r) for r in runs]} + "metrics": scrape, "updates": [_run_view(r) for r in runs]} async def update_home_assistant(entity_id: str) -> dict: diff --git a/plugins/homeassistant/metrics.py b/plugins/homeassistant/metrics.py new file mode 100644 index 0000000..b199f2a --- /dev/null +++ b/plugins/homeassistant/metrics.py @@ -0,0 +1,153 @@ +# plugins/homeassistant/metrics.py +"""Home Assistant's metrics, read with the one sign-in Steward already holds, so a +single Home Assistant user covers everything. The scrape stays a Prometheus target: +this lends it a token at each scrape, sends that token only to this Home Assistant, +and stops it when the sign-in has been refused.""" +from __future__ import annotations + +import asyncio +from datetime import datetime, timezone +from typing import TYPE_CHECKING + +from sqlalchemy import or_, select, update + +from steward.core.plugin_manager import loaded_plugin_submodule +from steward.core.prometheus import TokenUnavailable + +from . import client +from .models import HomeAssistantConnection +from .service import SOURCE + +if TYPE_CHECKING: + from quart import Quart + +METRICS_PATH = "/api/prometheus" +NAME = "HomeAssistant" +INTERVAL_SECONDS = 120 +TRY_SECONDS = 30 +# What a new target leaves out: bookkeeping on every entity, a few thousand series +# that say nothing the readings don't. +DROP = ["homeassistant_state_change*", "homeassistant_last_updated_time_seconds", "homeassistant_entity_available", + "homeassistant_entity_info", "homeassistant_area_info", "python_*", "process_*"] +REFUSED = "Home Assistant refused Steward's sign-in while reading its metrics, or has banned this address: look in its ip_bans.yaml." + + +class CannotRead(Exception): + """Why the metrics can't be read with this sign-in, said to the person.""" + + +def metrics_url(connection) -> str: + return connection.url.rstrip("/") + METRICS_PATH + + +# ── Lending the sign-in to a scrape ────────────────────────────────────────── + +async def scrape_token(app: "Quart", url: str) -> str: + """A token for scraping `url`. Raises TokenUnavailable, asking nothing of + anyone, once the sign-in was refused: another refused request counts toward a + ban. A target edited to point elsewhere isn't sent this user's token.""" + async with app.db_sessionmaker() as db: + connection = (await db.execute(select(HomeAssistantConnection))).scalars().first() + if connection is None: + raise TokenUnavailable("Home Assistant isn't connected: sign in on its page to read its metrics.") + if connection.refused: + raise TokenUnavailable(f"{connection.last_error} Steward has stopped asking until you sign in again " + "or choose Try again on the Home Assistant page.") + if url != metrics_url(connection): + raise TokenUnavailable(f"This isn't the address of the Home Assistant Steward signed in to, so its sign-in " + f"isn't sent here. Set the address to {metrics_url(connection)}, or type a token.") + try: + return await client.access_token(connection) + except (client.Refused, client.Unreachable) as exc: + raise TokenUnavailable(str(exc)) from None + + +async def token_refused(app: "Quart") -> None: + """A scrape's token was refused: stop everything using this sign-in.""" + client.forget_access() + async with app.db_sessionmaker() as db: + async with db.begin(): + connection = (await db.execute(select(HomeAssistantConnection))).scalars().first() + if connection is not None: + connection.refused, connection.last_error = True, REFUSED + + +# ── The Metrics section of the page ────────────────────────────────────────── + +def _prometheus(sub: str): + return loaded_plugin_submodule("prometheus", sub) + + +async def _target(db, connection): + """The target using this sign-in, or else the one at this address.""" + T = _prometheus("models").PrometheusTarget + rows = (await db.execute(select(T).where(or_(T.token_from == SOURCE, T.url == metrics_url(connection))))).scalars().all() + return next((t for t in rows if t.token_from == SOURCE), rows[0] if rows else None) + + +async def state(db, connection) -> dict | None: + """What the page says about the metrics. None when the Prometheus plugin is off; + `id` is None when no target reads them yet.""" + service = _prometheus("service") + if service is None or _prometheus("models") is None: + return None + target = await _target(db, connection) + if target is None: + return {"id": None, "name": None, "uses_sign_in": False, "reach": None, "says": None, "kept": 0} + found = service.target_state(target, datetime.now(timezone.utc)) + return {"id": target.id, "name": target.name, "uses_sign_in": target.token_from == SOURCE, + "reach": found.reach, "says": found.says, "kept": target.last_kept} + + +async def try_reading(app: "Quart", connection) -> None: + """Ask for the metrics once, so turning this on says at once if they aren't there.""" + scraper = _prometheus("scraper") + if scraper is None: + raise CannotRead("the Prometheus plugin isn't turned on.") + if connection.refused: + raise CannotRead("Home Assistant has refused this sign-in. Choose Try again or sign in again first.") + try: + token = await client.access_token(connection) + await asyncio.wait_for(scraper.scrape(metrics_url(connection), token, connection.verify_tls), TRY_SECONDS) + except scraper.NoMetricsHere: + raise CannotRead("Home Assistant isn't serving metrics. Add prometheus: to its configuration.yaml, restart it, " + "and come back: Steward doesn't change Home Assistant.") from None + except scraper.TokenRefused: + await token_refused(app) + raise CannotRead(REFUSED) from None + except TimeoutError: + raise CannotRead(f"Home Assistant's metrics didn't arrive within {TRY_SECONDS} s.") from None + except (scraper.ScrapeError, client.Refused, client.Unreachable) as exc: + raise CannotRead(str(exc)) from None + + +async def use_sign_in(app: "Quart", connection) -> tuple[str, str, bool]: + """Make the target, or move the one at this address onto this sign-in, keeping + its name, filters and history. Returns (id, name, whether it was made).""" + models, scheduler = _prometheus("models"), _prometheus("scheduler") + T = models.PrometheusTarget + async with app.db_sessionmaker() as db: + async with db.begin(): + target = await _target(db, connection) + created = target is None + if created: + if (await db.execute(select(T.id).where(T.name == NAME))).first(): + raise CannotRead(f"another Prometheus target is already named {NAME}. Rename it first.") + target = T(name=NAME, interval_seconds=INTERVAL_SECONDS, drop=list(DROP)) + db.add(target) + target.url, target.verify_tls = metrics_url(connection), connection.verify_tls + target.token, target.token_from, target.refused = "", SOURCE, False + target.last_scraped_at = target.last_error = None + await db.flush() + target_id, name = target.id, target.name + scheduler.RATES.forget(target_id) + return target_id, name, created + + +async def follow_sign_in(db, connection) -> None: + """Signing in at a new address takes the scrape using this sign-in with it.""" + models = _prometheus("models") + if models is not None: + T = models.PrometheusTarget + await db.execute(update(T).where(T.token_from == SOURCE).values( + url=metrics_url(connection), verify_tls=connection.verify_tls)) diff --git a/plugins/homeassistant/routes.py b/plugins/homeassistant/routes.py index 64cbbcd..1c0b5c9 100644 --- a/plugins/homeassistant/routes.py +++ b/plugins/homeassistant/routes.py @@ -18,7 +18,7 @@ from steward.core.crypto import encrypt_secret from steward.core.settings import public_base_url from steward.models.users import UserRole -from . import client +from . import client, metrics from .models import HomeAssistantConnection, HomeAssistantUpdate from .service import STALE_BACKUP_HOURS, backup_age_hours, backup_verdict, installable, parse_moment from .updates import CannotStart, start_update @@ -55,6 +55,7 @@ async def _render_index(error: str | None = None, status: int = 200): connection = await _connection(db) runs = list((await db.execute(select(HomeAssistantUpdate).order_by( HomeAssistantUpdate.started_at.desc()).limit(RECENT_UPDATES))).scalars()) + scrape = await metrics.state(db, connection) if connection else None if connection is None: return await render_template("homeassistant/index.html", connection=None, error=error), status now = datetime.now(timezone.utc) @@ -63,7 +64,7 @@ async def _render_index(error: str | None = None, status: int = 200): backup = connection.backup or {} tone, backup_says = backup_verdict(backup, now) return await render_template( - "homeassistant/index.html", connection=connection, error=error, reach=reach, says=says, + "homeassistant/index.html", connection=connection, error=error, reach=reach, says=says, metrics=scrape, installable=installable(connection), others=[u for u in updates if u["waiting"] and not u["kind"]], current=[u for u in updates if not u["waiting"] and u["kind"]], @@ -95,6 +96,26 @@ async def install(): return redirect("/plugins/homeassistant/") +@homeassistant_bp.post("/metrics") +@require_role(UserRole.admin) +async def connect_metrics(): + """Read Home Assistant's metrics with this sign-in, after asking for them once + to show they're there. An admin's explicit action, so it may ask.""" + async with current_app.db_sessionmaker() as db: + connection = await _connection(db) + if connection is None: + return redirect("/plugins/homeassistant/") + try: + await metrics.try_reading(current_app, connection) + target_id, name, created = await metrics.use_sign_in(current_app, connection) + except metrics.CannotRead as exc: + return await _render_index(f"Nothing changed: {exc}", 409) + await log_audit(current_app, *_actor(), "prometheus.target_created" if created else "prometheus.target_updated", + entity_type="prometheus_target", entity_id=target_id, + detail={"name": name, "token_from": metrics.SOURCE}) + return redirect("/plugins/homeassistant/") + + # ── The connection ─────────────────────────────────────────────────────────── async def _render_form(connection, url: str | None = None, verify_tls: bool | None = None, @@ -167,6 +188,7 @@ async def sign_in_callback(): connection.can_install, connection.signed_in_by = can_install, username connection.signed_in_at = datetime.now(timezone.utc) connection.refused, connection.last_error, connection.last_polled_at = False, None, None + await metrics.follow_sign_in(db, connection) await db.flush() connection_id = connection.id if previous is not None: @@ -192,7 +214,9 @@ async def _end_sign_in(connection) -> bool: @homeassistant_bp.post("/connection/retry") @require_role(UserRole.admin) async def retry_connection(): - """Ask again after a refusal, once a ban has been dealt with.""" + """Ask again after a refusal, once a ban has been dealt with. A metrics scrape + using this sign-in starts again with it.""" + client.forget_access() async with current_app.db_sessionmaker() as db: async with db.begin(): connection = await _connection(db) diff --git a/plugins/homeassistant/templates/homeassistant/form.html b/plugins/homeassistant/templates/homeassistant/form.html index f0e59c1..423f2e1 100644 --- a/plugins/homeassistant/templates/homeassistant/form.html +++ b/plugins/homeassistant/templates/homeassistant/form.html @@ -23,7 +23,7 @@ -

Steward stores a credential with the same access as the user who signs in. Home Assistant has no narrower permission, so an administrator's credential can do anything that administrator can, including controlling every device. Steward only uses it to read status, take backups and install updates.

+

Steward stores a credential with the same access as the user who signs in. Home Assistant has no narrower permission, so an administrator's credential can do anything that administrator can, including controlling every device. Steward only uses it to read status and metrics, take backups and install updates.

We recommend a service account: a Home Assistant user made just for Steward (Settings → People → Add person). What Steward does then appears under its own name in Home Assistant's logs, so it's easy to tell apart from what people do.

Installing updates needs an administrator. With any other user, Steward shows status only.

@@ -38,7 +38,7 @@

Remove this connection

-

Steward stops reading Home Assistant, forgets its credential and ends the sign-in there.

+

Steward stops reading Home Assistant, its metrics included, forgets its credential and ends the sign-in there.

{{ csrf_field() }} diff --git a/plugins/homeassistant/templates/homeassistant/index.html b/plugins/homeassistant/templates/homeassistant/index.html index d6b36f9..cdbad8a 100644 --- a/plugins/homeassistant/templates/homeassistant/index.html +++ b/plugins/homeassistant/templates/homeassistant/index.html @@ -73,6 +73,22 @@ {% if others %}

Waiting, for Home Assistant to install: {% for u in others %}{{ u.title }}{{ ", " if not loop.last }}{% endfor %}. Steward installs Core, OS and add-on updates only.

{% endif %}
+{% if metrics %} +
+

Metrics

+ {% if metrics.uses_sign_in %} +

{{ status_badge(metrics.reach) }} Read with this sign-in into the target {{ metrics.name }}: {{ metrics.says }}{% if metrics.reach == 'up' %} {{ metrics.kept }} series kept.{% endif %}

+ {% elif metrics.id %} +

The target {{ metrics.name }} reads these with a token of its own: {{ metrics.says }}

+ {% if admin %}

Use this sign-in instead, so one Home Assistant user covers everything. The target keeps its name, filters and history, and forgets its token.

+ {{ csrf_field() }}{% endif %} + {% else %} +

Home Assistant's entity readings can be kept as metrics too, for charts and alert rules, with this sign-in and nothing else to set up.

+ {% if admin %}
{{ csrf_field() }}
{% endif %} + {% endif %} +
+{% endif %} + {% if runs %}

Updates run from Steward

diff --git a/plugins/prometheus/migrations/versions/prometheus_004_token_from.py b/plugins/prometheus/migrations/versions/prometheus_004_token_from.py new file mode 100644 index 0000000..d523ce1 --- /dev/null +++ b/plugins/prometheus/migrations/versions/prometheus_004_token_from.py @@ -0,0 +1,23 @@ +"""A target can take its token from another plugin's sign-in + +Revision ID: prometheus_004_token_from +Revises: prometheus_003_refused +Create Date: 2026-11-01 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "prometheus_004_token_from" +down_revision: Union[str, None] = "prometheus_003_refused" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("prometheus_targets", sa.Column("token_from", sa.String(50), nullable=True)) + + +def downgrade() -> None: + op.drop_column("prometheus_targets", "token_from") diff --git a/plugins/prometheus/models.py b/plugins/prometheus/models.py index f723854..2e63a9c 100644 --- a/plugins/prometheus/models.py +++ b/plugins/prometheus/models.py @@ -13,7 +13,8 @@ from steward.models.base import Base class PrometheusTarget(Base): """A /metrics endpoint Steward scrapes, edited on the Prometheus page. - The bearer token is stored encrypted. keep and drop are lists of name + The bearer token is stored encrypted, or `token_from` names the plugin whose + sign-in supplies it at each scrape (its metrics.scrape_token). keep and drop are lists of name patterns ("fabledarchives_*") an admin chooses: Steward does the filtering, the endpoint serves everything. The last_* columns are the newest scrape's outcome, written by the scraper, so a page can say why a @@ -26,6 +27,7 @@ class PrometheusTarget(Base): name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True) url: Mapped[str] = mapped_column(String(500), nullable=False) token: Mapped[str] = mapped_column(Text, nullable=False, default="") + token_from: Mapped[str | None] = mapped_column(String(50), nullable=True) verify_tls: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) interval_seconds: Mapped[int] = mapped_column(Integer, nullable=False, default=60) keep: Mapped[list] = mapped_column(JSON, nullable=False, default=list) diff --git a/plugins/prometheus/routes.py b/plugins/prometheus/routes.py index 01ad8dc..082fb19 100644 --- a/plugins/prometheus/routes.py +++ b/plugins/prometheus/routes.py @@ -12,13 +12,14 @@ from sqlalchemy import delete, func, select, update from steward.auth.middleware import require_role from steward.core.audit import log_audit -from steward.core.crypto import decrypt_secret from steward.core.metric_series import latest_values, points +from steward.core.prometheus import TokenUnavailable from steward.models.metrics import PluginMetric, PluginMetricLatest from steward.models.users import UserRole from . import library from .models import PrometheusTarget, PrometheusWidgetTemplate +from .scheduler import target_token from .scraper import ScrapeError, scrape from .service import (OWN_METRICS, SOURCE, FormError, fold_states, form_values, is_moment, map_families, parse_form, patterns, shown_value, target_state) @@ -80,7 +81,8 @@ def _typed(form, target: PrometheusTarget | None = None) -> dict: return {"name": form.get("name", ""), "url": form.get("url", ""), "interval_seconds": form.get("interval_seconds", ""), "series_cap": form.get("series_cap", ""), "keep": form.get("keep", ""), "drop": form.get("drop", ""), - "verify_tls": bool(form.get("verify_tls")), "has_token": bool(target and target.token)} + "verify_tls": bool(form.get("verify_tls")), "has_token": bool(target and target.token), + "token_from": target.token_from if target else None} async def _name_taken(db, name: str, other_than: str | None = None) -> bool: @@ -235,10 +237,15 @@ async def test_target(): existing = await db.get(PrometheusTarget, form["target_id"]) url = (form.get("url") or "").strip() token = (form.get("token") or "").strip() - if not token and existing is not None and existing.token and not form.get("clear_token"): - token = decrypt_secret(existing.token, context=f"prometheus target {existing.name} token") found: dict = {} - if not url.startswith(("http://", "https://")): + if not token and existing is not None and not form.get("clear_token"): + try: + token = await target_token(current_app, existing, url) + except TokenUnavailable as exc: + found["error"] = str(exc) + if found: + pass + elif not url.startswith(("http://", "https://")): found["error"] = "Enter the address first: http://… or https://…" else: try: diff --git a/plugins/prometheus/scheduler.py b/plugins/prometheus/scheduler.py index dba104c..638cf02 100644 --- a/plugins/prometheus/scheduler.py +++ b/plugins/prometheus/scheduler.py @@ -48,14 +48,33 @@ def is_due(target, now: datetime) -> bool: return elapsed >= target.interval_seconds - TICK_SECONDS / 2 +def _token_source(name: str): + from steward.core.plugin_manager import loaded_plugin_submodule + return loaded_plugin_submodule(name, "metrics") + + +async def target_token(app: "Quart", target, url: str) -> str: + """The token to ask `url` with: the target's own, or one from the plugin whose + sign-in it uses. Raises TokenUnavailable when that plugin has none to give.""" + from steward.core.crypto import decrypt_secret + from steward.core.prometheus import TokenUnavailable + + if not target.token_from: + return decrypt_secret(target.token, context=f"prometheus target {target.name} token") + source = _token_source(target.token_from) + if source is None: + raise TokenUnavailable(f"Its token comes from the {target.token_from} plugin, which isn't turned on.") + return await source.scrape_token(app, url) + + async def scrape_due(app: "Quart", scrape=None) -> None: """Scrape every due target at once, then write each one's outcome to its row and its series to the metric tables. `scrape` stands in for scraper.scrape in tests.""" - from steward.core.crypto import decrypt_secret + from steward.core.prometheus import TokenUnavailable from .models import PrometheusTarget - from .scraper import ScrapeError + from .scraper import ScrapeError, TokenRefused from .scraper import scrape as fetch scrape = scrape or fetch @@ -68,11 +87,15 @@ async def scrape_due(app: "Quart", scrape=None) -> None: async def one(t): started = time.monotonic() - token = decrypt_secret(t.token, context=f"prometheus target {t.name} token") + + async def fetched(): + return await scrape(t.url, await target_token(app, t, t.url), t.verify_tls) + try: - families = await asyncio.wait_for(scrape(t.url, token, t.verify_tls), - SCRAPE_DEADLINE_SECONDS) + families = await asyncio.wait_for(fetched(), SCRAPE_DEADLINE_SECONDS) return families, (time.monotonic() - started) * 1000 + except TokenUnavailable as exc: + return ScrapeError(str(exc)), (time.monotonic() - started) * 1000 except TimeoutError: return ScrapeError(f"Gave up after {SCRAPE_DEADLINE_SECONDS} s."), \ (time.monotonic() - started) * 1000 @@ -81,6 +104,11 @@ async def scrape_due(app: "Quart", scrape=None) -> None: results = await asyncio.gather(*(one(t) for t in targets)) await record(app, targets, results, datetime.now(timezone.utc)) + # A refused sign-in is the lending plugin's to stop, so one Try again or sign-in + # there starts everything that uses it. + for t, (result, _) in zip(targets, results): + if t.token_from and isinstance(result, TokenRefused) and (source := _token_source(t.token_from)): + await source.token_refused(app) async def forget_gone(session, target: str, kept: set[tuple[str, str]]) -> None: @@ -132,7 +160,7 @@ async def record(app: "Quart", targets, results, now: datetime) -> None: if not isinstance(result, ScrapeError): logger.warning("Prometheus scrape of %s failed: %r", target.name, result) row.last_error = str(result) or "The scrape failed." - row.refused = isinstance(result, TokenRefused) + row.refused = isinstance(result, TokenRefused) and not row.token_from batch.add(source_module=SOURCE, resource_name=row.name, metric_name=IS_UP, value=0.0, recorded_at=now) continue diff --git a/plugins/prometheus/scraper.py b/plugins/prometheus/scraper.py index 9c015b8..f59a36b 100644 --- a/plugins/prometheus/scraper.py +++ b/plugins/prometheus/scraper.py @@ -23,6 +23,10 @@ class TokenRefused(ScrapeError): Home Assistant, for one, bans an address after a few refusals.""" +class NoMetricsHere(ScrapeError): + """The address answered 404: nothing is serving metrics there.""" + + async def scrape(url: str, token: str = "", verify_tls: bool = True, transport: httpx.AsyncBaseTransport | None = None) -> dict[str, prometheus.Family]: """The endpoint's families. `transport` stands in for the network in tests.""" @@ -36,6 +40,8 @@ async def scrape(url: str, token: str = "", verify_tls: bool = True, if response.status_code in (401, 403): raise TokenRefused(f"Answered {response.status_code}: the token was refused. " "Check the token, or make a new one on the service.") + if response.status_code == 404: + raise NoMetricsHere(f"Answered 404 {response.reason_phrase}, not metrics.") if response.status_code != 200: raise ScrapeError(f"Answered {response.status_code} {response.reason_phrase}, not metrics.") body = bytearray() diff --git a/plugins/prometheus/service.py b/plugins/prometheus/service.py index 6345083..dafa113 100644 --- a/plugins/prometheus/service.py +++ b/plugins/prometheus/service.py @@ -212,7 +212,7 @@ def form_values(target: PrometheusTarget | None) -> dict: return {"name": target.name, "url": target.url, "interval_seconds": target.interval_seconds, "series_cap": target.series_cap, "verify_tls": target.verify_tls, "keep": "\n".join(target.keep or []), - "drop": "\n".join(target.drop or []), "has_token": bool(target.token)} + "drop": "\n".join(target.drop or []), "has_token": bool(target.token), "token_from": target.token_from} def _bounded(form, key: str, default: int, low: int, high: int, says: str) -> int: @@ -250,6 +250,7 @@ def parse_form(form, existing: PrometheusTarget | None = None) -> dict: else: token = "" return {"name": name, "url": url, "token": token, + "token_from": None if typed or existing is None else existing.token_from, "verify_tls": bool(form.get("verify_tls")), "interval_seconds": interval, "keep": patterns(form.get("keep") or ""), "drop": patterns(form.get("drop") or ""), "series_cap": cap} diff --git a/plugins/prometheus/templates/prometheus/form.html b/plugins/prometheus/templates/prometheus/form.html index 586c32a..201718b 100644 --- a/plugins/prometheus/templates/prometheus/form.html +++ b/plugins/prometheus/templates/prometheus/form.html @@ -37,8 +37,12 @@
+ placeholder="{{ 'from a sign-in; type to change' if v.token_from else ('kept; type to change' if v.has_token else 'none') }}"> + {% if v.token_from %} + Comes from the sign-in on the {{ v.token_from }} plugin's page, so that one user covers this too. Typing a token here uses that instead. + {% else %} Sent as "Authorization: Bearer …". Blank {{ "keeps the saved one" if v.has_token else "sends none" }}. + {% endif %} {% if v.has_token %}