fix(prometheus): a refused token stops the scraping, as Home Assistant bans an address after a few refusals (#6355)

Also names the Home Assistant connection by its address in the audit log and
timeline, where its id showed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/prometheus/migrations/versions/prometheus_003_refused.py

@@ -0,0 +1,23 @@
+"""A target whose token was refused stops being scraped until it is saved
+
+Revision ID: prometheus_003_refused
+Revises: prometheus_002_templates
+Create Date: 2026-11-01
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "prometheus_003_refused"
+down_revision: Union[str, None] = "prometheus_002_templates"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("prometheus_targets", sa.Column("refused", sa.Boolean, nullable=False, server_default=sa.false()))
+
+
+def downgrade() -> None:
+    op.drop_column("prometheus_targets", "refused")

plugins/prometheus/models.py

@@ -17,7 +17,8 @@ class PrometheusTarget(Base):
     patterns ("fabledarchives_*") an admin chooses: Steward does the
     filtering, the endpoint serves everything. The last_* columns are the
     newest scrape's outcome, written by the scraper, so a page can say why a
-    target isn't answering without asking it.
+    target isn't answering without asking it. `refused` says its token was
+    refused, which stops the scraping until the target is saved or resumed.
     """
     __tablename__ = "prometheus_targets"
 
@@ -31,6 +32,7 @@ class PrometheusTarget(Base):
     drop: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
     series_cap: Mapped[int] = mapped_column(Integer, nullable=False, default=1000)
     enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+    refused: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     created_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
     last_scraped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)

plugins/prometheus/routes.py

@@ -153,6 +153,7 @@ async def save_target(target_id: str):
             # Where it is or what is kept may have changed: the last outcome no
             # longer describes it, so the page says "not scraped yet" until the next scrape.
             target.last_scraped_at = target.last_ok_at = target.last_error = None
+            target.refused = False
             target.last_kept = target.last_dropped = target.last_capped = 0
             if old_name != target.name:
                 # Its history is stored under its name, and its labelled series
@@ -188,6 +189,7 @@ async def toggle_target(target_id: str):
                 # Resumed: it reads "not scraped yet" until the next scrape,
                 # rather than stale from before the pause.
                 target.last_scraped_at = target.last_error = None
+                target.refused = False
             enabled, name = target.enabled, target.name
     # A rate needs two readings in a row; one from before the pause isn't one.
     from . import scheduler

plugins/prometheus/scheduler.py

@@ -40,7 +40,7 @@ def make_scrape_task(app: "Quart") -> ScheduledTask:
 def is_due(target, now: datetime) -> bool:
     """Whether a target's interval has elapsed since its last scrape, with
     half a tick of slack so tick jitter doesn't push it back a whole tick."""
-    if not target.enabled:
+    if not target.enabled or target.refused:
         return False
     if target.last_scraped_at is None:
         return True
@@ -115,7 +115,7 @@ async def record(app: "Quart", targets, results, now: datetime) -> None:
     from steward.core.alerts import MetricBatch
 
     from .models import PrometheusTarget
-    from .scraper import ScrapeError
+    from .scraper import ScrapeError, TokenRefused
     from .service import (IS_UP, OWN_METRICS, SCRAPE_MS, SERIES_CAPPED, SERIES_KEPT, SOURCE,
                           map_families)
 
@@ -132,6 +132,7 @@ async def record(app: "Quart", targets, results, now: datetime) -> None:
                     if not isinstance(result, ScrapeError):
                         logger.warning("Prometheus scrape of %s failed: %r", target.name, result)
                     row.last_error = str(result) or "The scrape failed."
+                    row.refused = isinstance(result, TokenRefused)
                     batch.add(source_module=SOURCE, resource_name=row.name, metric_name=IS_UP,
                               value=0.0, recorded_at=now)
                     continue

plugins/prometheus/scraper.py

@@ -18,6 +18,11 @@ class ScrapeError(Exception):
     """Why a scrape got nothing, said to the person."""
 
 
+class TokenRefused(ScrapeError):
+    """The endpoint refused the token. Asking again won't change that, and
+    Home Assistant, for one, bans an address after a few refusals."""
+
+
 async def scrape(url: str, token: str = "", verify_tls: bool = True,
                  transport: httpx.AsyncBaseTransport | None = None) -> dict[str, prometheus.Family]:
     """The endpoint's families. `transport` stands in for the network in tests."""
@@ -29,8 +34,8 @@ async def scrape(url: str, token: str = "", verify_tls: bool = True,
                                      transport=transport, follow_redirects=False) as client:
             async with client.stream("GET", url, headers=headers) as response:
                 if response.status_code in (401, 403):
-                    raise ScrapeError(f"Answered {response.status_code}: the token was refused. "
-                                      "Check the token, or make a new one on the service.")
+                    raise TokenRefused(f"Answered {response.status_code}: the token was refused. "
+                                       "Check the token, or make a new one on the service.")
                 if response.status_code != 200:
                     raise ScrapeError(f"Answered {response.status_code} {response.reason_phrase}, not metrics.")
                 body = bytearray()

plugins/prometheus/service.py

@@ -271,6 +271,9 @@ class TargetState:
 def target_state(target: PrometheusTarget, now: datetime) -> TargetState:
     if not target.enabled:
         return TargetState("paused", "Paused: not scraped until it's resumed.")
+    if target.refused:
+        return TargetState("down", f"{target.last_error} Steward stopped asking, as some services ban an "
+                                   "address after a few refusals. Save a new token to start again.")
     if target.last_scraped_at is None:
         return TargetState("pending", "Not scraped yet.")
     if (now - target.last_scraped_at).total_seconds() > STALE_SCRAPES * max(target.interval_seconds, 1):