fix(prometheus): a refused token stops the scraping, as Home Assistant bans an address after a few refusals (#6355)

Also names the Home Assistant connection by its address in the audit log and
timeline, where its id showed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/plugins/prometheus.md

@@ -46,6 +46,8 @@ Each target also records four metrics about itself, under its own name:
 
 A value's unit is read from Prometheus naming: `_bytes` as bytes, `_seconds` as a span, `_timestamp_seconds` as a moment ("3h ago"), `_per_sec` as a rate.
 
+**A refused token stops the scraping.** When an endpoint answers 401 or 403, Steward stops asking and says so on the target's page and in the overview, because asking again won't change the answer and some services, Home Assistant among them, ban an address after a few refusals. Saving the target, usually with a new token, or pausing and resuming it, starts it again. Any other failure is retried at the next scrape.
+
 A **state set** is one series per possible state, the current one 1 and the rest 0, such as `fabledarchives_backup_target_state{target, state}`. The target page shows it as one row per target (or job), naming the current state; the other states are listed on hover. A metric is read as a state set when its name ends in `_state` and every series has a `state` label reading 0 or 1. The OpenMetrics form, where the label is named after the metric (`foo{foo="a"} 1`), counts too. Each state is still stored as its own series, so an alert rule can watch one state.
 
 **Only what the endpoint reports now is current.** When a scrape succeeds, any series it no longer has loses its newest value. That happens when the endpoint stopped reporting it, for example an old `build_info` after an upgrade, or when a filter or the cap now leaves it out. Its history stays on the chart until retention removes it. A failed scrape leaves the newest values as they were.

plugins/homeassistant/routes.py

@@ -174,7 +174,7 @@ async def sign_in_callback():
                                            verify_tls=previous[3]))
     await log_audit(current_app, user_id, username, "homeassistant.connection_saved",
                     entity_type="homeassistant_connection", entity_id=connection_id,
-                    detail={"url": url, "can_install": can_install})
+                    detail={"name": url, "can_install": can_install})
     return redirect("/plugins/homeassistant/")
 
 
@@ -209,11 +209,11 @@ async def delete_connection():
             connection = await _connection(db)
             if connection is None:
                 return redirect("/plugins/homeassistant/")
-            connection_id = connection.id
+            connection_id, url = connection.id, connection.url
             await db.delete(connection)
     ended = await _end_sign_in(connection)
     await log_audit(current_app, *_actor(), "homeassistant.connection_removed",
-                    entity_type="homeassistant_connection", entity_id=connection_id, detail={"sign_in_ended": ended})
+                    entity_type="homeassistant_connection", entity_id=connection_id, detail={"name": url, "sign_in_ended": ended})
     if not ended:
         return await _render_index("Removed from Steward, but Home Assistant couldn't be reached to end the sign-in. "
                                    "End it yourself under Security on that user's profile in Home Assistant.")

plugins/prometheus/migrations/versions/prometheus_003_refused.py

@@ -0,0 +1,23 @@
+"""A target whose token was refused stops being scraped until it is saved
+
+Revision ID: prometheus_003_refused
+Revises: prometheus_002_templates
+Create Date: 2026-11-01
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "prometheus_003_refused"
+down_revision: Union[str, None] = "prometheus_002_templates"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("prometheus_targets", sa.Column("refused", sa.Boolean, nullable=False, server_default=sa.false()))
+
+
+def downgrade() -> None:
+    op.drop_column("prometheus_targets", "refused")

plugins/prometheus/models.py

@@ -17,7 +17,8 @@ class PrometheusTarget(Base):
     patterns ("fabledarchives_*") an admin chooses: Steward does the
     filtering, the endpoint serves everything. The last_* columns are the
     newest scrape's outcome, written by the scraper, so a page can say why a
-    target isn't answering without asking it.
+    target isn't answering without asking it. `refused` says its token was
+    refused, which stops the scraping until the target is saved or resumed.
     """
     __tablename__ = "prometheus_targets"
 
@@ -31,6 +32,7 @@ class PrometheusTarget(Base):
     drop: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
     series_cap: Mapped[int] = mapped_column(Integer, nullable=False, default=1000)
     enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+    refused: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     created_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
     last_scraped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)

plugins/prometheus/routes.py

@@ -153,6 +153,7 @@ async def save_target(target_id: str):
             # Where it is or what is kept may have changed: the last outcome no
             # longer describes it, so the page says "not scraped yet" until the next scrape.
             target.last_scraped_at = target.last_ok_at = target.last_error = None
+            target.refused = False
             target.last_kept = target.last_dropped = target.last_capped = 0
             if old_name != target.name:
                 # Its history is stored under its name, and its labelled series
@@ -188,6 +189,7 @@ async def toggle_target(target_id: str):
                 # Resumed: it reads "not scraped yet" until the next scrape,
                 # rather than stale from before the pause.
                 target.last_scraped_at = target.last_error = None
+                target.refused = False
             enabled, name = target.enabled, target.name
     # A rate needs two readings in a row; one from before the pause isn't one.
     from . import scheduler

plugins/prometheus/scheduler.py

@@ -40,7 +40,7 @@ def make_scrape_task(app: "Quart") -> ScheduledTask:
 def is_due(target, now: datetime) -> bool:
     """Whether a target's interval has elapsed since its last scrape, with
     half a tick of slack so tick jitter doesn't push it back a whole tick."""
-    if not target.enabled:
+    if not target.enabled or target.refused:
         return False
     if target.last_scraped_at is None:
         return True
@@ -115,7 +115,7 @@ async def record(app: "Quart", targets, results, now: datetime) -> None:
     from steward.core.alerts import MetricBatch
 
     from .models import PrometheusTarget
-    from .scraper import ScrapeError
+    from .scraper import ScrapeError, TokenRefused
     from .service import (IS_UP, OWN_METRICS, SCRAPE_MS, SERIES_CAPPED, SERIES_KEPT, SOURCE,
                           map_families)
 
@@ -132,6 +132,7 @@ async def record(app: "Quart", targets, results, now: datetime) -> None:
                     if not isinstance(result, ScrapeError):
                         logger.warning("Prometheus scrape of %s failed: %r", target.name, result)
                     row.last_error = str(result) or "The scrape failed."
+                    row.refused = isinstance(result, TokenRefused)
                     batch.add(source_module=SOURCE, resource_name=row.name, metric_name=IS_UP,
                               value=0.0, recorded_at=now)
                     continue

plugins/prometheus/scraper.py

@@ -18,6 +18,11 @@ class ScrapeError(Exception):
     """Why a scrape got nothing, said to the person."""
 
 
+class TokenRefused(ScrapeError):
+    """The endpoint refused the token. Asking again won't change that, and
+    Home Assistant, for one, bans an address after a few refusals."""
+
+
 async def scrape(url: str, token: str = "", verify_tls: bool = True,
                  transport: httpx.AsyncBaseTransport | None = None) -> dict[str, prometheus.Family]:
     """The endpoint's families. `transport` stands in for the network in tests."""
@@ -29,8 +34,8 @@ async def scrape(url: str, token: str = "", verify_tls: bool = True,
                                      transport=transport, follow_redirects=False) as client:
             async with client.stream("GET", url, headers=headers) as response:
                 if response.status_code in (401, 403):
-                    raise ScrapeError(f"Answered {response.status_code}: the token was refused. "
-                                      "Check the token, or make a new one on the service.")
+                    raise TokenRefused(f"Answered {response.status_code}: the token was refused. "
+                                       "Check the token, or make a new one on the service.")
                 if response.status_code != 200:
                     raise ScrapeError(f"Answered {response.status_code} {response.reason_phrase}, not metrics.")
                 body = bytearray()

plugins/prometheus/service.py

@@ -271,6 +271,9 @@ class TargetState:
 def target_state(target: PrometheusTarget, now: datetime) -> TargetState:
     if not target.enabled:
         return TargetState("paused", "Paused: not scraped until it's resumed.")
+    if target.refused:
+        return TargetState("down", f"{target.last_error} Steward stopped asking, as some services ban an "
+                                   "address after a few refusals. Save a new token to start again.")
     if target.last_scraped_at is None:
         return TargetState("pending", "Not scraped yet.")
     if (now - target.last_scraped_at).total_seconds() > STALE_SCRAPES * max(target.interval_seconds, 1):

tests/integration/test_prometheus_scrape.py

@@ -134,3 +134,38 @@ def test_a_scrape_records_series_rates_and_a_down_target_fires_its_rule(app):
     assert rows[down].last_ok_at is None
     assert down_latest[(down, "is_up")] == 0.0
     assert state.state == AlertStateEnum.firing
+
+
+def test_a_refused_token_is_asked_once(app):
+    """Home Assistant bans an address after a few refusals, so a refused token
+    isn't retried at each interval, however long ago the last scrape was."""
+    from sqlalchemy import select
+
+    scheduler = _prom("scheduler")
+    PrometheusTarget = _prom("models").PrometheusTarget
+    TokenRefused = _prom("scraper").TokenRefused
+    name = f"refused-{uuid.uuid4().hex[:8]}"
+    asked = []
+
+    async def fake_scrape(url, token, verify_tls):
+        asked.append(url)
+        raise TokenRefused("Answered 401: the token was refused.")
+
+    async def go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add(PrometheusTarget(name=name, url=f"http://{name}/metrics"))
+        await scheduler.scrape_due(app, scrape=fake_scrape)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                row = (await s.execute(select(PrometheusTarget).where(PrometheusTarget.name == name))).scalar_one()
+                refused = row.refused
+                row.last_scraped_at = row.last_scraped_at.replace(year=2000)
+        await scheduler.scrape_due(app, scrape=fake_scrape)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await s.delete((await s.execute(select(PrometheusTarget).where(PrometheusTarget.name == name))).scalar_one())
+        return refused
+
+    assert asyncio.run(go()) is True
+    assert asked == [f"http://{name}/metrics"]

tests/plugins/prometheus/test_mcp_tools.py

@@ -15,7 +15,7 @@ def _target(name, *, enabled=True, scraped=timedelta(seconds=30), error=None,
             url="http://archives:8080/metrics"):
     return NS(id=f"id-{name}", name=name, url=url, token="bearer-s3cret", enabled=enabled,
               interval_seconds=60, last_scraped_at=NOW - scraped if scraped else None,
-              last_ok_at=NOW - timedelta(hours=1), last_error=error, last_kept=120, last_dropped=30,
+              last_ok_at=NOW - timedelta(hours=1), last_error=error, refused=False, last_kept=120, last_dropped=30,
               last_capped=0, series_cap=1000, keep=["archives_*"], drop=[])
 
 

tests/plugins/prometheus/test_service.py

@@ -7,7 +7,7 @@ import pytest
 
 from plugins.prometheus.models import PrometheusTarget
 from plugins.prometheus.scheduler import is_due
-from plugins.prometheus.scraper import ScrapeError, scrape
+from plugins.prometheus.scraper import ScrapeError, TokenRefused, scrape
 from plugins.prometheus.service import (FormError, fold_states, label_pairs, map_families, parse_form,
                                         patterns, rate_name, resource_for, target_state, wanted)
 from steward.core import prometheus
@@ -165,6 +165,16 @@ def test_state_and_due():
     assert target_state(t, now).reach == "paused" and not is_due(t, now)
 
 
+def test_a_refused_token_stops_the_scraping_and_the_page_says_so():
+    now = datetime(2026, 10, 12, 12, 0, tzinfo=timezone.utc)
+    t = PrometheusTarget(name="a", url="http://a", enabled=True, interval_seconds=60, refused=True,
+                         last_error="Answered 401: the token was refused.",
+                         last_scraped_at=now - timedelta(hours=5))
+    state = target_state(t, now)
+    assert state.reach == "down" and "stopped asking" in state.says and "Answered 401" in state.says
+    assert not is_due(t, now)
+
+
 @pytest.mark.asyncio
 async def test_a_scrape_sends_the_token_and_says_why_it_failed():
     seen = {}
@@ -180,7 +190,7 @@ async def test_a_scrape_sends_the_token_and_says_why_it_failed():
     transport = httpx.MockTransport(handler)
     fams = await scrape("http://svc/metrics", "tok", transport=transport)
     assert list(fams) == ["up_thing"] and seen["auth"] == "Bearer tok"
-    with pytest.raises(ScrapeError, match="token was refused"):
+    with pytest.raises(TokenRefused, match="token was refused"):
         await scrape("http://svc/denied", "bad", transport=transport)
     with pytest.raises(ScrapeError, match="no metrics"):
         await scrape("http://svc/html", transport=transport)