fix(prometheus): a refused token stops the scraping, as Home Assistant bans an address after a few refusals (#6355)
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 1m36s
CI / integration (push) Successful in 1m55s
CI / publish (push) Successful in 20s

Also names the Home Assistant connection by its address in the audit log and
timeline, where its id showed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-01 09:41:53 -05:00
co-authored by Claude Opus 5.5
parent a1dfa7b821
commit 66a5dd12fd
11 changed files with 94 additions and 11 deletions
@@ -134,3 +134,38 @@ def test_a_scrape_records_series_rates_and_a_down_target_fires_its_rule(app):
assert rows[down].last_ok_at is None
assert down_latest[(down, "is_up")] == 0.0
assert state.state == AlertStateEnum.firing
def test_a_refused_token_is_asked_once(app):
"""Home Assistant bans an address after a few refusals, so a refused token
isn't retried at each interval, however long ago the last scrape was."""
from sqlalchemy import select
scheduler = _prom("scheduler")
PrometheusTarget = _prom("models").PrometheusTarget
TokenRefused = _prom("scraper").TokenRefused
name = f"refused-{uuid.uuid4().hex[:8]}"
asked = []
async def fake_scrape(url, token, verify_tls):
asked.append(url)
raise TokenRefused("Answered 401: the token was refused.")
async def go():
async with app.db_sessionmaker() as s:
async with s.begin():
s.add(PrometheusTarget(name=name, url=f"http://{name}/metrics"))
await scheduler.scrape_due(app, scrape=fake_scrape)
async with app.db_sessionmaker() as s:
async with s.begin():
row = (await s.execute(select(PrometheusTarget).where(PrometheusTarget.name == name))).scalar_one()
refused = row.refused
row.last_scraped_at = row.last_scraped_at.replace(year=2000)
await scheduler.scrape_due(app, scrape=fake_scrape)
async with app.db_sessionmaker() as s:
async with s.begin():
await s.delete((await s.execute(select(PrometheusTarget).where(PrometheusTarget.name == name))).scalar_one())
return refused
assert asyncio.run(go()) is True
assert asked == [f"http://{name}/metrics"]
+1 -1
View File
@@ -15,7 +15,7 @@ def _target(name, *, enabled=True, scraped=timedelta(seconds=30), error=None,
url="http://archives:8080/metrics"):
return NS(id=f"id-{name}", name=name, url=url, token="bearer-s3cret", enabled=enabled,
interval_seconds=60, last_scraped_at=NOW - scraped if scraped else None,
last_ok_at=NOW - timedelta(hours=1), last_error=error, last_kept=120, last_dropped=30,
last_ok_at=NOW - timedelta(hours=1), last_error=error, refused=False, last_kept=120, last_dropped=30,
last_capped=0, series_cap=1000, keep=["archives_*"], drop=[])
+12 -2
View File
@@ -7,7 +7,7 @@ import pytest
from plugins.prometheus.models import PrometheusTarget
from plugins.prometheus.scheduler import is_due
from plugins.prometheus.scraper import ScrapeError, scrape
from plugins.prometheus.scraper import ScrapeError, TokenRefused, scrape
from plugins.prometheus.service import (FormError, fold_states, label_pairs, map_families, parse_form,
patterns, rate_name, resource_for, target_state, wanted)
from steward.core import prometheus
@@ -165,6 +165,16 @@ def test_state_and_due():
assert target_state(t, now).reach == "paused" and not is_due(t, now)
def test_a_refused_token_stops_the_scraping_and_the_page_says_so():
now = datetime(2026, 10, 12, 12, 0, tzinfo=timezone.utc)
t = PrometheusTarget(name="a", url="http://a", enabled=True, interval_seconds=60, refused=True,
last_error="Answered 401: the token was refused.",
last_scraped_at=now - timedelta(hours=5))
state = target_state(t, now)
assert state.reach == "down" and "stopped asking" in state.says and "Answered 401" in state.says
assert not is_due(t, now)
@pytest.mark.asyncio
async def test_a_scrape_sends_the_token_and_says_why_it_failed():
seen = {}
@@ -180,7 +190,7 @@ async def test_a_scrape_sends_the_token_and_says_why_it_failed():
transport = httpx.MockTransport(handler)
fams = await scrape("http://svc/metrics", "tok", transport=transport)
assert list(fams) == ["up_thing"] and seen["auth"] == "Bearer tok"
with pytest.raises(ScrapeError, match="token was refused"):
with pytest.raises(TokenRefused, match="token was refused"):
await scrape("http://svc/denied", "bad", transport=transport)
with pytest.raises(ScrapeError, match="no metrics"):
await scrape("http://svc/html", transport=transport)