fix(prometheus): a refused token stops the scraping, as Home Assistant bans an address after a few refusals (#6355)
Also names the Home Assistant connection by its address in the audit log and timeline, where its id showed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -134,3 +134,38 @@ def test_a_scrape_records_series_rates_and_a_down_target_fires_its_rule(app):
|
||||
assert rows[down].last_ok_at is None
|
||||
assert down_latest[(down, "is_up")] == 0.0
|
||||
assert state.state == AlertStateEnum.firing
|
||||
|
||||
|
||||
def test_a_refused_token_is_asked_once(app):
|
||||
"""Home Assistant bans an address after a few refusals, so a refused token
|
||||
isn't retried at each interval, however long ago the last scrape was."""
|
||||
from sqlalchemy import select
|
||||
|
||||
scheduler = _prom("scheduler")
|
||||
PrometheusTarget = _prom("models").PrometheusTarget
|
||||
TokenRefused = _prom("scraper").TokenRefused
|
||||
name = f"refused-{uuid.uuid4().hex[:8]}"
|
||||
asked = []
|
||||
|
||||
async def fake_scrape(url, token, verify_tls):
|
||||
asked.append(url)
|
||||
raise TokenRefused("Answered 401: the token was refused.")
|
||||
|
||||
async def go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
s.add(PrometheusTarget(name=name, url=f"http://{name}/metrics"))
|
||||
await scheduler.scrape_due(app, scrape=fake_scrape)
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
row = (await s.execute(select(PrometheusTarget).where(PrometheusTarget.name == name))).scalar_one()
|
||||
refused = row.refused
|
||||
row.last_scraped_at = row.last_scraped_at.replace(year=2000)
|
||||
await scheduler.scrape_due(app, scrape=fake_scrape)
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await s.delete((await s.execute(select(PrometheusTarget).where(PrometheusTarget.name == name))).scalar_one())
|
||||
return refused
|
||||
|
||||
assert asyncio.run(go()) is True
|
||||
assert asked == [f"http://{name}/metrics"]
|
||||
|
||||
@@ -15,7 +15,7 @@ def _target(name, *, enabled=True, scraped=timedelta(seconds=30), error=None,
|
||||
url="http://archives:8080/metrics"):
|
||||
return NS(id=f"id-{name}", name=name, url=url, token="bearer-s3cret", enabled=enabled,
|
||||
interval_seconds=60, last_scraped_at=NOW - scraped if scraped else None,
|
||||
last_ok_at=NOW - timedelta(hours=1), last_error=error, last_kept=120, last_dropped=30,
|
||||
last_ok_at=NOW - timedelta(hours=1), last_error=error, refused=False, last_kept=120, last_dropped=30,
|
||||
last_capped=0, series_cap=1000, keep=["archives_*"], drop=[])
|
||||
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import pytest
|
||||
|
||||
from plugins.prometheus.models import PrometheusTarget
|
||||
from plugins.prometheus.scheduler import is_due
|
||||
from plugins.prometheus.scraper import ScrapeError, scrape
|
||||
from plugins.prometheus.scraper import ScrapeError, TokenRefused, scrape
|
||||
from plugins.prometheus.service import (FormError, fold_states, label_pairs, map_families, parse_form,
|
||||
patterns, rate_name, resource_for, target_state, wanted)
|
||||
from steward.core import prometheus
|
||||
@@ -165,6 +165,16 @@ def test_state_and_due():
|
||||
assert target_state(t, now).reach == "paused" and not is_due(t, now)
|
||||
|
||||
|
||||
def test_a_refused_token_stops_the_scraping_and_the_page_says_so():
|
||||
now = datetime(2026, 10, 12, 12, 0, tzinfo=timezone.utc)
|
||||
t = PrometheusTarget(name="a", url="http://a", enabled=True, interval_seconds=60, refused=True,
|
||||
last_error="Answered 401: the token was refused.",
|
||||
last_scraped_at=now - timedelta(hours=5))
|
||||
state = target_state(t, now)
|
||||
assert state.reach == "down" and "stopped asking" in state.says and "Answered 401" in state.says
|
||||
assert not is_due(t, now)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_scrape_sends_the_token_and_says_why_it_failed():
|
||||
seen = {}
|
||||
@@ -180,7 +190,7 @@ async def test_a_scrape_sends_the_token_and_says_why_it_failed():
|
||||
transport = httpx.MockTransport(handler)
|
||||
fams = await scrape("http://svc/metrics", "tok", transport=transport)
|
||||
assert list(fams) == ["up_thing"] and seen["auth"] == "Bearer tok"
|
||||
with pytest.raises(ScrapeError, match="token was refused"):
|
||||
with pytest.raises(TokenRefused, match="token was refused"):
|
||||
await scrape("http://svc/denied", "bad", transport=transport)
|
||||
with pytest.raises(ScrapeError, match="no metrics"):
|
||||
await scrape("http://svc/html", transport=transport)
|
||||
|
||||
Reference in New Issue
Block a user