feat(traefik): keep each router's 4xx and 5xx responses by status code (#5711)
The scraper folded traefik_service_requests_total{code} into two shares and
dropped the code, so a problem couldn't say whether it was scanners (404), a
login wall (401/403) or the app down (502/503/504). Each scrape now stores the
responses per router and code that occurred (traefik_status_codes, migration
traefik_006), pruned with the router metrics and left out of backups (System
180 ruling: Traefik history is not backed up). A problem card lists its codes
over the last 15 minutes, most first, or says none occurred, or that codes
aren't recorded yet.
A counter series new since the previous scrape now counts from 0 instead of
being skipped: Traefik creates a router's 502 series at its first 502, so a
first burst of 5xx read as 0% until it happened again.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -152,3 +152,28 @@ def test_recent_window_weights_shares_by_traffic_and_keeps_to_the_window(app):
|
||||
assert set(got) == {busy} # nothing in empty's window
|
||||
assert got[busy].error_rate_4xx_pct == pytest.approx(1.0) # 0.1 × 100 over 10 req/s
|
||||
assert got[busy].latency_p95_ms == pytest.approx(20.0)
|
||||
|
||||
|
||||
def test_status_codes_sum_each_routers_codes_in_the_window_most_first(app):
|
||||
"""#5711: the codes behind a router's problem, summed over the window it
|
||||
is graded on; an older row and a router not asked for are not read."""
|
||||
from plugins.traefik.models import TraefikStatusCode
|
||||
from plugins.traefik.routes import status_codes
|
||||
|
||||
at = datetime(2097, 6, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=5 * (uuid.uuid4().int % 2_000))
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
a, b, other = (f"{x}-{tag}@docker" for x in ("a", "b", "other"))
|
||||
|
||||
def row(name, minute, code, count):
|
||||
return TraefikStatusCode(router_name=name, scraped_at=at + timedelta(minutes=minute), code=code, count=count)
|
||||
|
||||
async def go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
s.add_all([row(a, 1, "404", 30), row(a, 2, "404", 12), row(a, 2, "502", 50),
|
||||
row(a, -20, "500", 999), row(b, 3, "401", 4), row(other, 3, "404", 7)])
|
||||
async with s.begin():
|
||||
return await status_codes(s, [a, b], at)
|
||||
|
||||
got = asyncio.run(go())
|
||||
assert got == {a: [("502", 50), ("404", 42)], b: [("401", 4)]}
|
||||
|
||||
Reference in New Issue
Block a user