bvandeusenandClaude Opus 5.5
f3cfb89612
#5741 "Traefik routers are graded against their own normal". CI run 9126: the new route test got 404, because CI doesn't enable the Traefik plugin (enabling it would start its scraper against the shared database). The test now mounts traefik_bp on its own app, as plugin_manager.load_plugins does, with the plugin.yaml scrape interval. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
239 lines
12 KiB
Python
239 lines
12 KiB
Python
"""Integration: the newest Traefik row per router, read by walking the
|
||
(router_name, scraped_at) index rather than grouping every row (#5254).
|
||
Requires STEWARD_DATABASE_URL."""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import os
|
||
import uuid
|
||
from datetime import datetime, timedelta, timezone
|
||
|
||
import pytest
|
||
|
||
pytestmark = pytest.mark.integration
|
||
|
||
|
||
@pytest.fixture
|
||
def app():
|
||
if not os.environ.get("STEWARD_DATABASE_URL"):
|
||
pytest.skip("needs Postgres")
|
||
from steward.app import create_app
|
||
return create_app(testing=False)
|
||
|
||
|
||
def test_each_router_reads_as_its_newest_scrape(app):
|
||
"""#5355: the newest scrape per router comes from plugin_metric_latest,
|
||
which the scraper's MetricBatch keeps, not from a walk of traefik_metrics
|
||
(7 s on the live instance). A backlog arriving late doesn't move it back,
|
||
and a certificate's figure under the same source is not a router."""
|
||
from steward.core.alerts import MetricBatch
|
||
from plugins.traefik.routes import ROUTER_METRICS, latest_per_router
|
||
|
||
now = datetime.now(timezone.utc).replace(microsecond=0)
|
||
tag = uuid.uuid4().hex[:8]
|
||
a, b, cert = f"a-{tag}@docker", f"b-{tag}@docker", f"cert-{tag}.example"
|
||
|
||
def scrape(batch, name, ago, rate):
|
||
for m in ROUTER_METRICS:
|
||
batch.add("traefik", name, m, rate if m == "request_rate" else 1.0,
|
||
recorded_at=now - timedelta(minutes=ago))
|
||
|
||
async def go():
|
||
async with app.db_sessionmaker() as s:
|
||
for name, ago, rate in [(a, 3, 1.0), (a, 1, 2.0), (b, 5, 4.0), (a, 2, 3.0)]:
|
||
async with s.begin():
|
||
batch = MetricBatch()
|
||
scrape(batch, name, ago, rate)
|
||
await batch.record(s)
|
||
async with s.begin():
|
||
batch = MetricBatch()
|
||
batch.add("traefik", cert, "cert_expiry_days", 30.0, recorded_at=now)
|
||
await batch.record(s)
|
||
async with s.begin():
|
||
return {m.router_name: m for m in await latest_per_router(s)}
|
||
|
||
got = asyncio.run(go())
|
||
assert got[a].request_rate == 2.0 and got[a].scraped_at == now - timedelta(minutes=1)
|
||
assert got[a].latency_p95_ms == 1.0 and got[a].response_bytes_rate == 1.0
|
||
assert got[b].request_rate == 4.0
|
||
assert cert not in got
|
||
|
||
|
||
def test_the_request_chart_sums_routers_and_weights_the_5xx_share(app):
|
||
"""#5256: two routers in one scrape are 4 req/s in total, not an average
|
||
of 2; a 5xx share of 50% on the 3 req/s router is 37.5% of that scrape's
|
||
traffic. A second scrape 30 s later shares the minute's bucket, so the
|
||
bucket averages the two totals and weights the share over both."""
|
||
from plugins.traefik.models import TraefikMetric
|
||
from plugins.traefik.routes import request_history
|
||
|
||
# Far ahead of every other test's rows, so only these fall in the window.
|
||
at = datetime(2099, 1, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=uuid.uuid4().int % 10_000)
|
||
|
||
def row(name, rate, e5, when=None):
|
||
return TraefikMetric(router_name=name, scraped_at=when or at, request_rate=rate, error_rate_4xx_pct=0.0,
|
||
error_rate_5xx_pct=e5, latency_p50_ms=1.0, latency_p95_ms=2.0,
|
||
latency_p99_ms=3.0, response_bytes_rate=0.0)
|
||
|
||
async def go():
|
||
async with app.db_sessionmaker() as s:
|
||
async with s.begin():
|
||
later = at + timedelta(seconds=30)
|
||
s.add_all([row("quiet@docker", 1.0, 0.0), row("busy@docker", 3.0, 50.0),
|
||
row("quiet@docker", 2.0, 0.0, later), row("busy@docker", 4.0, 0.0, later)])
|
||
async with s.begin():
|
||
return await request_history(s, at, 60)
|
||
|
||
got = asyncio.run(go())
|
||
# One bucket: totals 4 and 6 average to 5; 150 weighted 5xx over 10 req/s is 15%.
|
||
assert got == [{"at": at, "req_rate": 5.0, "err_pct": 15.0}]
|
||
|
||
|
||
def test_router_history_reads_every_router_in_one_query_bucketed(app):
|
||
"""#5214: the rows page read each router's history in its own query, a
|
||
heap fetch per row, and took 12-28 s. One query now buckets every named
|
||
router; a router not asked for is not read, and one with no rows in the
|
||
range still has an (empty) entry."""
|
||
from plugins.traefik.models import TraefikMetric
|
||
from plugins.traefik.routes import router_history
|
||
|
||
at = datetime(2098, 6, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=5 * (uuid.uuid4().int % 2_000))
|
||
tag = uuid.uuid4().hex[:8]
|
||
a, b, c, idle = (f"{x}-{tag}@docker" for x in ("a", "b", "c", "idle"))
|
||
|
||
def row(name, minute, rate, p95=2.0):
|
||
return TraefikMetric(router_name=name, scraped_at=at + timedelta(minutes=minute), request_rate=rate,
|
||
error_rate_4xx_pct=0.0, error_rate_5xx_pct=0.0, latency_p50_ms=1.0,
|
||
latency_p95_ms=p95, latency_p99_ms=3.0, response_bytes_rate=0.0)
|
||
|
||
async def go():
|
||
async with app.db_sessionmaker() as s:
|
||
async with s.begin():
|
||
s.add_all([row(a, 0, 1.0, 10.0), row(a, 1, 3.0, 30.0), row(a, 5, 7.0),
|
||
row(b, 0, 4.0), row(c, 0, 9.0), row(idle, -60, 5.0)])
|
||
async with s.begin():
|
||
return await router_history(s, [a, b, idle], at, 300)
|
||
|
||
got = asyncio.run(go())
|
||
assert set(got) == {a, b, idle} # c was not asked for
|
||
# a: minutes 0 and 1 share the first 5-minute bucket and average; minute 5 starts the next.
|
||
assert [r.request_rate for r in got[a]] == [2.0, 7.0]
|
||
assert got[a][0].latency_p95_ms == 20.0
|
||
assert [r.request_rate for r in got[b]] == [4.0]
|
||
assert got[idle] == [] # its only row is before the range
|
||
|
||
|
||
def test_recent_window_weights_shares_by_traffic_and_keeps_to_the_window(app):
|
||
"""#5214: a router is graded on its last 15 minutes. An idle scrape's
|
||
100% 4xx counts for its trickle of traffic, not as a third of the
|
||
average; a row older than the window is not read."""
|
||
from plugins.traefik.models import TraefikMetric
|
||
from plugins.traefik.routes import recent_window
|
||
|
||
now = datetime(2097, 3, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=uuid.uuid4().int % 10_000)
|
||
tag = uuid.uuid4().hex[:8]
|
||
busy, empty = f"busy-{tag}@docker", f"empty-{tag}@docker"
|
||
|
||
def row(name, ago, rate, e4, p95=10.0):
|
||
return TraefikMetric(router_name=name, scraped_at=now - timedelta(minutes=ago), request_rate=rate,
|
||
error_rate_4xx_pct=e4, error_rate_5xx_pct=0.0, latency_p50_ms=1.0,
|
||
latency_p95_ms=p95, latency_p99_ms=3.0, response_bytes_rate=0.0)
|
||
|
||
async def go():
|
||
async with app.db_sessionmaker() as s:
|
||
async with s.begin():
|
||
s.add_all([row(busy, 1, 9.9, 0.0, 10.0), row(busy, 2, 0.1, 100.0, 30.0),
|
||
row(busy, 30, 50.0, 100.0), # outside the window
|
||
row(empty, 30, 1.0, 100.0)])
|
||
async with s.begin():
|
||
return await recent_window(s, [busy, empty], now)
|
||
|
||
got = asyncio.run(go())
|
||
assert set(got) == {busy} # nothing in empty's window
|
||
assert got[busy].error_rate_4xx_pct == pytest.approx(1.0) # 0.1 × 100 over 10 req/s
|
||
assert got[busy].latency_p95_ms == pytest.approx(20.0)
|
||
|
||
|
||
def test_status_codes_sum_each_routers_codes_in_the_window_most_first(app):
|
||
"""#5711: the codes behind a router's problem, summed over the window it
|
||
is graded on; an older row and a router not asked for are not read."""
|
||
from plugins.traefik.models import TraefikStatusCode
|
||
from plugins.traefik.routes import status_codes
|
||
|
||
at = datetime(2097, 6, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=5 * (uuid.uuid4().int % 2_000))
|
||
tag = uuid.uuid4().hex[:8]
|
||
a, b, other = (f"{x}-{tag}@docker" for x in ("a", "b", "other"))
|
||
|
||
def row(name, minute, code, count):
|
||
return TraefikStatusCode(router_name=name, scraped_at=at + timedelta(minutes=minute), code=code, count=count)
|
||
|
||
async def go():
|
||
async with app.db_sessionmaker() as s:
|
||
async with s.begin():
|
||
s.add_all([row(a, 1, "404", 30), row(a, 2, "404", 12), row(a, 2, "502", 50),
|
||
row(a, -20, "500", 999), row(b, 3, "401", 4), row(other, 3, "404", 7)])
|
||
async with s.begin():
|
||
return await status_codes(s, [a, b], at)
|
||
|
||
got = asyncio.run(go())
|
||
assert got == {a: [("502", 50), ("404", 42)], b: [("401", 4)]}
|
||
|
||
|
||
def test_the_page_grades_a_router_against_its_normal_and_lists_its_anomalies(app):
|
||
"""#5741: the rows fragment and the Routers widget read each router's
|
||
normal for the hour and the routers' metric anomalies. A router that
|
||
answers 60% 4xx every day is healthy and says it is usual; its open p95
|
||
anomaly is listed on the page."""
|
||
import time
|
||
from steward.core.alerts import MetricBatch
|
||
from steward.models.norms import MetricAnomaly, MetricNorm
|
||
from steward.models.users import User, UserRole
|
||
from plugins.traefik.models import TraefikMetric
|
||
from plugins.traefik.routes import ROUTER_METRICS
|
||
|
||
now = datetime.now(timezone.utc).replace(microsecond=0)
|
||
tag = uuid.uuid4().hex[:8]
|
||
router, uid = f"errors-{tag}@docker", str(uuid.uuid4())
|
||
figures = {"request_rate": 5.0, "error_rate_4xx_pct": 60.0, "error_rate_5xx_pct": 0.0,
|
||
"latency_p50_ms": 5.0, "latency_p95_ms": 20.0, "latency_p99_ms": 30.0,
|
||
"response_bytes_rate": 100.0}
|
||
|
||
async def go():
|
||
async with app.db_sessionmaker() as s:
|
||
async with s.begin():
|
||
s.add(User(id=uid, username=f"tr-{tag}", email=f"tr-{tag}@test", password_hash="x",
|
||
role=UserRole.viewer, is_active=True))
|
||
s.add_all([TraefikMetric(router_name=router, scraped_at=now - timedelta(minutes=m),
|
||
**figures) for m in range(1, 6)])
|
||
batch = MetricBatch()
|
||
for metric in ROUTER_METRICS:
|
||
batch.add("traefik", router, metric, figures[metric], recorded_at=now)
|
||
await batch.record(s)
|
||
s.add(MetricNorm(source_module="traefik", resource_name=router,
|
||
metric_name="error_rate_4xx_pct", slot=now.hour, kind="day",
|
||
usual_hold=58.0, usual_peak=70.0, days_seen=7, updated_at=now))
|
||
s.add(MetricAnomaly(source_module="traefik", resource_name=router,
|
||
metric_name="latency_p95_ms", started_at=now - timedelta(minutes=20),
|
||
last_started_at=now - timedelta(minutes=20), occurrences=1,
|
||
peak_level=900.0, peak_at=now, usual_hold=20.0, usual_peak=40.0,
|
||
review="to_review"))
|
||
client = app.test_client()
|
||
async with client.session_transaction() as sess:
|
||
sess.update(user_id=uid, user_role="viewer", username=f"tr-{tag}", login_at=time.time())
|
||
rows = await client.get("/plugins/traefik/rows?range=1h")
|
||
widget = await client.get("/plugins/traefik/widget")
|
||
return (rows.status_code, await rows.get_data(as_text=True),
|
||
widget.status_code, await widget.get_data(as_text=True))
|
||
|
||
# CI doesn't enable the plugin (it would start scraping), so mount its
|
||
# blueprint here the way plugin_manager.load_plugins does.
|
||
from plugins.traefik.routes import traefik_bp
|
||
app.register_blueprint(traefik_bp, url_prefix="/plugins/traefik")
|
||
app.config["PLUGINS"].setdefault("traefik", {"enabled": True, "scrape_interval_seconds": 60})
|
||
|
||
rows_status, page, widget_status, widget = asyncio.run(go())
|
||
assert rows_status == 200 and widget_status == 200
|
||
assert "4xx 60.0% is usual 58.0% at this hour" in page # healthy, and says why
|
||
assert "Out of its normal now" in page and router in page # the open anomaly's card
|
||
assert "4xx 60.0% is usual" not in widget # the widget's why is for problems
|