"""Integration: the newest Traefik row per router, read by walking the (router_name, scraped_at) index rather than grouping every row (#5254). Requires STEWARD_DATABASE_URL.""" from __future__ import annotations import asyncio import os import uuid from datetime import datetime, timedelta, timezone import pytest pytestmark = pytest.mark.integration @pytest.fixture def app(): if not os.environ.get("STEWARD_DATABASE_URL"): pytest.skip("needs Postgres") from steward.app import create_app return create_app(testing=False) def test_each_router_reads_as_its_newest_scrape(app): """#5355: the newest scrape per router comes from plugin_metric_latest, which the scraper's MetricBatch keeps, not from a walk of traefik_metrics (7 s on the live instance). A backlog arriving late doesn't move it back, and a certificate's figure under the same source is not a router.""" from steward.core.alerts import MetricBatch from plugins.traefik.routes import ROUTER_METRICS, latest_per_router now = datetime.now(timezone.utc).replace(microsecond=0) tag = uuid.uuid4().hex[:8] a, b, cert = f"a-{tag}@docker", f"b-{tag}@docker", f"cert-{tag}.example" def scrape(batch, name, ago, rate): for m in ROUTER_METRICS: batch.add("traefik", name, m, rate if m == "request_rate" else 1.0, recorded_at=now - timedelta(minutes=ago)) async def go(): async with app.db_sessionmaker() as s: for name, ago, rate in [(a, 3, 1.0), (a, 1, 2.0), (b, 5, 4.0), (a, 2, 3.0)]: async with s.begin(): batch = MetricBatch() scrape(batch, name, ago, rate) await batch.record(s) async with s.begin(): batch = MetricBatch() batch.add("traefik", cert, "cert_expiry_days", 30.0, recorded_at=now) await batch.record(s) async with s.begin(): return {m.router_name: m for m in await latest_per_router(s)} got = asyncio.run(go()) assert got[a].request_rate == 2.0 and got[a].scraped_at == now - timedelta(minutes=1) assert got[a].latency_p95_ms == 1.0 and got[a].response_bytes_rate == 1.0 assert got[b].request_rate == 4.0 assert cert not in got def test_the_request_chart_sums_routers_and_weights_the_5xx_share(app): """#5256: two routers in one scrape are 4 req/s in total, not an average of 2; a 5xx share of 50% on the 3 req/s router is 37.5% of that scrape's traffic. A second scrape 30 s later shares the minute's bucket, so the bucket averages the two totals and weights the share over both.""" from plugins.traefik.models import TraefikMetric from plugins.traefik.routes import request_history # Far ahead of every other test's rows, so only these fall in the window. at = datetime(2099, 1, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=uuid.uuid4().int % 10_000) def row(name, rate, e5, when=None): return TraefikMetric(router_name=name, scraped_at=when or at, request_rate=rate, error_rate_4xx_pct=0.0, error_rate_5xx_pct=e5, latency_p50_ms=1.0, latency_p95_ms=2.0, latency_p99_ms=3.0, response_bytes_rate=0.0) async def go(): async with app.db_sessionmaker() as s: async with s.begin(): later = at + timedelta(seconds=30) s.add_all([row("quiet@docker", 1.0, 0.0), row("busy@docker", 3.0, 50.0), row("quiet@docker", 2.0, 0.0, later), row("busy@docker", 4.0, 0.0, later)]) async with s.begin(): return await request_history(s, at, 60) got = asyncio.run(go()) # One bucket: totals 4 and 6 average to 5; 150 weighted 5xx over 10 req/s is 15%. assert got == [{"at": at, "req_rate": 5.0, "err_pct": 15.0}] def test_router_history_reads_every_router_in_one_query_bucketed(app): """#5214: the rows page read each router's history in its own query, a heap fetch per row, and took 12-28 s. One query now buckets every named router; a router not asked for is not read, and one with no rows in the range still has an (empty) entry.""" from plugins.traefik.models import TraefikMetric from plugins.traefik.routes import router_history at = datetime(2098, 6, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=5 * (uuid.uuid4().int % 2_000)) tag = uuid.uuid4().hex[:8] a, b, c, idle = (f"{x}-{tag}@docker" for x in ("a", "b", "c", "idle")) def row(name, minute, rate, p95=2.0): return TraefikMetric(router_name=name, scraped_at=at + timedelta(minutes=minute), request_rate=rate, error_rate_4xx_pct=0.0, error_rate_5xx_pct=0.0, latency_p50_ms=1.0, latency_p95_ms=p95, latency_p99_ms=3.0, response_bytes_rate=0.0) async def go(): async with app.db_sessionmaker() as s: async with s.begin(): s.add_all([row(a, 0, 1.0, 10.0), row(a, 1, 3.0, 30.0), row(a, 5, 7.0), row(b, 0, 4.0), row(c, 0, 9.0), row(idle, -60, 5.0)]) async with s.begin(): return await router_history(s, [a, b, idle], at, 300) got = asyncio.run(go()) assert set(got) == {a, b, idle} # c was not asked for # a: minutes 0 and 1 share the first 5-minute bucket and average; minute 5 starts the next. assert [r.request_rate for r in got[a]] == [2.0, 7.0] assert got[a][0].latency_p95_ms == 20.0 assert [r.request_rate for r in got[b]] == [4.0] assert got[idle] == [] # its only row is before the range def test_recent_window_weights_shares_by_traffic_and_keeps_to_the_window(app): """#5214: a router is graded on its last 15 minutes. An idle scrape's 100% 4xx counts for its trickle of traffic, not as a third of the average; a row older than the window is not read.""" from plugins.traefik.models import TraefikMetric from plugins.traefik.routes import recent_window now = datetime(2097, 3, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=uuid.uuid4().int % 10_000) tag = uuid.uuid4().hex[:8] busy, empty = f"busy-{tag}@docker", f"empty-{tag}@docker" def row(name, ago, rate, e4, p95=10.0): return TraefikMetric(router_name=name, scraped_at=now - timedelta(minutes=ago), request_rate=rate, error_rate_4xx_pct=e4, error_rate_5xx_pct=0.0, latency_p50_ms=1.0, latency_p95_ms=p95, latency_p99_ms=3.0, response_bytes_rate=0.0) async def go(): async with app.db_sessionmaker() as s: async with s.begin(): s.add_all([row(busy, 1, 9.9, 0.0, 10.0), row(busy, 2, 0.1, 100.0, 30.0), row(busy, 30, 50.0, 100.0), # outside the window row(empty, 30, 1.0, 100.0)]) async with s.begin(): return await recent_window(s, [busy, empty], now) got = asyncio.run(go()) assert set(got) == {busy} # nothing in empty's window assert got[busy].error_rate_4xx_pct == pytest.approx(1.0) # 0.1 × 100 over 10 req/s assert got[busy].latency_p95_ms == pytest.approx(20.0)