"""Integration: the newest Traefik row per router, read by walking the (router_name, scraped_at) index rather than grouping every row (#5254). Requires STEWARD_DATABASE_URL.""" from __future__ import annotations import asyncio import os import uuid from datetime import datetime, timedelta, timezone import pytest pytestmark = pytest.mark.integration @pytest.fixture def app(): if not os.environ.get("STEWARD_DATABASE_URL"): pytest.skip("needs Postgres") from steward.app import create_app return create_app(testing=False) def test_each_router_reads_as_its_newest_scrape(app): """#5355: the newest scrape per router comes from plugin_metric_latest, which the scraper's MetricBatch keeps, not from a walk of traefik_metrics (7 s on the live instance). A backlog arriving late doesn't move it back, and a certificate's figure under the same source is not a router.""" from steward.core.alerts import MetricBatch from plugins.traefik.routes import ROUTER_METRICS, latest_per_router now = datetime.now(timezone.utc).replace(microsecond=0) tag = uuid.uuid4().hex[:8] a, b, cert = f"a-{tag}@docker", f"b-{tag}@docker", f"cert-{tag}.example" def scrape(batch, name, ago, rate): for m in ROUTER_METRICS: batch.add("traefik", name, m, rate if m == "request_rate" else 1.0, recorded_at=now - timedelta(minutes=ago)) async def go(): async with app.db_sessionmaker() as s: for name, ago, rate in [(a, 3, 1.0), (a, 1, 2.0), (b, 5, 4.0), (a, 2, 3.0)]: async with s.begin(): batch = MetricBatch() scrape(batch, name, ago, rate) await batch.record(s) async with s.begin(): batch = MetricBatch() batch.add("traefik", cert, "cert_expiry_days", 30.0, recorded_at=now) await batch.record(s) async with s.begin(): return {m.router_name: m for m in await latest_per_router(s)} got = asyncio.run(go()) assert got[a].request_rate == 2.0 and got[a].scraped_at == now - timedelta(minutes=1) assert got[a].latency_p95_ms == 1.0 and got[a].response_bytes_rate == 1.0 assert got[b].request_rate == 4.0 assert cert not in got def test_the_request_chart_sums_routers_and_weights_the_5xx_share(app): """#5256: two routers in one scrape are 4 req/s in total, not an average of 2; a 5xx share of 50% on the 3 req/s router is 37.5% of that scrape's traffic. A second scrape 30 s later shares the minute's bucket, so the bucket averages the two totals and weights the share over both.""" from plugins.traefik.models import TraefikMetric from plugins.traefik.routes import request_history # Far ahead of every other test's rows, so only these fall in the window. at = datetime(2099, 1, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=uuid.uuid4().int % 10_000) def row(name, rate, e5, when=None): return TraefikMetric(router_name=name, scraped_at=when or at, request_rate=rate, error_rate_4xx_pct=0.0, error_rate_5xx_pct=e5, latency_p50_ms=1.0, latency_p95_ms=2.0, latency_p99_ms=3.0, response_bytes_rate=0.0) async def go(): async with app.db_sessionmaker() as s: async with s.begin(): later = at + timedelta(seconds=30) s.add_all([row("quiet@docker", 1.0, 0.0), row("busy@docker", 3.0, 50.0), row("quiet@docker", 2.0, 0.0, later), row("busy@docker", 4.0, 0.0, later)]) async with s.begin(): return await request_history(s, at, 60) got = asyncio.run(go()) # One bucket: totals 4 and 6 average to 5; 150 weighted 5xx over 10 req/s is 15%. assert got == [{"at": at, "req_rate": 5.0, "err_pct": 15.0}] def test_router_history_reads_every_router_in_one_query_bucketed(app): """#5214: the rows page read each router's history in its own query, a heap fetch per row, and took 12-28 s. One query now buckets every named router; a router not asked for is not read, and one with no rows in the range still has an (empty) entry.""" from plugins.traefik.models import TraefikMetric from plugins.traefik.routes import router_history at = datetime(2098, 6, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=5 * (uuid.uuid4().int % 2_000)) tag = uuid.uuid4().hex[:8] a, b, c, idle = (f"{x}-{tag}@docker" for x in ("a", "b", "c", "idle")) def row(name, minute, rate, p95=2.0): return TraefikMetric(router_name=name, scraped_at=at + timedelta(minutes=minute), request_rate=rate, error_rate_4xx_pct=0.0, error_rate_5xx_pct=0.0, latency_p50_ms=1.0, latency_p95_ms=p95, latency_p99_ms=3.0, response_bytes_rate=0.0) async def go(): async with app.db_sessionmaker() as s: async with s.begin(): s.add_all([row(a, 0, 1.0, 10.0), row(a, 1, 3.0, 30.0), row(a, 5, 7.0), row(b, 0, 4.0), row(c, 0, 9.0), row(idle, -60, 5.0)]) async with s.begin(): return await router_history(s, [a, b, idle], at, 300) got = asyncio.run(go()) assert set(got) == {a, b, idle} # c was not asked for # a: minutes 0 and 1 share the first 5-minute bucket and average; minute 5 starts the next. assert [r.request_rate for r in got[a]] == [2.0, 7.0] assert got[a][0].latency_p95_ms == 20.0 assert [r.request_rate for r in got[b]] == [4.0] assert got[idle] == [] # its only row is before the range def test_recent_window_weights_shares_by_traffic_and_keeps_to_the_window(app): """#5214: a router is graded on its last 15 minutes. An idle scrape's 100% 4xx counts for its trickle of traffic, not as a third of the average; a row older than the window is not read.""" from plugins.traefik.models import TraefikMetric from plugins.traefik.routes import recent_window now = datetime(2097, 3, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=uuid.uuid4().int % 10_000) tag = uuid.uuid4().hex[:8] busy, empty = f"busy-{tag}@docker", f"empty-{tag}@docker" def row(name, ago, rate, e4, p95=10.0): return TraefikMetric(router_name=name, scraped_at=now - timedelta(minutes=ago), request_rate=rate, error_rate_4xx_pct=e4, error_rate_5xx_pct=0.0, latency_p50_ms=1.0, latency_p95_ms=p95, latency_p99_ms=3.0, response_bytes_rate=0.0) async def go(): async with app.db_sessionmaker() as s: async with s.begin(): s.add_all([row(busy, 1, 9.9, 0.0, 10.0), row(busy, 2, 0.1, 100.0, 30.0), row(busy, 30, 50.0, 100.0), # outside the window row(empty, 30, 1.0, 100.0)]) async with s.begin(): return await recent_window(s, [busy, empty], now) got = asyncio.run(go()) assert set(got) == {busy} # nothing in empty's window assert got[busy].error_rate_4xx_pct == pytest.approx(1.0) # 0.1 × 100 over 10 req/s assert got[busy].latency_p95_ms == pytest.approx(20.0) def test_status_codes_sum_each_routers_codes_in_the_window_most_first(app): """#5711: the codes behind a router's problem, summed over the window it is graded on; an older row and a router not asked for are not read.""" from plugins.traefik.models import TraefikStatusCode from plugins.traefik.routes import status_codes at = datetime(2097, 6, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=5 * (uuid.uuid4().int % 2_000)) tag = uuid.uuid4().hex[:8] a, b, other = (f"{x}-{tag}@docker" for x in ("a", "b", "other")) def row(name, minute, code, count): return TraefikStatusCode(router_name=name, scraped_at=at + timedelta(minutes=minute), code=code, count=count) async def go(): async with app.db_sessionmaker() as s: async with s.begin(): s.add_all([row(a, 1, "404", 30), row(a, 2, "404", 12), row(a, 2, "502", 50), row(a, -20, "500", 999), row(b, 3, "401", 4), row(other, 3, "404", 7)]) async with s.begin(): return await status_codes(s, [a, b], at) got = asyncio.run(go()) assert got == {a: [("502", 50), ("404", 42)], b: [("401", 4)]} def test_the_page_grades_a_router_against_its_normal_and_lists_its_anomalies(app): """#5741: the rows fragment and the Routers widget read each router's normal for the hour and the routers' metric anomalies. A router that answers 60% 4xx every day is healthy and says it is usual; its open p95 anomaly is listed on the page.""" import time from steward.core.alerts import MetricBatch from steward.models.norms import MetricAnomaly, MetricNorm from steward.models.users import User, UserRole from plugins.traefik.models import TraefikMetric from plugins.traefik.routes import ROUTER_METRICS now = datetime.now(timezone.utc).replace(microsecond=0) tag = uuid.uuid4().hex[:8] router, uid = f"errors-{tag}@docker", str(uuid.uuid4()) figures = {"request_rate": 5.0, "error_rate_4xx_pct": 60.0, "error_rate_5xx_pct": 0.0, "latency_p50_ms": 5.0, "latency_p95_ms": 20.0, "latency_p99_ms": 30.0, "response_bytes_rate": 100.0} async def go(): async with app.db_sessionmaker() as s: async with s.begin(): s.add(User(id=uid, username=f"tr-{tag}", email=f"tr-{tag}@test", password_hash="x", role=UserRole.viewer, is_active=True)) s.add_all([TraefikMetric(router_name=router, scraped_at=now - timedelta(minutes=m), **figures) for m in range(1, 6)]) batch = MetricBatch() for metric in ROUTER_METRICS: batch.add("traefik", router, metric, figures[metric], recorded_at=now) await batch.record(s) s.add(MetricNorm(source_module="traefik", resource_name=router, metric_name="error_rate_4xx_pct", slot=now.hour, kind="day", usual_hold=58.0, usual_peak=70.0, days_seen=7, updated_at=now)) s.add(MetricAnomaly(source_module="traefik", resource_name=router, metric_name="latency_p95_ms", started_at=now - timedelta(minutes=20), last_started_at=now - timedelta(minutes=20), occurrences=1, peak_level=900.0, peak_at=now, usual_hold=20.0, usual_peak=40.0, review="to_review")) client = app.test_client() async with client.session_transaction() as sess: sess.update(user_id=uid, user_role="viewer", username=f"tr-{tag}", login_at=time.time()) rows = await client.get("/plugins/traefik/rows?range=1h") widget = await client.get("/plugins/traefik/widget") return (rows.status_code, await rows.get_data(as_text=True), widget.status_code, await widget.get_data(as_text=True)) # CI doesn't enable the plugin (it would start scraping), so mount its # blueprint here the way plugin_manager.load_plugins does. from plugins.traefik.routes import traefik_bp app.register_blueprint(traefik_bp, url_prefix="/plugins/traefik") app.config["PLUGINS"].setdefault("traefik", {"enabled": True, "scrape_interval_seconds": 60}) rows_status, page, widget_status, widget = asyncio.run(go()) assert rows_status == 200 and widget_status == 200 assert "4xx 60.0% is usual 58.0% at this hour" in page # healthy, and says why assert "Out of its normal now" in page and router in page # the open anomaly's card assert "4xx 60.0% is usual" not in widget # the widget's why is for problems