bvandeusenandClaude Opus 5.5
5f04f6af66
The phase log on the live instance: 'traefik rows: slow 7.4s (latest=7.04s history=0.11s window=0.04s ...)'. latest_per_router walked traefik_metrics' index one router at a time; each step lands on the router's oldest entries, which retention deletes every minute (likely dead ones). The scraper already records every router figure through MetricBatch, which keeps one row per router and figure in plugin_metric_latest (#5208's table, read as latest_by_resource does); latest_per_router reads that now, and the router figures carry the scrape's own time. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
155 lines
7.3 KiB
Python
155 lines
7.3 KiB
Python
"""Integration: the newest Traefik row per router, read by walking the
|
||
(router_name, scraped_at) index rather than grouping every row (#5254).
|
||
Requires STEWARD_DATABASE_URL."""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import os
|
||
import uuid
|
||
from datetime import datetime, timedelta, timezone
|
||
|
||
import pytest
|
||
|
||
pytestmark = pytest.mark.integration
|
||
|
||
|
||
@pytest.fixture
|
||
def app():
|
||
if not os.environ.get("STEWARD_DATABASE_URL"):
|
||
pytest.skip("needs Postgres")
|
||
from steward.app import create_app
|
||
return create_app(testing=False)
|
||
|
||
|
||
def test_each_router_reads_as_its_newest_scrape(app):
|
||
"""#5355: the newest scrape per router comes from plugin_metric_latest,
|
||
which the scraper's MetricBatch keeps, not from a walk of traefik_metrics
|
||
(7 s on the live instance). A backlog arriving late doesn't move it back,
|
||
and a certificate's figure under the same source is not a router."""
|
||
from steward.core.alerts import MetricBatch
|
||
from plugins.traefik.routes import ROUTER_METRICS, latest_per_router
|
||
|
||
now = datetime.now(timezone.utc).replace(microsecond=0)
|
||
tag = uuid.uuid4().hex[:8]
|
||
a, b, cert = f"a-{tag}@docker", f"b-{tag}@docker", f"cert-{tag}.example"
|
||
|
||
def scrape(batch, name, ago, rate):
|
||
for m in ROUTER_METRICS:
|
||
batch.add("traefik", name, m, rate if m == "request_rate" else 1.0,
|
||
recorded_at=now - timedelta(minutes=ago))
|
||
|
||
async def go():
|
||
async with app.db_sessionmaker() as s:
|
||
for name, ago, rate in [(a, 3, 1.0), (a, 1, 2.0), (b, 5, 4.0), (a, 2, 3.0)]:
|
||
async with s.begin():
|
||
batch = MetricBatch()
|
||
scrape(batch, name, ago, rate)
|
||
await batch.record(s)
|
||
async with s.begin():
|
||
batch = MetricBatch()
|
||
batch.add("traefik", cert, "cert_expiry_days", 30.0, recorded_at=now)
|
||
await batch.record(s)
|
||
async with s.begin():
|
||
return {m.router_name: m for m in await latest_per_router(s)}
|
||
|
||
got = asyncio.run(go())
|
||
assert got[a].request_rate == 2.0 and got[a].scraped_at == now - timedelta(minutes=1)
|
||
assert got[a].latency_p95_ms == 1.0 and got[a].response_bytes_rate == 1.0
|
||
assert got[b].request_rate == 4.0
|
||
assert cert not in got
|
||
|
||
|
||
def test_the_request_chart_sums_routers_and_weights_the_5xx_share(app):
|
||
"""#5256: two routers in one scrape are 4 req/s in total, not an average
|
||
of 2; a 5xx share of 50% on the 3 req/s router is 37.5% of that scrape's
|
||
traffic. A second scrape 30 s later shares the minute's bucket, so the
|
||
bucket averages the two totals and weights the share over both."""
|
||
from plugins.traefik.models import TraefikMetric
|
||
from plugins.traefik.routes import request_history
|
||
|
||
# Far ahead of every other test's rows, so only these fall in the window.
|
||
at = datetime(2099, 1, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=uuid.uuid4().int % 10_000)
|
||
|
||
def row(name, rate, e5, when=None):
|
||
return TraefikMetric(router_name=name, scraped_at=when or at, request_rate=rate, error_rate_4xx_pct=0.0,
|
||
error_rate_5xx_pct=e5, latency_p50_ms=1.0, latency_p95_ms=2.0,
|
||
latency_p99_ms=3.0, response_bytes_rate=0.0)
|
||
|
||
async def go():
|
||
async with app.db_sessionmaker() as s:
|
||
async with s.begin():
|
||
later = at + timedelta(seconds=30)
|
||
s.add_all([row("quiet@docker", 1.0, 0.0), row("busy@docker", 3.0, 50.0),
|
||
row("quiet@docker", 2.0, 0.0, later), row("busy@docker", 4.0, 0.0, later)])
|
||
async with s.begin():
|
||
return await request_history(s, at, 60)
|
||
|
||
got = asyncio.run(go())
|
||
# One bucket: totals 4 and 6 average to 5; 150 weighted 5xx over 10 req/s is 15%.
|
||
assert got == [{"at": at, "req_rate": 5.0, "err_pct": 15.0}]
|
||
|
||
|
||
def test_router_history_reads_every_router_in_one_query_bucketed(app):
|
||
"""#5214: the rows page read each router's history in its own query, a
|
||
heap fetch per row, and took 12-28 s. One query now buckets every named
|
||
router; a router not asked for is not read, and one with no rows in the
|
||
range still has an (empty) entry."""
|
||
from plugins.traefik.models import TraefikMetric
|
||
from plugins.traefik.routes import router_history
|
||
|
||
at = datetime(2098, 6, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=5 * (uuid.uuid4().int % 2_000))
|
||
tag = uuid.uuid4().hex[:8]
|
||
a, b, c, idle = (f"{x}-{tag}@docker" for x in ("a", "b", "c", "idle"))
|
||
|
||
def row(name, minute, rate, p95=2.0):
|
||
return TraefikMetric(router_name=name, scraped_at=at + timedelta(minutes=minute), request_rate=rate,
|
||
error_rate_4xx_pct=0.0, error_rate_5xx_pct=0.0, latency_p50_ms=1.0,
|
||
latency_p95_ms=p95, latency_p99_ms=3.0, response_bytes_rate=0.0)
|
||
|
||
async def go():
|
||
async with app.db_sessionmaker() as s:
|
||
async with s.begin():
|
||
s.add_all([row(a, 0, 1.0, 10.0), row(a, 1, 3.0, 30.0), row(a, 5, 7.0),
|
||
row(b, 0, 4.0), row(c, 0, 9.0), row(idle, -60, 5.0)])
|
||
async with s.begin():
|
||
return await router_history(s, [a, b, idle], at, 300)
|
||
|
||
got = asyncio.run(go())
|
||
assert set(got) == {a, b, idle} # c was not asked for
|
||
# a: minutes 0 and 1 share the first 5-minute bucket and average; minute 5 starts the next.
|
||
assert [r.request_rate for r in got[a]] == [2.0, 7.0]
|
||
assert got[a][0].latency_p95_ms == 20.0
|
||
assert [r.request_rate for r in got[b]] == [4.0]
|
||
assert got[idle] == [] # its only row is before the range
|
||
|
||
|
||
def test_recent_window_weights_shares_by_traffic_and_keeps_to_the_window(app):
|
||
"""#5214: a router is graded on its last 15 minutes. An idle scrape's
|
||
100% 4xx counts for its trickle of traffic, not as a third of the
|
||
average; a row older than the window is not read."""
|
||
from plugins.traefik.models import TraefikMetric
|
||
from plugins.traefik.routes import recent_window
|
||
|
||
now = datetime(2097, 3, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=uuid.uuid4().int % 10_000)
|
||
tag = uuid.uuid4().hex[:8]
|
||
busy, empty = f"busy-{tag}@docker", f"empty-{tag}@docker"
|
||
|
||
def row(name, ago, rate, e4, p95=10.0):
|
||
return TraefikMetric(router_name=name, scraped_at=now - timedelta(minutes=ago), request_rate=rate,
|
||
error_rate_4xx_pct=e4, error_rate_5xx_pct=0.0, latency_p50_ms=1.0,
|
||
latency_p95_ms=p95, latency_p99_ms=3.0, response_bytes_rate=0.0)
|
||
|
||
async def go():
|
||
async with app.db_sessionmaker() as s:
|
||
async with s.begin():
|
||
s.add_all([row(busy, 1, 9.9, 0.0, 10.0), row(busy, 2, 0.1, 100.0, 30.0),
|
||
row(busy, 30, 50.0, 100.0), # outside the window
|
||
row(empty, 30, 1.0, 100.0)])
|
||
async with s.begin():
|
||
return await recent_window(s, [busy, empty], now)
|
||
|
||
got = asyncio.run(go())
|
||
assert set(got) == {busy} # nothing in empty's window
|
||
assert got[busy].error_rate_4xx_pct == pytest.approx(1.0) # 0.1 × 100 over 10 req/s
|
||
assert got[busy].latency_p95_ms == pytest.approx(20.0)
|