fix(traefik): bucket history with date_bin; integer-epoch division never grouped scrapes (#5256)
CI 8551 integration: "integer out of range" for a 2099 timestamp. The SQL in the log showed a second problem: `CAST(epoch AS INTEGER) / 60` compiled to `/ CAST(60 AS NUMERIC)`, i.e. decimal division. Every scrape got its own "bucket", so the request chart and the Traefik page's sparklines never averaged anything. The int4 cast would also overflow in 2038. Both now use date_bin from the epoch, as host_agent/metrics_query.py does. The integration test adds a second scrape 30 s later in the same minute and checks that the bucket averages the totals and weights the 5xx share over both scrapes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from quart import Blueprint, current_app, render_template, request
|
from quart import Blueprint, current_app, render_template, request
|
||||||
from sqlalchemy import Integer, cast, func, select, text
|
from sqlalchemy import func, select, text
|
||||||
import json
|
import json
|
||||||
|
|
||||||
from steward.auth.middleware import require_role
|
from steward.auth.middleware import require_role
|
||||||
@@ -186,9 +186,7 @@ async def rows():
|
|||||||
routers = sorted(newest)
|
routers = sorted(newest)
|
||||||
|
|
||||||
b_secs = bucket_seconds(since)
|
b_secs = bucket_seconds(since)
|
||||||
bucket_col = (
|
bucket_col = _time_bucket(TraefikMetric.scraped_at, b_secs)
|
||||||
cast(func.extract('epoch', TraefikMetric.scraped_at), Integer) / b_secs
|
|
||||||
).label("bucket")
|
|
||||||
|
|
||||||
router_data = []
|
router_data = []
|
||||||
for router in routers:
|
for router in routers:
|
||||||
@@ -375,6 +373,15 @@ async def widget_access_log():
|
|||||||
summary=summary, widget_id=widget_id)
|
summary=summary, widget_id=widget_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _time_bucket(column, seconds: int):
|
||||||
|
"""The start of the `seconds`-wide window a timestamp falls in, as
|
||||||
|
host_agent/metrics_query.py buckets. Integer epoch / int rendered as
|
||||||
|
numeric division (no two scrapes shared a bucket) and overflowed int4
|
||||||
|
after 2038 (#5256)."""
|
||||||
|
return func.date_bin(func.make_interval(0, 0, 0, 0, 0, 0, int(seconds)), column,
|
||||||
|
func.to_timestamp(0)).label("bucket")
|
||||||
|
|
||||||
|
|
||||||
async def request_history(db, since, bucket_seconds: int) -> list[dict]:
|
async def request_history(db, since, bucket_seconds: int) -> list[dict]:
|
||||||
"""[{"at", "req_rate", "err_pct"}] per time bucket since `since`: each
|
"""[{"at", "req_rate", "err_pct"}] per time bucket since `since`: each
|
||||||
scrape summed over routers (the total req/s), averaged over the bucket;
|
scrape summed over routers (the total req/s), averaged over the bucket;
|
||||||
@@ -390,7 +397,7 @@ async def request_history(db, since, bucket_seconds: int) -> list[dict]:
|
|||||||
.group_by(TraefikMetric.scraped_at)
|
.group_by(TraefikMetric.scraped_at)
|
||||||
.subquery()
|
.subquery()
|
||||||
)
|
)
|
||||||
bucket_col = (cast(func.extract("epoch", per_scrape.c.at), Integer) / bucket_seconds).label("bucket")
|
bucket_col = _time_bucket(per_scrape.c.at, bucket_seconds)
|
||||||
rows = (await db.execute(
|
rows = (await db.execute(
|
||||||
select(
|
select(
|
||||||
func.avg(per_scrape.c.req).label("req_rate"),
|
func.avg(per_scrape.c.req).label("req_rate"),
|
||||||
|
|||||||
@@ -48,24 +48,29 @@ def test_each_router_reads_as_its_newest_row(app):
|
|||||||
|
|
||||||
def test_the_request_chart_sums_routers_and_weights_the_5xx_share(app):
|
def test_the_request_chart_sums_routers_and_weights_the_5xx_share(app):
|
||||||
"""#5256: two routers in one scrape are 4 req/s in total, not an average
|
"""#5256: two routers in one scrape are 4 req/s in total, not an average
|
||||||
of 2; a 5xx share of 50% on the 3 req/s router is 37.5% of all traffic."""
|
of 2; a 5xx share of 50% on the 3 req/s router is 37.5% of that scrape's
|
||||||
|
traffic. A second scrape 30 s later shares the minute's bucket, so the
|
||||||
|
bucket averages the two totals and weights the share over both."""
|
||||||
from plugins.traefik.models import TraefikMetric
|
from plugins.traefik.models import TraefikMetric
|
||||||
from plugins.traefik.routes import request_history
|
from plugins.traefik.routes import request_history
|
||||||
|
|
||||||
# Far ahead of every other test's rows, so only these fall in the window.
|
# Far ahead of every other test's rows, so only these fall in the window.
|
||||||
at = datetime(2099, 1, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=uuid.uuid4().int % 10_000)
|
at = datetime(2099, 1, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=uuid.uuid4().int % 10_000)
|
||||||
|
|
||||||
def row(name, rate, e5):
|
def row(name, rate, e5, when=None):
|
||||||
return TraefikMetric(router_name=name, scraped_at=at, request_rate=rate, error_rate_4xx_pct=0.0,
|
return TraefikMetric(router_name=name, scraped_at=when or at, request_rate=rate, error_rate_4xx_pct=0.0,
|
||||||
error_rate_5xx_pct=e5, latency_p50_ms=1.0, latency_p95_ms=2.0,
|
error_rate_5xx_pct=e5, latency_p50_ms=1.0, latency_p95_ms=2.0,
|
||||||
latency_p99_ms=3.0, response_bytes_rate=0.0)
|
latency_p99_ms=3.0, response_bytes_rate=0.0)
|
||||||
|
|
||||||
async def go():
|
async def go():
|
||||||
async with app.db_sessionmaker() as s:
|
async with app.db_sessionmaker() as s:
|
||||||
async with s.begin():
|
async with s.begin():
|
||||||
s.add_all([row("quiet@docker", 1.0, 0.0), row("busy@docker", 3.0, 50.0)])
|
later = at + timedelta(seconds=30)
|
||||||
|
s.add_all([row("quiet@docker", 1.0, 0.0), row("busy@docker", 3.0, 50.0),
|
||||||
|
row("quiet@docker", 2.0, 0.0, later), row("busy@docker", 4.0, 0.0, later)])
|
||||||
async with s.begin():
|
async with s.begin():
|
||||||
return await request_history(s, at, 60)
|
return await request_history(s, at, 60)
|
||||||
|
|
||||||
got = asyncio.run(go())
|
got = asyncio.run(go())
|
||||||
assert got == [{"at": at, "req_rate": 4.0, "err_pct": 37.5}]
|
# One bucket: totals 4 and 6 average to 5; 150 weighted 5xx over 10 req/s is 15%.
|
||||||
|
assert got == [{"at": at, "req_rate": 5.0, "err_pct": 15.0}]
|
||||||
|
|||||||
Reference in New Issue
Block a user