diff --git a/docs/plugins/traefik.md b/docs/plugins/traefik.md index 55cda69..0295be6 100644 --- a/docs/plugins/traefik.md +++ b/docs/plugins/traefik.md @@ -83,7 +83,7 @@ The full Traefik page at `/plugins/traefik/` reads top to bottom, for the time r 1. **The verdict.** It names the routers that need a look, says why, and says when the problem started ("api, 3.0% of responses are 5xx, started 2h ago"). When a router has been a problem for the whole range, it says so instead. Otherwise it reads "All N routers healthy". 2. **The request chart.** Requests per second across all routers and the share answered 5xx, the same chart as the dashboard widget. -3. **A card for each router that needs a look:** why, since when, and its request rate, bandwidth, p95/p99 latency and 4xx/5xx shares, each with its history. +3. **A card for each router that needs a look:** why, since when, the 4xx and 5xx status codes it answered in the last 15 minutes, most first ("404 ×1,812, 502 ×40"), and its request rate, bandwidth, p95/p99 latency and 4xx/5xx shares, each with its history. The code is what separates scanners probing for pages (404), a login wall (401/403) and an app that isn't answering (502/503/504). Steward records codes from the version that added them on; until then a card says they aren't recorded yet. 4. **Traefik's own figures:** when its config last reloaded, open connections, memory. 5. **The healthy routers,** folded into one table, then the routers Traefik no longer reports, then TLS certificates by soonest expiry, then Traffic Origin when the access log is on. diff --git a/plugins/traefik/__init__.py b/plugins/traefik/__init__.py index e98bc3a..6ab0550 100644 --- a/plugins/traefik/__init__.py +++ b/plugins/traefik/__init__.py @@ -11,10 +11,13 @@ _app: "Quart | None" = None def setup(app: "Quart") -> None: global _app _app = app - from .models import TraefikMetric, TraefikCert, TraefikGlobalStat, TraefikAccessSummary # noqa: F401 + from .models import ( # noqa: F401 + TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode, + ) from steward.core.cleanup import prune_by_age # Samples older than data.retention_days are pruned by the cleanup (#5155). prune_by_age(TraefikMetric.scraped_at) + prune_by_age(TraefikStatusCode.scraped_at) prune_by_age(TraefikGlobalStat.scraped_at) prune_by_age(TraefikAccessSummary.period_end) diff --git a/plugins/traefik/migrations/versions/traefik_006_status_codes.py b/plugins/traefik/migrations/versions/traefik_006_status_codes.py new file mode 100644 index 0000000..0715c7e --- /dev/null +++ b/plugins/traefik/migrations/versions/traefik_006_status_codes.py @@ -0,0 +1,40 @@ +"""Each router's 4xx and 5xx responses by status code + +The scraper folded traefik_service_requests_total{code} into a 4xx share and +a 5xx share and dropped the code, so a problem couldn't say whether it was +scanners probing (404), a login wall (401/403) or the app down (502/503/504) +(#5711). One row per router, scrape and code that occurred; the page reads a +router's codes over a time window, answered from the index alone. + +Revision ID: traefik_006_status_codes +Revises: traefik_005_reload_at +Create Date: 2026-10-14 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "traefik_006_status_codes" +down_revision: Union[str, None] = "traefik_005_reload_at" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "traefik_status_codes", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("router_name", sa.String(255), nullable=False), + sa.Column("scraped_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("code", sa.String(8), nullable=False), + sa.Column("count", sa.Integer, nullable=False), + ) + op.create_index("ix_traefik_status_codes_router_scraped", "traefik_status_codes", + ["router_name", "scraped_at"], postgresql_include=["code", "count"]) + op.create_index("ix_traefik_status_codes_scraped", "traefik_status_codes", ["scraped_at"]) + + +def downgrade() -> None: + op.drop_index("ix_traefik_status_codes_scraped", "traefik_status_codes") + op.drop_index("ix_traefik_status_codes_router_scraped", "traefik_status_codes") + op.drop_table("traefik_status_codes") diff --git a/plugins/traefik/models.py b/plugins/traefik/models.py index bb1cb4c..1e70cca 100644 --- a/plugins/traefik/models.py +++ b/plugins/traefik/models.py @@ -67,3 +67,17 @@ class TraefikGlobalStat(Base): # traefik_config_last_reload_success gauge is that moment, not a 0/1 flag. config_last_reload_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) process_memory_bytes: Mapped[float | None] = mapped_column(Float, nullable=True) + + +class TraefikStatusCode(Base): + """How many responses a router answered with one 4xx or 5xx code in one + scrape interval (#5711): what lets a problem say whether it is scanners + probing (404), a login wall (401/403) or the app down (502/503/504). + Only codes that occurred are stored.""" + __tablename__ = "traefik_status_codes" + + id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id) + router_name: Mapped[str] = mapped_column(String(255), nullable=False) + scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + code: Mapped[str] = mapped_column(String(8), nullable=False) + count: Mapped[int] = mapped_column(Integer, nullable=False) diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py index 195c2c2..434d57b 100644 --- a/plugins/traefik/routes.py +++ b/plugins/traefik/routes.py @@ -13,7 +13,7 @@ from steward.models.users import UserRole from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, subsample, time_bucket from steward.core.formatting import cert_left, status_class from steward.models.metrics import PluginMetricLatest -from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric +from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode from .scheduler import router_domains from .scraper import cert_names, covers @@ -110,6 +110,25 @@ async def recent_window(db, names: list[str], now: datetime) -> dict[str, Simple return out +async def status_codes(db, names: list[str], since: datetime) -> dict[str, list[tuple[str, int]]]: + """{router: [(code, responses), …]} for the 4xx and 5xx codes each router + answered since `since`, most first (#5711). A router with none is absent.""" + if not names: + return {} + total = func.sum(TraefikStatusCode.count) + rows = (await db.execute( + select(TraefikStatusCode.router_name, TraefikStatusCode.code, total.label("n")) + .where(TraefikStatusCode.router_name.in_(names)) + .where(TraefikStatusCode.scraped_at >= since) + .group_by(TraefikStatusCode.router_name, TraefikStatusCode.code) + .order_by(TraefikStatusCode.router_name, total.desc(), TraefikStatusCode.code) + )).all() + out: dict[str, list[tuple[str, int]]] = {} + for r in rows: + out.setdefault(r.router_name, []).append((r.code, int(r.n))) + return out + + def grade_routers(routers: list[dict], window: dict, thresholds: dict) -> None: """Set each router's state and why from its recent window, or from its latest scrape when the window holds nothing.""" @@ -284,6 +303,15 @@ async def rows(): r["sparkline_4xx"] = _sparkline([p.error_rate_4xx_pct for p in points]) r["sparkline_5xx"] = _sparkline([p.error_rate_5xx_pct for p in points]) phases.mark("sparklines") + # The codes behind each problem, over the window it is graded on. + codes = await status_codes(db, [r["name"] for r in current if r["state"] != "ok"], now - STATE_WINDOW) + for r in current: + r["codes"] = codes.get(r["name"], []) + # Codes are kept from #5711 on: until the first of them is a window + # old, a router with none listed may simply predate the recording. + first_code = await db.scalar(select(func.min(TraefikStatusCode.scraped_at))) + codes_known = first_code is not None and first_code <= now - STATE_WINDOW + phases.mark("codes") # Latest global stat (always most recent, not range-filtered) result = await db.execute( @@ -329,6 +357,7 @@ async def rows(): healthy=healthy, gone=gone, current_range=range_key, + codes_known=codes_known, error_4xx=error_4xx, error_5xx=error_5xx, global_stat=global_stat, diff --git a/plugins/traefik/scheduler.py b/plugins/traefik/scheduler.py index 50f8a47..519f5e1 100644 --- a/plugins/traefik/scheduler.py +++ b/plugins/traefik/scheduler.py @@ -51,11 +51,12 @@ async def _do_scrape(app: "Quart") -> None: global _prev_metrics, _prev_time, _router_domains from datetime import datetime, timezone - from .models import TraefikCert, TraefikGlobalStat, TraefikMetric + from .models import TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode from .scraper import ( api_base, compute_global_stats, compute_router_metrics, + compute_status_codes, extract_certs, fetch_metrics, fetch_router_domains, @@ -81,6 +82,7 @@ async def _do_scrape(app: "Quart") -> None: elapsed = now - _prev_time if _prev_time else 60.0 router_metrics = compute_router_metrics(current, _prev_metrics or None, elapsed) global_stats = compute_global_stats(current, _prev_metrics or None) + status_codes = compute_status_codes(current, _prev_metrics or None) cert_list = extract_certs(current) _prev_metrics = current @@ -125,6 +127,12 @@ async def _do_scrape(app: "Quart") -> None: recorded_at=scraped_at, ) + # ── 4xx and 5xx responses by code (#5711) ───────────────────────── + for router_name, codes in status_codes.items(): + for code, count in codes.items(): + session.add(TraefikStatusCode(router_name=router_name, scraped_at=scraped_at, + code=code, count=count)) + # ── Global stats ────────────────────────────────────────────────── session.add(TraefikGlobalStat(scraped_at=scraped_at, **global_stats)) diff --git a/plugins/traefik/scraper.py b/plugins/traefik/scraper.py index 7af2538..026d7be 100644 --- a/plugins/traefik/scraper.py +++ b/plugins/traefik/scraper.py @@ -43,6 +43,33 @@ def parse_prometheus(text: str) -> ParsedMetrics: # Per-router metric computation # ────────────────────────────────────────────────────────────────────────────── +def _rise(prev: dict[frozenset, float] | None, labels: frozenset, value: float) -> float: + """How far a counter rose since the previous scrape. On the first scrape + (`prev` None) nothing can be compared, so 0. A series the previous scrape + didn't have started from 0 since then, so all of it counts: Traefik makes + a router's first 502 series when the first 502 happens (#5711).""" + if prev is None: + return 0.0 + return max(0.0, value - prev.get(labels, 0.0)) + + +def compute_status_codes(current: ParsedMetrics, previous: ParsedMetrics | None) -> dict[str, dict[str, int]]: + """{service: {code: responses since the previous scrape}} for each 4xx and + 5xx code that occurred, from traefik_service_requests_total{code} (#5711). + Codes that didn't occur are left out; the first scrape has none.""" + prev = None if previous is None else previous.get("traefik_service_requests_total", {}) + out: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int)) + for labels, value in current.get("traefik_service_requests_total", {}).items(): + label_dict = dict(labels) + service, code = label_dict.get("service", ""), label_dict.get("code", "") + if not service or code[:1] not in ("4", "5"): + continue + count = round(_rise(prev, labels, value)) + if count: + out[service][code] += count + return {service: dict(codes) for service, codes in out.items()} + + def compute_router_metrics( current: ParsedMetrics, previous: ParsedMetrics | None, @@ -63,12 +90,13 @@ def compute_router_metrics( "response_bytes_rate": float, # bytes/sec }} """ + first = not previous prev = previous or {} elapsed = max(elapsed_seconds, 1.0) # ── Request counters ────────────────────────────────────────────────────── req_samples = current.get("traefik_service_requests_total", {}) - prev_req = prev.get("traefik_service_requests_total", {}) + prev_req = None if first else prev.get("traefik_service_requests_total", {}) service_total: dict[str, float] = defaultdict(float) service_4xx: dict[str, float] = defaultdict(float) @@ -80,8 +108,7 @@ def compute_router_metrics( if not service: continue code = label_dict.get("code", "") - prev_value = prev_req.get(labels, value) # no delta on first scrape - delta = max(0.0, value - prev_value) + delta = _rise(prev_req, labels, value) service_total[service] += delta if code.startswith("4"): service_4xx[service] += delta @@ -90,15 +117,14 @@ def compute_router_metrics( # ── Response bytes counters ─────────────────────────────────────────────── bytes_samples = current.get("traefik_service_responses_bytes_total", {}) - prev_bytes = prev.get("traefik_service_responses_bytes_total", {}) + prev_bytes = None if first else prev.get("traefik_service_responses_bytes_total", {}) service_bytes: dict[str, float] = defaultdict(float) for labels, value in bytes_samples.items(): service = dict(labels).get("service", "") if not service: continue - prev_value = prev_bytes.get(labels, value) - service_bytes[service] += max(0.0, value - prev_value) + service_bytes[service] += _rise(prev_bytes, labels, value) # ── Histogram buckets ───────────────────────────────────────────────────── hist_buckets = current.get("traefik_service_request_duration_seconds_bucket", {}) diff --git a/plugins/traefik/templates/traefik/rows.html b/plugins/traefik/templates/traefik/rows.html index 1931d5d..e13fb89 100644 --- a/plugins/traefik/templates/traefik/rows.html +++ b/plugins/traefik/templates/traefik/rows.html @@ -42,6 +42,15 @@

{{ r.why }} {%- if r.since_before %}, all of the last {{ current_range }}{% elif r.since %}, started {{ r.since|ago }}{% endif %}

+ {% if r.codes %} +

Last 15 min: + {% for code, n in r.codes[:5] %}{{ code }} ×{{ "{:,}".format(n) }}{{ ", " if not loop.last }}{% endfor %} + {%- if r.codes|length > 5 %}, {{ r.codes|length - 5 }} more code{{ "s" if r.codes|length - 5 != 1 }}{% endif %}

+ {% elif codes_known %} +

No 4xx or 5xx responses in the last 15 min.

+ {% else %} +

Status codes not recorded yet: Steward keeps them from this version on.

+ {% endif %} diff --git a/steward/backup/contents.py b/steward/backup/contents.py index 69c4af2..a17ccdd 100644 --- a/steward/backup/contents.py +++ b/steward/backup/contents.py @@ -38,6 +38,7 @@ EXCLUDED: frozenset[str] = frozenset({ "docker_images", "docker_logs", "docker_metrics", "docker_metrics_hourly", "docker_swarm_nodes", "docker_swarm_services", "traefik_access_summaries", "traefik_certs", "traefik_global_stats", "traefik_metrics", + "traefik_status_codes", "unifi_alarms", "unifi_client_snapshots", "unifi_devices", "unifi_dpi_snapshots", "unifi_events", "unifi_fw_rules", "unifi_networks", "unifi_port_forwards", "unifi_speedtest_results", "unifi_wan_stats", "unifi_wlan_stats", diff --git a/tests/integration/test_traefik_latest.py b/tests/integration/test_traefik_latest.py index f69cd1b..0e6bdde 100644 --- a/tests/integration/test_traefik_latest.py +++ b/tests/integration/test_traefik_latest.py @@ -152,3 +152,28 @@ def test_recent_window_weights_shares_by_traffic_and_keeps_to_the_window(app): assert set(got) == {busy} # nothing in empty's window assert got[busy].error_rate_4xx_pct == pytest.approx(1.0) # 0.1 × 100 over 10 req/s assert got[busy].latency_p95_ms == pytest.approx(20.0) + + +def test_status_codes_sum_each_routers_codes_in_the_window_most_first(app): + """#5711: the codes behind a router's problem, summed over the window it + is graded on; an older row and a router not asked for are not read.""" + from plugins.traefik.models import TraefikStatusCode + from plugins.traefik.routes import status_codes + + at = datetime(2097, 6, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=5 * (uuid.uuid4().int % 2_000)) + tag = uuid.uuid4().hex[:8] + a, b, other = (f"{x}-{tag}@docker" for x in ("a", "b", "other")) + + def row(name, minute, code, count): + return TraefikStatusCode(router_name=name, scraped_at=at + timedelta(minutes=minute), code=code, count=count) + + async def go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add_all([row(a, 1, "404", 30), row(a, 2, "404", 12), row(a, 2, "502", 50), + row(a, -20, "500", 999), row(b, 3, "401", 4), row(other, 3, "404", 7)]) + async with s.begin(): + return await status_codes(s, [a, b], at) + + got = asyncio.run(go()) + assert got == {a: [("502", 50), ("404", 42)], b: [("401", 4)]} diff --git a/tests/plugins/traefik/test_status_codes.py b/tests/plugins/traefik/test_status_codes.py new file mode 100644 index 0000000..7f6407c --- /dev/null +++ b/tests/plugins/traefik/test_status_codes.py @@ -0,0 +1,78 @@ +"""Each router's 4xx and 5xx responses by code (#5711): counted from +traefik_service_requests_total{code} between scrapes, and shown on the +problem card so it can say whether it's scanners, a login wall or the app.""" +from datetime import datetime, timedelta, timezone + +from plugins.traefik.routes import router_state +from plugins.traefik.scraper import compute_router_metrics, compute_status_codes +from steward.core.settings import to_thresholds_cfg +from tests._templates import render_plugin_template +from tests.plugins.traefik.test_traefik_page import _latest + +REQ = "traefik_service_requests_total" + + +def _req(*series): + """{REQ: {labels: value}} from (service, code, method, value) tuples.""" + return {REQ: {frozenset({"service": s, "code": c, "method": m}.items()): v for s, c, m, v in series}} + + +def test_the_codes_since_the_last_scrape_are_counted_per_router(): + before = _req(("app@docker", "200", "GET", 1000), ("app@docker", "404", "GET", 100), + ("app@docker", "404", "POST", 10), ("api@docker", "401", "GET", 5)) + after = _req(("app@docker", "200", "GET", 1500), ("app@docker", "404", "GET", 160), + ("app@docker", "404", "POST", 12), ("api@docker", "401", "GET", 5), + ("app@docker", "502", "GET", 3)) # a series the last scrape didn't have + got = compute_status_codes(after, before) + # 2xx is left out; 404 sums over methods; a new 502 series counts whole; api's 401 didn't move. + assert got == {"app@docker": {"404": 62, "502": 3}} + + +def test_the_first_scrape_counts_nothing_and_a_reset_counts_nothing(): + series = _req(("app@docker", "500", "GET", 40)) + assert compute_status_codes(series, None) == {} + assert compute_status_codes(_req(("app@docker", "500", "GET", 2)), series) == {} # Traefik restarted + + +def test_a_router_first_502s_count_in_its_5xx_share(): + """A series new since the last scrape started from 0: before #5711 its + whole first burst was dropped, so a router's first 502s read as 0%.""" + before = _req(("app@docker", "200", "GET", 100)) + after = _req(("app@docker", "200", "GET", 190), ("app@docker", "502", "GET", 10)) + got = compute_router_metrics(after, before, 60.0)["app@docker"] + assert got["error_rate_5xx_pct"] == 10.0 and abs(got["request_rate"] - 100 / 60) < 1e-9 + + +TH = to_thresholds_cfg({}) + + +def _problem(**codes_ctx): + latest = _latest(e4=6.0) + state, why = router_state(latest, TH) + return {"name": "app", "latest": latest, "state": state, "why": why, **codes_ctx, + **{f"sparkline_{k}": "" for k in ("req", "bytes", "p95", "4xx", "5xx")}} + + +async def _card(app, router, codes_known=True): + async with app.test_request_context("/plugins/traefik/rows"): + page = await render_plugin_template( + app, "traefik", "traefik/rows.html", problems=[router], healthy=[], gone=[], global_stat=None, + certs=[], old_certs=[], range_key="24h", current_range="24h", error_4xx=1.0, error_5xx=0.1, + codes_known=codes_known) + page = " ".join(page.split()) + return page[page.index(">app"):] + + +async def test_the_card_lists_the_codes_most_first_before_its_figures(app): + codes = [("404", 1812), ("502", 40), ("401", 9), ("403", 3), ("429", 2), ("499", 1), ("500", 1)] + card = await _card(app, _problem(codes=codes)) + assert card.index('Last 15 min: 404 ×1,812, 502 ×40') \ + < card.index(" ×2, 2 more codes" in card and "499" not in card + + +async def test_a_card_without_codes_says_whether_they_were_recorded(app): + none = await _card(app, _problem(codes=[]), codes_known=True) + assert "No 4xx or 5xx responses in the last 15 min." in none + early = await _card(app, _problem(codes=[]), codes_known=False) + assert "Status codes not recorded yet" in early
Req/s{{ "%.2f"|format(r.latest.request_rate) }}{{ r.sparkline_req|safe }}
Bandwidth{{ r.latest.response_bytes_rate|rate }}{{ r.sparkline_bytes|safe }}