Compare commits
2
Commits
31e93e96cf
...
7ce61a2099
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7ce61a2099 | ||
|
|
7a444da871 |
@@ -0,0 +1,41 @@
|
||||
"""Traefik's last good config reload is a moment; routers are graded on a window
|
||||
|
||||
- traefik_config_last_reload_success is the Unix time of the last successful
|
||||
reload. It was stored as a float and read as 1.0 = success, so the page said
|
||||
"Reload failed" on every scrape (#5355). The column becomes the moment; the
|
||||
old values were timestamps misread, so they go with it.
|
||||
- Routers are now graded on their last 15 minutes (routes.recent_window),
|
||||
which reads the 4xx share too: the covering router index carries it, so
|
||||
that read stays index-only.
|
||||
|
||||
Revision ID: traefik_005_reload_at
|
||||
Revises: traefik_004_covering_indexes
|
||||
Create Date: 2026-10-08
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "traefik_005_reload_at"
|
||||
down_revision: Union[str, None] = "traefik_004_covering_indexes"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
_CHARTED = ["request_rate", "latency_p95_ms", "error_rate_5xx_pct"]
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.drop_column("traefik_global_stats", "config_last_reload_success")
|
||||
op.add_column("traefik_global_stats",
|
||||
sa.Column("config_last_reload_at", sa.DateTime(timezone=True), nullable=True))
|
||||
op.drop_index("ix_traefik_metrics_router_scraped", "traefik_metrics")
|
||||
op.create_index("ix_traefik_metrics_router_scraped", "traefik_metrics", ["router_name", "scraped_at"],
|
||||
postgresql_include=_CHARTED + ["error_rate_4xx_pct"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_traefik_metrics_router_scraped", "traefik_metrics")
|
||||
op.create_index("ix_traefik_metrics_router_scraped", "traefik_metrics", ["router_name", "scraped_at"],
|
||||
postgresql_include=_CHARTED)
|
||||
op.drop_column("traefik_global_stats", "config_last_reload_at")
|
||||
op.add_column("traefik_global_stats", sa.Column("config_last_reload_success", sa.Float, nullable=True))
|
||||
@@ -63,5 +63,7 @@ class TraefikGlobalStat(Base):
|
||||
)
|
||||
open_conns_total: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
config_reloads_total: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
config_last_reload_success: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
# When Traefik last loaded its configuration successfully: its
|
||||
# traefik_config_last_reload_success gauge is that moment, not a 0/1 flag.
|
||||
config_last_reload_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
process_memory_bytes: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
|
||||
+68
-14
@@ -2,6 +2,7 @@
|
||||
from __future__ import annotations
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from quart import Blueprint, current_app, render_template, request
|
||||
from sqlalchemy import func, select, text
|
||||
import json
|
||||
@@ -46,20 +47,71 @@ ERROR_4XX_PCT = 1.0
|
||||
_GRADE_ORDER = {"crit": 0, "warn": 1, "ok": 2}
|
||||
|
||||
|
||||
def router_state(latest, thresholds: dict) -> tuple[str, str | None]:
|
||||
"""(grade, why) for a router's latest scrape: crit while it serves 5xx,
|
||||
warn on a p95 past the latency critical cutoff (Settings → Thresholds)
|
||||
or a high 4xx share, otherwise ok with nothing to say."""
|
||||
if (latest.error_rate_5xx_pct or 0) > ERROR_5XX_PCT:
|
||||
return "crit", f"{latest.error_rate_5xx_pct:.1f}% of responses are 5xx"
|
||||
p95 = latest.latency_p95_ms
|
||||
def router_state(reading, thresholds: dict, over: str = "") -> tuple[str, str | None]:
|
||||
"""(grade, why) for a router's reading (recent_window's, or one scrape's):
|
||||
crit while it serves 5xx, warn on a p95 past the latency critical cutoff
|
||||
(Settings → Thresholds) or a high 4xx share, otherwise ok with nothing to
|
||||
say. `over` names the span the reading covers, for the why."""
|
||||
if (reading.error_rate_5xx_pct or 0) > ERROR_5XX_PCT:
|
||||
return "crit", f"{reading.error_rate_5xx_pct:.1f}% of responses are 5xx{over}"
|
||||
p95 = reading.latency_p95_ms
|
||||
if p95 is not None and status_class(p95, "latency", thresholds) == "crit":
|
||||
return "warn", f"p95 latency is {p95:.0f} ms"
|
||||
if (latest.error_rate_4xx_pct or 0) > ERROR_4XX_PCT:
|
||||
return "warn", f"{latest.error_rate_4xx_pct:.1f}% of responses are 4xx"
|
||||
return "warn", f"p95 latency is {p95:.0f} ms{over}"
|
||||
if (reading.error_rate_4xx_pct or 0) > ERROR_4XX_PCT:
|
||||
return "warn", f"{reading.error_rate_4xx_pct:.1f}% of responses are 4xx{over}"
|
||||
return "ok", None
|
||||
|
||||
|
||||
# A router is graded on its last STATE_WINDOW of traffic, not on one scrape:
|
||||
# a share near a cutoff crossed it on one scrape and fell back on the next,
|
||||
# so problem cards came and went every poll (#5214). Over the window a real
|
||||
# problem stays listed until it has been clean a while, and a one-scrape
|
||||
# blip is diluted rather than shown.
|
||||
STATE_WINDOW = timedelta(minutes=15)
|
||||
STATE_WINDOW_WORDS = " over the last 15 min"
|
||||
|
||||
|
||||
async def recent_window(db, names: list[str], now: datetime) -> dict[str, SimpleNamespace]:
|
||||
"""{router: one reading of its last STATE_WINDOW}: the 4xx and 5xx shares
|
||||
weighted by each scrape's traffic (an idle scrape's 100% counts for
|
||||
little), p95 averaged. A router with no rows in the window is absent."""
|
||||
if not names:
|
||||
return {}
|
||||
rate = TraefikMetric.request_rate
|
||||
result = await db.execute(
|
||||
select(
|
||||
TraefikMetric.router_name,
|
||||
func.sum(rate).label("req"),
|
||||
func.sum(rate * TraefikMetric.error_rate_4xx_pct).label("e4w"),
|
||||
func.sum(rate * TraefikMetric.error_rate_5xx_pct).label("e5w"),
|
||||
func.avg(TraefikMetric.error_rate_4xx_pct).label("e4"),
|
||||
func.avg(TraefikMetric.error_rate_5xx_pct).label("e5"),
|
||||
func.avg(TraefikMetric.latency_p95_ms).label("p95"),
|
||||
)
|
||||
.where(TraefikMetric.router_name.in_(names))
|
||||
.where(TraefikMetric.scraped_at >= now - STATE_WINDOW)
|
||||
.group_by(TraefikMetric.router_name)
|
||||
)
|
||||
out = {}
|
||||
for r in result.all():
|
||||
req = r.req or 0.0
|
||||
out[r.router_name] = SimpleNamespace(
|
||||
error_rate_4xx_pct=r.e4w / req if req > 0 else r.e4,
|
||||
error_rate_5xx_pct=r.e5w / req if req > 0 else r.e5,
|
||||
latency_p95_ms=r.p95,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def grade_routers(routers: list[dict], window: dict, thresholds: dict) -> None:
|
||||
"""Set each router's state and why from its recent window, or from its
|
||||
latest scrape when the window holds nothing."""
|
||||
for r in routers:
|
||||
reading = window.get(r["name"])
|
||||
r["state"], r["why"] = (router_state(reading, thresholds, STATE_WINDOW_WORDS) if reading
|
||||
else router_state(r["latest"], thresholds))
|
||||
|
||||
|
||||
# A router missing from this many scrapes in a row is no longer reported:
|
||||
# Traefik dropped it (a removed service), so its last state is history.
|
||||
GONE_AFTER_SCRAPES = 3
|
||||
@@ -188,10 +240,11 @@ async def rows():
|
||||
newest = await latest_per_router(db)
|
||||
current, gone = gone_routers([{"name": m.router_name, "latest": m} for m in newest], scrape)
|
||||
|
||||
history = await router_history(db, [r["name"] for r in current], since, bucket_seconds(since))
|
||||
names = [r["name"] for r in current]
|
||||
history = await router_history(db, names, since, bucket_seconds(since))
|
||||
grade_routers(current, await recent_window(db, names, now), thresholds)
|
||||
for r in current:
|
||||
points = history[r["name"]]
|
||||
r["state"], r["why"] = router_state(r["latest"], thresholds)
|
||||
r["sparkline_req"] = _sparkline([p.request_rate for p in points])
|
||||
r["sparkline_p95"] = _sparkline([p.latency_p95_ms for p in points])
|
||||
r["sparkline_5xx"] = _sparkline([p.error_rate_5xx_pct for p in points])
|
||||
@@ -259,14 +312,15 @@ async def widget():
|
||||
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
latest = await latest_per_router(db)
|
||||
current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
|
||||
window = await recent_window(db, [r["name"] for r in current], now)
|
||||
certs = (await db.execute(select(TraefikCert))).scalars().all()
|
||||
|
||||
current, _gone = gone_routers([{"name": m.router_name, "latest": m} for m in latest], scrape)
|
||||
served, _old = current_certs(certs, scrape)
|
||||
domains = router_domains()
|
||||
by_router = certs_by_router([r["name"] for r in current], domains or {}, served, now)
|
||||
grade_routers(current, window, thresholds)
|
||||
for r in current:
|
||||
r["state"], r["why"] = router_state(r["latest"], thresholds)
|
||||
r["cert"] = by_router.get(r["name"])
|
||||
if r["cert"]:
|
||||
grade = status_class(r["cert"]["days"], "tls_days", thresholds)
|
||||
|
||||
@@ -152,21 +152,21 @@ def compute_router_metrics(
|
||||
def compute_global_stats(
|
||||
current: ParsedMetrics,
|
||||
previous: ParsedMetrics | None,
|
||||
) -> dict[str, float | None]:
|
||||
) -> dict[str, float | datetime | None]:
|
||||
"""Extract process-level and config health metrics from a scrape.
|
||||
|
||||
Returns:
|
||||
{
|
||||
"open_conns_total": float | None,
|
||||
"config_reloads_total": float | None,
|
||||
"config_last_reload_success": float | None, # 1.0 = success, 0.0 = failure
|
||||
"config_last_reload_at": datetime | None, # last successful reload
|
||||
"process_memory_bytes": float | None,
|
||||
}
|
||||
"""
|
||||
stats: dict[str, float | None] = {
|
||||
stats: dict[str, float | datetime | None] = {
|
||||
"open_conns_total": None,
|
||||
"config_reloads_total": None,
|
||||
"config_last_reload_success": None,
|
||||
"config_last_reload_at": None,
|
||||
"process_memory_bytes": None,
|
||||
}
|
||||
|
||||
@@ -180,11 +180,15 @@ def compute_global_stats(
|
||||
if reloads:
|
||||
stats["config_reloads_total"] = sum(reloads.values())
|
||||
|
||||
# Last reload success gauge
|
||||
# The moment of the last successful reload, as a Unix timestamp: Traefik
|
||||
# names it "last_reload_success", but it is a time, not a 0/1 flag. Read
|
||||
# as a flag, every scrape said "Reload failed" (#5355).
|
||||
last_success = current.get("traefik_config_last_reload_success", {})
|
||||
if last_success:
|
||||
# Usually a single sample with no labels
|
||||
stats["config_last_reload_success"] = next(iter(last_success.values()))
|
||||
ts = next(iter(last_success.values()))
|
||||
if ts > 0:
|
||||
stats["config_last_reload_at"] = datetime.fromtimestamp(ts, timezone.utc)
|
||||
|
||||
# Process RSS memory
|
||||
mem = current.get("process_resident_memory_bytes", {})
|
||||
|
||||
@@ -8,9 +8,8 @@
|
||||
{{ stat_tile("Needs attention", problems|length, state="crit" if problems else "inactive") }}
|
||||
{{ stat_tile("Healthy routers", healthy|length, state="up" if healthy else "inactive") }}
|
||||
{% if global_stat %}
|
||||
{% if global_stat.config_last_reload_success is not none %}
|
||||
{{ stat_tile("Config", "OK" if global_stat.config_last_reload_success == 1.0 else "Reload failed",
|
||||
state="up" if global_stat.config_last_reload_success == 1.0 else "crit",
|
||||
{% if global_stat.config_last_reload_at %}
|
||||
{{ stat_tile("Config reloaded", global_stat.config_last_reload_at|ago,
|
||||
hint=(global_stat.config_reloads_total|int ~ " reloads") if global_stat.config_reloads_total is not none else none) }}
|
||||
{% endif %}
|
||||
{% if global_stat.open_conns_total is not none %}{{ stat_tile("Open connections", global_stat.open_conns_total|int) }}{% endif %}
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
from __future__ import annotations
|
||||
import json
|
||||
from quart import Blueprint, render_template, request, current_app
|
||||
from quart import Blueprint, render_template, request, current_app, session
|
||||
from sqlalchemy import select
|
||||
from steward.ansible.inventory_gen import scope_labels
|
||||
from steward.audit.service import AREAS, action_text, detail_facts, entity_labels
|
||||
from steward.audit.service import AREAS, action_text, detail_facts, entity_labels, mark_day_groups
|
||||
from steward.auth.middleware import require_role
|
||||
from steward.core import formatting as fmt
|
||||
from steward.core.paging import keyset, next_page
|
||||
from steward.models.audit import AuditEvent
|
||||
from steward.models.users import UserRole
|
||||
@@ -46,7 +47,9 @@ async def list_events():
|
||||
item["action"] = action_text(e.action)
|
||||
item["facts"] = detail_facts(item["detail"], scopes)
|
||||
|
||||
ctx = {"events": parsed, "after": after, "action_filter": action_filter, "areas": AREAS}
|
||||
last_group = mark_day_groups(parsed, fmt.zone(session.get("tz")), request.args.get("group", ""))
|
||||
ctx = {"events": parsed, "after": after, "action_filter": action_filter, "areas": AREAS,
|
||||
"last_group": last_group}
|
||||
if request.args.get("before"):
|
||||
return await render_template("audit/_rows.html", **ctx)
|
||||
return await render_template("audit/list.html", **ctx)
|
||||
|
||||
@@ -9,6 +9,7 @@ from __future__ import annotations
|
||||
from sqlalchemy import select
|
||||
|
||||
from steward.ansible.inventory_gen import run_title, scope_labels
|
||||
from steward.core.formatting import day_group
|
||||
from steward.models.alerts import AlertRule, MaintenanceWindow
|
||||
from steward.models.ansible import AnsibleRun
|
||||
from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget
|
||||
@@ -207,3 +208,15 @@ def detail_facts(detail: dict, scopes: dict[str, str]) -> list[dict]:
|
||||
text = _plain(value)
|
||||
facts.append({"label": label, "text": text, "url": url})
|
||||
return facts
|
||||
|
||||
|
||||
def mark_day_groups(items: list[dict], tz, group: str = "", now=None) -> str:
|
||||
"""Give each newest-first entry its recency group (formatting.day_group,
|
||||
in the reader's zone) and whether it opens one, so the page draws a
|
||||
divider there. `group` is the one the previous page ended in: a group
|
||||
split across pages is opened once. Returns the group this page ends in."""
|
||||
for item in items:
|
||||
item["group"] = day_group(item["event"].timestamp, tz, now)
|
||||
item["opens_group"] = item["group"] != group
|
||||
group = item["group"]
|
||||
return group
|
||||
|
||||
+23
-12
@@ -9,7 +9,7 @@ here. A moment is written in the reader's own zone, on a 24-hour clock
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone, tzinfo
|
||||
from datetime import datetime, timezone, tzinfo
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError, available_timezones
|
||||
@@ -120,29 +120,40 @@ def ago(when: datetime | None, now: datetime | None = None, tz: tzinfo | None =
|
||||
|
||||
|
||||
def when(moment: datetime | None, tz: tzinfo | None = None, now: datetime | None = None) -> Markup:
|
||||
"""The exact moment in the reader's zone on a 24-hour clock, as short as
|
||||
it can be read: 'Today 06:58', 'Yesterday 23:41', 'Tomorrow 02:00',
|
||||
'Tue 6 Oct 14:12' this year, '6 Oct 2025 14:12' before it."""
|
||||
"""The exact moment in the reader's zone on a 24-hour clock, always with
|
||||
its date: 'Thu 8 Oct 06:58' this year, '6 Oct 2025 14:12' before it. A
|
||||
list that groups by recency says Today or Yesterday in a divider
|
||||
(day_group), never in place of the date (#5214)."""
|
||||
if moment is None:
|
||||
return Markup(NONE)
|
||||
tz = tz or timezone.utc
|
||||
moment = _utc(moment)
|
||||
local = moment.astimezone(tz)
|
||||
today = (now or datetime.now(timezone.utc)).astimezone(tz).date()
|
||||
this_year = (now or datetime.now(timezone.utc)).astimezone(tz).year
|
||||
day, hm = local.date(), f"{local:%H:%M}"
|
||||
if day == today:
|
||||
text = f"Today {hm}"
|
||||
elif day == today - timedelta(days=1):
|
||||
text = f"Yesterday {hm}"
|
||||
elif day == today + timedelta(days=1):
|
||||
text = f"Tomorrow {hm}"
|
||||
elif day.year == today.year:
|
||||
if day.year == this_year:
|
||||
text = f"{local:%a} {day.day} {local:%b} {hm}"
|
||||
else:
|
||||
text = f"{day.day} {local:%b} {day.year} {hm}"
|
||||
return _time(moment, local, text)
|
||||
|
||||
|
||||
DAY_GROUPS = ("Today", "Yesterday", "Last 7 days", "Older")
|
||||
|
||||
|
||||
def day_group(moment: datetime, tz: tzinfo | None = None, now: datetime | None = None) -> str:
|
||||
"""Which of DAY_GROUPS a past moment falls in, by calendar day in the
|
||||
reader's zone: the dividers of a newest-first log."""
|
||||
tz = tz or timezone.utc
|
||||
day = _utc(moment).astimezone(tz).date()
|
||||
back = ((now or datetime.now(timezone.utc)).astimezone(tz).date() - day).days
|
||||
if back <= 0:
|
||||
return DAY_GROUPS[0]
|
||||
if back == 1:
|
||||
return DAY_GROUPS[1]
|
||||
return DAY_GROUPS[2] if back < 7 else DAY_GROUPS[3]
|
||||
|
||||
|
||||
def status_class(value: Any, kind: str, thresholds: dict) -> str:
|
||||
"""Grade a reading against Settings → Thresholds (to_thresholds_cfg):
|
||||
'ok', 'warn' or 'crit', or 'unknown' when there is no reading.
|
||||
|
||||
@@ -227,6 +227,9 @@ strong, b, th { font-weight: 500; }
|
||||
.table td.nowrap { white-space: nowrap; }
|
||||
/* An audit entry's details, one fact a line (milestone 490). */
|
||||
.audit-facts { font-size: 0.8rem; color: var(--text-muted); }
|
||||
/* A divider row naming the group the rows under it fall in (Today, Older). */
|
||||
.table tr.table-divider th { padding: 1rem 1rem 0.4rem; background: var(--bg-card); border-bottom: 1px solid var(--border-mid); }
|
||||
.table tbody tr.table-divider:hover th { background: var(--bg-card); }
|
||||
.cell-note { font-size: 0.78rem; color: var(--text-muted); margin-top: 0.2rem; }
|
||||
/* An alert episode's own transitions, folded under it (#5316). */
|
||||
.episode-steps { margin-top: 0.3rem; font-size: 0.8rem; }
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
{# audit/_rows.html — one page of the audit log's rows, then the sentinel that
|
||||
loads the next page as the reader nears it (core.paging, pref 172). #}
|
||||
loads the next page as the reader nears it (core.paging, pref 172). A
|
||||
divider row opens each recency group: Today, Yesterday, Last 7 days, Older. #}
|
||||
{% from "_macros.html" import scroll_more, scroll_end %}
|
||||
{% for item in events %}
|
||||
{% set e = item.event %}
|
||||
{% if item.opens_group %}<tr class="table-divider"><th colspan="5" scope="rowgroup">{{ item.group }}</th></tr>{% endif %}
|
||||
<tr>
|
||||
<td class="nowrap">{{ e.timestamp|when }}</td>
|
||||
<td class="nowrap">{{ e.username }}</td>
|
||||
@@ -22,7 +24,8 @@
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% if after %}
|
||||
{{ scroll_more("/audit/?before=" ~ (after|urlencode) ~ (("&action=" ~ (action_filter|urlencode)) if action_filter else ""), cols=5) }}
|
||||
{{ scroll_more("/audit/?before=" ~ (after|urlencode) ~ (("&action=" ~ (action_filter|urlencode)) if action_filter else "")
|
||||
~ "&group=" ~ (last_group|urlencode), cols=5) }}
|
||||
{% else %}
|
||||
{{ scroll_end(cols=5) }}
|
||||
{% endif %}
|
||||
|
||||
@@ -49,11 +49,11 @@ async def test_the_page_names_and_links_the_entity(app):
|
||||
async def test_the_log_continues_itself_instead_of_a_load_more_link(app):
|
||||
events = [{"event": _ev("host", "h1", "host.renamed"), "detail": {}, "entity": None}]
|
||||
async with app.test_request_context("/audit/"):
|
||||
more = await render_template("audit/list.html", events=events,
|
||||
more = await render_template("audit/list.html", events=events, last_group="Today",
|
||||
after="2026-10-07T03:00:00+00:00|0190abc", action_filter="host.")
|
||||
last = await render_template("audit/_rows.html", events=events, after=None, action_filter="")
|
||||
assert ('data-scroll-more hx-get="/audit/?before=2026-10-07T03%3A00%3A00%2B00%3A00%7C0190abc'
|
||||
'&action=host."') in more
|
||||
'&action=host.&group=Today"') in more
|
||||
assert "Load more" not in more
|
||||
assert "data-scroll-more" not in last and "End of the list." in last
|
||||
|
||||
@@ -107,3 +107,30 @@ async def test_each_entry_shows_its_exact_moment_in_the_readers_zone(app):
|
||||
assert '<div><span class="tone-dim">was</span> web-0</div>' in page
|
||||
table = page[page.index('<table class="table">'):]
|
||||
assert "style=" not in table
|
||||
|
||||
|
||||
def test_dividers_open_each_day_group_once_even_across_pages():
|
||||
"""#5214: the operator wants Today, Yesterday, the last week and older as
|
||||
dividers, with every row's own date beside it."""
|
||||
from steward.audit.service import mark_day_groups
|
||||
now = datetime(2026, 10, 8, 18, 0, tzinfo=timezone.utc)
|
||||
at = lambda d, h=12: NS(timestamp=datetime(2026, 10, d, h, 0, tzinfo=timezone.utc))
|
||||
page1 = [{"event": at(8)}, {"event": at(8, 9)}, {"event": at(7)}, {"event": at(5)}]
|
||||
assert mark_day_groups(page1, timezone.utc, "", now) == "Last 7 days"
|
||||
assert [(i["group"], i["opens_group"]) for i in page1] == [
|
||||
("Today", True), ("Today", False), ("Yesterday", True), ("Last 7 days", True)]
|
||||
# The next page carries the group the last ended in, so it doesn't reopen it.
|
||||
page2 = [{"event": at(3)}, {"event": at(1)}]
|
||||
assert mark_day_groups(page2, timezone.utc, "Last 7 days", now) == "Older"
|
||||
assert [(i["group"], i["opens_group"]) for i in page2] == [("Last 7 days", False), ("Older", True)]
|
||||
|
||||
|
||||
async def test_a_divider_row_heads_each_group(app):
|
||||
events = [{"event": _ev("host", "h1", "host.renamed"), "detail": {}, "entity": None, "facts": [],
|
||||
"group": "Today", "opens_group": True},
|
||||
{"event": _ev("host", "h1", "host.renamed"), "detail": {}, "entity": None, "facts": [],
|
||||
"group": "Today", "opens_group": False}]
|
||||
async with app.test_request_context("/audit/"):
|
||||
page = await render_template("audit/_rows.html", events=events, after=None, action_filter="")
|
||||
assert page.count('<tr class="table-divider"><th colspan="5" scope="rowgroup">Today</th></tr>') == 1
|
||||
assert "Today 1" not in page and ">Tue 6 Oct " in page # the row keeps its date
|
||||
|
||||
@@ -57,7 +57,8 @@ def test_ago_of_nothing_and_of_another_zone():
|
||||
assert 'title="Tue 6 Oct 2026 18:00:00 UTC"' in str(f.ago(NOW.astimezone(est), now=NOW))
|
||||
|
||||
|
||||
# Milestone 490: a moment in the reader's own zone, on a 24-hour clock.
|
||||
# Milestone 490: a moment in the reader's own zone, on a 24-hour clock, always
|
||||
# with its date: "Today" and "Yesterday" are dividers, never the time (#5214).
|
||||
NY = f.zone("America/New_York")
|
||||
|
||||
|
||||
@@ -67,14 +68,14 @@ def test_ago_hovers_the_moment_in_the_readers_zone():
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moment, text, title", [
|
||||
(NOW - timedelta(hours=1), "Today 13:00", "Tue 6 Oct 2026 13:00:00 EDT"),
|
||||
(NOW - timedelta(hours=1), "Tue 6 Oct 13:00", "Tue 6 Oct 2026 13:00:00 EDT"),
|
||||
# 03:30 UTC on the 6th is 23:30 on the 5th in New York: the zone moves the day.
|
||||
(datetime(2026, 10, 6, 3, 30, tzinfo=timezone.utc), "Yesterday 23:30", "Mon 5 Oct 2026 23:30:00 EDT"),
|
||||
(datetime(2026, 10, 7, 6, 0, tzinfo=timezone.utc), "Tomorrow 02:00", "Wed 7 Oct 2026 02:00:00 EDT"),
|
||||
(datetime(2026, 10, 6, 3, 30, tzinfo=timezone.utc), "Mon 5 Oct 23:30", "Mon 5 Oct 2026 23:30:00 EDT"),
|
||||
(datetime(2026, 10, 7, 6, 0, tzinfo=timezone.utc), "Wed 7 Oct 02:00", "Wed 7 Oct 2026 02:00:00 EDT"),
|
||||
(datetime(2026, 9, 30, 12, 0, tzinfo=timezone.utc), "Wed 30 Sep 08:00", "Wed 30 Sep 2026 08:00:00 EDT"),
|
||||
# Before this year the year shows; in winter the zone is EST, an hour further back.
|
||||
(datetime(2025, 12, 25, 12, 0, tzinfo=timezone.utc), "25 Dec 2025 07:00", "Thu 25 Dec 2025 07:00:00 EST"),
|
||||
(datetime(2026, 10, 6, 21, 5), "Today 17:05", "Tue 6 Oct 2026 17:05:00 EDT"), # naive is UTC
|
||||
(datetime(2026, 10, 6, 21, 5), "Tue 6 Oct 17:05", "Tue 6 Oct 2026 17:05:00 EDT"), # naive is UTC
|
||||
])
|
||||
def test_when_writes_the_exact_moment_in_the_readers_zone(moment, text, title):
|
||||
stamp = moment.replace(tzinfo=timezone.utc) if moment.tzinfo is None else moment
|
||||
@@ -83,11 +84,24 @@ def test_when_writes_the_exact_moment_in_the_readers_zone(moment, text, title):
|
||||
|
||||
|
||||
def test_when_is_a_24_hour_clock_and_utc_without_a_zone():
|
||||
assert ">Today 18:00<" in str(f.when(NOW, now=NOW))
|
||||
assert ">Today 23:59<" in str(f.when(NOW.replace(hour=23, minute=59), now=NOW))
|
||||
assert ">Tue 6 Oct 18:00<" in str(f.when(NOW, now=NOW))
|
||||
assert ">Tue 6 Oct 23:59<" in str(f.when(NOW.replace(hour=23, minute=59), now=NOW))
|
||||
assert str(f.when(None)) == "—"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moment, group", [
|
||||
(NOW - timedelta(hours=1), "Today"),
|
||||
(datetime(2026, 10, 6, 3, 30, tzinfo=timezone.utc), "Yesterday"), # 23:30 on the 5th in New York
|
||||
(datetime(2026, 10, 5, 3, 30, tzinfo=timezone.utc), "Last 7 days"),
|
||||
(datetime(2026, 9, 30, 12, 0, tzinfo=timezone.utc), "Last 7 days"), # six days back
|
||||
(datetime(2026, 9, 29, 12, 0, tzinfo=timezone.utc), "Older"), # seven
|
||||
(datetime(2025, 12, 25, 12, 0, tzinfo=timezone.utc), "Older"),
|
||||
])
|
||||
def test_day_group_counts_calendar_days_in_the_readers_zone(moment, group):
|
||||
assert f.day_group(moment, NY, now=NOW) == group
|
||||
assert group in f.DAY_GROUPS
|
||||
|
||||
|
||||
def test_zone_falls_back_to_utc_and_names_itself():
|
||||
assert f.zone(None) is timezone.utc and f.zone("") is timezone.utc
|
||||
assert f.zone("Not/AZone") is timezone.utc and f.zone("../etc/passwd") is timezone.utc
|
||||
|
||||
@@ -108,3 +108,34 @@ def test_router_history_reads_every_router_in_one_query_bucketed(app):
|
||||
assert got[a][0].latency_p95_ms == 20.0
|
||||
assert [r.request_rate for r in got[b]] == [4.0]
|
||||
assert got[idle] == [] # its only row is before the range
|
||||
|
||||
|
||||
def test_recent_window_weights_shares_by_traffic_and_keeps_to_the_window(app):
|
||||
"""#5214: a router is graded on its last 15 minutes. An idle scrape's
|
||||
100% 4xx counts for its trickle of traffic, not as a third of the
|
||||
average; a row older than the window is not read."""
|
||||
from plugins.traefik.models import TraefikMetric
|
||||
from plugins.traefik.routes import recent_window
|
||||
|
||||
now = datetime(2097, 3, 1, 12, 0, tzinfo=timezone.utc) + timedelta(minutes=uuid.uuid4().int % 10_000)
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
busy, empty = f"busy-{tag}@docker", f"empty-{tag}@docker"
|
||||
|
||||
def row(name, ago, rate, e4, p95=10.0):
|
||||
return TraefikMetric(router_name=name, scraped_at=now - timedelta(minutes=ago), request_rate=rate,
|
||||
error_rate_4xx_pct=e4, error_rate_5xx_pct=0.0, latency_p50_ms=1.0,
|
||||
latency_p95_ms=p95, latency_p99_ms=3.0, response_bytes_rate=0.0)
|
||||
|
||||
async def go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
s.add_all([row(busy, 1, 9.9, 0.0, 10.0), row(busy, 2, 0.1, 100.0, 30.0),
|
||||
row(busy, 30, 50.0, 100.0), # outside the window
|
||||
row(empty, 30, 1.0, 100.0)])
|
||||
async with s.begin():
|
||||
return await recent_window(s, [busy, empty], now)
|
||||
|
||||
got = asyncio.run(go())
|
||||
assert set(got) == {busy} # nothing in empty's window
|
||||
assert got[busy].error_rate_4xx_pct == pytest.approx(1.0) # 0.1 × 100 over 10 req/s
|
||||
assert got[busy].latency_p95_ms == pytest.approx(20.0)
|
||||
|
||||
@@ -246,3 +246,42 @@ async def test_the_request_chart_says_its_figures_before_the_chart(app):
|
||||
assert "Requests/s (all routers)" in page and "max: 100" not in page
|
||||
assert "couldn't be drawn" in page # a failure says why in place of the chart
|
||||
assert "No Traefik requests recorded in the last 6h." in empty
|
||||
|
||||
|
||||
def test_routers_are_graded_on_their_recent_window_not_one_scrape():
|
||||
"""#5214: a 4xx share near the cutoff crossed it on one scrape and fell
|
||||
back on the next, so problem cards came and went every poll."""
|
||||
from plugins.traefik.routes import STATE_WINDOW_WORDS, grade_routers
|
||||
spiky, quiet = _latest(e4=1.5), _latest(e4=1.5)
|
||||
routers = [{"name": "spiky", "latest": spiky}, {"name": "quiet", "latest": quiet}]
|
||||
# spiky's last 15 min average well under the cutoff; quiet has no window rows.
|
||||
grade_routers(routers, {"spiky": NS(error_rate_4xx_pct=0.4, error_rate_5xx_pct=0.0, latency_p95_ms=20.0)}, TH)
|
||||
assert routers[0]["state"] == "ok"
|
||||
assert routers[1]["state"] == "warn" and not routers[1]["why"].endswith(STATE_WINDOW_WORDS)
|
||||
held = [{"name": "held", "latest": _latest()}] # clean now, bad over the window
|
||||
grade_routers(held, {"held": NS(error_rate_4xx_pct=3.0, error_rate_5xx_pct=0.0, latency_p95_ms=20.0)}, TH)
|
||||
assert held[0]["state"] == "warn"
|
||||
assert held[0]["why"] == "3.0% of responses are 4xx over the last 15 min"
|
||||
|
||||
|
||||
def test_the_last_reload_gauge_is_a_moment_not_a_flag():
|
||||
"""#5355: traefik_config_last_reload_success is a Unix time; read as a
|
||||
0/1 flag it said "Reload failed" on every scrape."""
|
||||
from plugins.traefik.scraper import compute_global_stats
|
||||
at = compute_global_stats({"traefik_config_last_reload_success": {frozenset(): 1791460000.0},
|
||||
"traefik_config_reloads_total": {frozenset(): 343.0}}, None)
|
||||
assert at["config_last_reload_at"] == datetime.fromtimestamp(1791460000.0, timezone.utc)
|
||||
assert at["config_reloads_total"] == 343.0
|
||||
never = compute_global_stats({"traefik_config_last_reload_success": {frozenset(): 0.0}}, None)
|
||||
assert never["config_last_reload_at"] is None
|
||||
|
||||
|
||||
async def test_the_config_tile_says_when_it_last_reloaded(app):
|
||||
stat = NS(config_last_reload_at=datetime.now(timezone.utc) - timedelta(minutes=3),
|
||||
config_reloads_total=343.0, open_conns_total=None, process_memory_bytes=None)
|
||||
async with app.test_request_context("/plugins/traefik/rows"):
|
||||
page = await render_plugin_template(app, "traefik", "traefik/rows.html", problems=[], healthy=[],
|
||||
gone=[], global_stat=stat, certs=[], old_certs=[], range_key="24h",
|
||||
error_4xx=1.0, error_5xx=0.1)
|
||||
assert "Config reloaded" in page and "3m ago" in page and "343 reloads" in page
|
||||
assert "Reload failed" not in page
|
||||
|
||||
Reference in New Issue
Block a user