FabledSteward/tests/plugins/traefik/test_baselines.py

"""#5844: one metrics URL can reach several Traefik processes (swarm replicas,
pods, a keepalived pair, a restart). Each is counted only against its own last
scrape, so one replica's lifetime totals never read as another's new traffic."""
from datetime import datetime, timezone
from functools import partial

import httpx
import pytest

from plugins.traefik import scraper
from plugins.traefik.scraper import (
    BASELINE_TTL_SECONDS, Baselines, combine, compute_router_metrics, instance_of, merge_global_stats, unique_certs,
)

REQ = "traefik_service_requests_total"


def _scrape(started: float | None, **counts):
    """A scrape from the process started at `started`, with {code: total} for qbittorrent@file."""
    out = {REQ: {frozenset({"service": "qbittorrent@file", "code": c, "method": "GET"}.items()): v
                 for c, v in counts.items()}}
    if started is not None:
        out["process_start_time_seconds"] = {frozenset(): started}
    return out


def test_each_process_is_compared_only_with_itself():
    b = Baselines()
    a1, b1 = _scrape(1000.0, **{"200": 10}), _scrape(2000.0, **{"502": 336})
    assert b.swap(a1, 0.0) == (None, 0.0)
    # The other replica answers: it has no baseline yet, so nothing is counted.
    assert b.swap(b1, 60.0) == (None, 0.0)
    # Back to the first: compared with its own scrape two minutes ago.
    a2 = _scrape(1000.0, **{"200": 22})
    assert b.swap(a2, 120.0) == (a1, 120.0)
    assert b.swap(_scrape(2000.0, **{"502": 336}), 180.0) == (b1, 120.0)


def test_landing_on_the_other_replica_no_longer_reads_its_totals_as_traffic():
    """The live symptom: an idle retired router flashing 336 502s, 100% 5xx."""
    b = Baselines()
    b.swap(_scrape(1000.0, **{"200": 10}), 0.0)
    previous, elapsed = b.swap(_scrape(2000.0, **{"502": 336}), 60.0)
    got = compute_router_metrics(_scrape(2000.0, **{"502": 336}), previous, elapsed)
    assert got["qbittorrent@file"]["request_rate"] == 0.0
    assert got["qbittorrent@file"]["error_rate_5xx_pct"] == 0.0


def test_a_restarted_traefik_starts_a_new_baseline():
    b = Baselines()
    b.swap(_scrape(1000.0, **{"200": 5000}), 0.0)
    assert b.swap(_scrape(3000.0, **{"200": 3}), 60.0) == (None, 0.0)


def test_without_a_start_time_every_scrape_is_one_process_as_before():
    b = Baselines()
    first = _scrape(None, **{"200": 1})
    assert instance_of(first) == ""
    b.swap(first, 0.0)
    assert b.swap(_scrape(None, **{"200": 2}), 60.0) == (first, 60.0)


def test_a_baseline_older_than_the_ttl_is_dropped_not_compared():
    b = Baselines()
    b.swap(_scrape(1000.0, **{"200": 1}), 0.0)
    assert b.swap(_scrape(1000.0, **{"200": 900}), BASELINE_TTL_SECONDS + 1) == (None, 0.0)


# ── Every replica, added up (#5844) ──────────────────────────────────────────

def _router(started, service, **counts):
    out = {REQ: {frozenset({"service": service, "code": c, "method": "GET"}.items()): v
                 for c, v in counts.items()}}
    out["process_start_time_seconds"] = {frozenset(): started}
    return out


def test_two_replicas_add_up_and_a_new_one_waits_for_its_baseline():
    a0, a1 = _router(1.0, "app@swarm", **{"200": 100}), _router(1.0, "app@swarm", **{"200": 160})
    b0, b1 = _router(2.0, "app@swarm", **{"200": 50}), _router(2.0, "app@swarm", **{"200": 70, "502": 4})
    c1 = _router(3.0, "app@swarm", **{"200": 9000})                 # first seen: no baseline
    current, previous, elapsed = combine([(a1, a0, 60.0), (b1, b0, 60.0), (c1, None, 0.0)])
    got = compute_router_metrics(current, previous, elapsed)["app@swarm"]
    # (160-100) + (70-50) + 4 new 502s = 84 requests in 60 s; C's 9000 are not traffic.
    assert got["request_rate"] == pytest.approx(84 / 60)
    assert got["error_rate_5xx_pct"] == pytest.approx(4 / 84 * 100)


def test_the_busy_replica_counts_though_the_idle_one_has_no_routers():
    """The live case: the VIP sends everything to one replica; the other has
    no traefik_service_* series at all."""
    busy0, busy1 = _router(1.0, "roundtable@swarm", **{"200": 10}), _router(1.0, "roundtable@swarm", **{"200": 40})
    idle = {"process_start_time_seconds": {frozenset(): 2.0}}
    current, previous, elapsed = combine([(busy1, busy0, 60.0), (idle, idle, 60.0)])
    assert compute_router_metrics(current, previous, elapsed)["roundtable@swarm"]["request_rate"] == 0.5


def test_with_no_baseline_yet_every_router_is_listed_at_zero():
    current, previous, _ = combine([(_router(1.0, "a@swarm", **{"200": 5}), None, 0.0),
                                    (_router(2.0, "b@swarm", **{"200": 7}), None, 0.0)])
    assert previous is None
    got = compute_router_metrics(current, previous, 0.0)
    assert set(got) == {"a@swarm", "b@swarm"} and got["a@swarm"]["request_rate"] == 0.0


def test_traefiks_own_figures_add_up_or_take_the_newest():
    t1, t2 = datetime(2026, 10, 15, 18, 0, tzinfo=timezone.utc), datetime(2026, 10, 15, 18, 43, tzinfo=timezone.utc)
    got = merge_global_stats([
        {"open_conns_total": 40.0, "config_reloads_total": 70.0, "config_last_reload_at": t1, "process_memory_bytes": 2e8},
        {"open_conns_total": 1.0, "config_reloads_total": 72.0, "config_last_reload_at": t2, "process_memory_bytes": 1e8},
    ])
    assert got == {"open_conns_total": 41.0, "config_reloads_total": 72.0, "config_last_reload_at": t2,
                   "process_memory_bytes": 3e8}


def test_a_certificate_both_replicas_serve_is_listed_once():
    cert = {"traefik_tls_certs_not_after": {frozenset({"serial": "ab", "cn": "x.example", "sans": ""}.items()): 2e9}}
    assert [c["serial"] for c in unique_certs([cert, cert])] == ["ab"]


EXPO = {
    "10.0.2.5": "process_start_time_seconds 1.0\ntraefik_service_requests_total{service=\"a@swarm\",code=\"200\",method=\"GET\"} 10\n",
    "10.0.2.6": "process_start_time_seconds 2.0\n",
    "10.0.2.7": "process_start_time_seconds 1.0\n",                # the same process again
}


async def test_a_name_for_several_replicas_is_scraped_address_by_address(monkeypatch):
    seen_hosts = []

    def answer(request: httpx.Request) -> httpx.Response:
        seen_hosts.append(request.headers["host"])
        if request.url.host == "10.0.2.8":
            return httpx.Response(503)
        return httpx.Response(200, text=EXPO[request.url.host])

    async def addresses(host, port):
        assert (host, port) == ("tasks.traefik_router", 8080)
        return ["10.0.2.5", "10.0.2.6", "10.0.2.7", "10.0.2.8"]

    monkeypatch.setattr(scraper, "_addresses", addresses)
    monkeypatch.setattr(scraper.httpx, "AsyncClient",
                        partial(httpx.AsyncClient, transport=httpx.MockTransport(answer)))
    got = await scraper.fetch_instances("http://tasks.traefik_router:8080/metrics")
    # 10.0.2.8 didn't answer and 10.0.2.7 is 10.0.2.5's process: two Traefiks.
    assert sorted(instance_of(g) for g in got) == ["1.00", "2.00"]
    assert set(seen_hosts) == {"tasks.traefik_router:8080"}


async def test_a_name_with_one_address_is_one_scrape_as_before(monkeypatch):
    async def addresses(host, port):
        return ["10.0.2.5"]

    monkeypatch.setattr(scraper, "_addresses", addresses)
    monkeypatch.setattr(scraper.httpx, "AsyncClient", partial(
        httpx.AsyncClient, transport=httpx.MockTransport(lambda r: httpx.Response(200, text=EXPO["10.0.2.5"]))))
    [got] = await scraper.fetch_instances("http://trfk.traefik.internal/metrics")
    assert instance_of(got) == "1.00"