FabledSteward/tests/plugins/host_agent/test_host_health.py

"""The host page says how the host is in one header, and grades every reading by
Settings → Thresholds, nowhere else (#5122)."""
import re
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace

import pytest

from steward.core.formatting import worst_reading
from steward.core.settings import to_thresholds_cfg
from tests._templates import render_plugin_template
from tests.test_style_tokens import hex_colours

TH = to_thresholds_cfg({})  # cpu/mem/disk warn 80 crit 90; load warn 80 crit 100


@pytest.mark.parametrize("readings, worst", [
    ([("CPU", 40, "cpu"), ("Memory", 50, "mem")], None),
    ([("CPU", None, "cpu")], None),                                       # no reading is not a problem
    ([("CPU", 85, "cpu"), ("Memory", 95, "mem")], ("Memory", 95, "crit")),  # crit beats warn
    ([("CPU", 92, "cpu"), ("Memory", 95, "mem")], ("CPU", 92, "crit")),     # first of equals
    ([("Disk /", 81, "disk")], ("Disk /", 81, "warn")),
])
def test_worst_reading(readings, worst):
    assert worst_reading(readings, TH) == worst


async def _vitals(app, *, cpu=40.0, mem=50.0, stale=False, seen_min=1, temp=None, hottest=None):
    readings = (("CPU", cpu, "cpu"), ("Memory", mem, "mem"), ("Disk /", 30.0, "disk"),
                ("Load per core", 20, "load"), ("Temperature", temp, "temp"))
    async with app.test_request_context("/plugins/host_agent/vitals/h1"):
        page = await render_plugin_template(
            app, "host_agent", "_host_vitals.html", reporting=True, stale=stale,
            reg=SimpleNamespace(host_id="h1", agent_version="1.9.0",
                                last_seen_at=datetime.now(timezone.utc) - timedelta(minutes=seen_min)),
            cpu=cpu, mem=mem, disk_root=30.0, load_per_core=20, temp=temp, hottest=hottest,
            sparks={"cpu": "", "mem": "", "disk": "", "load": "", "temp": "<svg></svg>"},
            psi={"cpu": None, "mem": None, "io": None}, worst=worst_reading(readings, TH))
    return re.sub(r">\s+<", "><", " ".join(page.split()))


async def test_a_host_over_a_threshold_says_which_reading(app):
    page = await _vitals(app, cpu=40.0, mem=93.0)
    assert '<span class="badge badge-ok">Live</span>' in page
    assert '<span class="tone-crit">Memory 93%</span> is past its critical threshold.' in page
    assert 'class="stat-tile stat-crit"><span class="stat-tile-label">Memory</span>' in page
    assert 'class="stat-tile stat-ok"><span class="stat-tile-label">CPU</span>' in page
    assert "seen <time" in page and "agent v1.9.0" in page
    assert not hex_colours(page), hex_colours(page)


async def test_a_quiet_host_says_so(app):
    assert "Every reading is within its thresholds." in await _vitals(app)


async def test_the_strip_at_the_top_of_the_page_leads_to_the_full_metrics(app):
    """The temperatures, charts and every mount are on the full-metrics page;
    its link sat in the Agent card below the fold and was missed."""
    assert 'href="/plugins/host_agent/h1/"' in await _vitals(app)


async def test_the_strip_shows_the_hottest_sensor_with_its_name_and_line(app):
    page = await _vitals(app, temp=62.0, hottest="Package id 0")
    assert "--tiles:5;" in page
    assert ('class="stat-tile stat-ok"><span class="stat-tile-label">Temperature</span>'
            '<span class="stat-tile-value">62°C</span>') in page
    assert '<svg></svg><span class="stat-tile-sensor" title="Package id 0">Package id 0</span>' in page
    without = await _vitals(app)
    assert "--tiles:4;" in without and ">Temperature<" not in without


async def test_a_host_past_its_heat_threshold_says_so_in_degrees(app):
    page = await _vitals(app, temp=88.0, hottest="Package id 0")      # crit at 85
    assert '<span class="tone-crit">Temperature 88°C</span> is past its critical threshold.' in page
    assert 'class="stat-tile stat-crit"><span class="stat-tile-label">Temperature</span>' in page


async def test_a_stale_host_says_since_when_before_any_reading(app):
    page = await _vitals(app, mem=93.0, stale=True, seen_min=20)
    assert '<span class="badge badge-warn">Stale</span>' in page
    assert "No report since <time" in page and ">20m ago</time>; the readings below are from then." in page
    assert "past its" not in page          # an old reading is not news


async def test_the_metrics_page_grades_by_the_same_cutoffs(app):
    # 75 % was amber under the old hard-coded 70/90; Settings → Thresholds says warn at 80.
    async with app.test_request_context("/plugins/host_agent/h1/metrics"):
        page = await render_plugin_template(
            app, "host_agent", "_host_metrics.html", host=SimpleNamespace(address="10.0.0.2"), stale=False,
            reg=SimpleNamespace(distro=None, kernel=None, arch=None, agent_version=None,
                                last_seen_at=datetime.now(timezone.utc)),
            hostlvl={"cpu_pct": 75.0, "mem_used_pct": 95.0, "temp_c_max": 72.0},
            cores=[(0, 85.0)], nets={}, disks_io={}, temps={"cpu": 90.0},
            mounts={"/": {"disk_used_pct": 50.0, "disk_used_bytes": 1, "disk_total_bytes": 2}})
    assert 'class="stat-val tone-ok" style="font-size:1.6rem;">75%' in page
    assert 'class="stat-val tone-crit" style="font-size:1.6rem;">95%' in page
    assert 'class="meter-fill meter-ok" style="width:75.0%;"' in page
    assert 'class="meter-fill meter-warn" style="width:85.0%;"' in page   # the core
    assert 'class="stat-val tone-warn" style="font-size:1.6rem;">72°C' in page
    assert '<span class="tone-crit" style="font-variant-numeric:tabular-nums;">90°C' in page
    assert not hex_colours(page), hex_colours(page)


async def test_sensors_not_counted_are_listed_once_per_reason(app):
    """Util01's five open inputs read as one list with its reason, not five
    repeats of it; a sensor judged against the CPU carries its own (#5783)."""
    held = "reads a value no wired sensor gives"
    falls = "falls as the CPU warms (r -0.81 over 168 h)"
    async with app.test_request_context("/plugins/host_agent/h1/metrics"):
        page = await render_plugin_template(
            app, "host_agent", "_host_metrics.html", host=SimpleNamespace(address="10.0.0.2"), stale=False,
            reg=SimpleNamespace(distro=None, kernel=None, arch=None, agent_version=None,
                                last_seen_at=datetime.now(timezone.utc)),
            hostlvl={"temp_c_max": 52.0}, cores=[], nets={}, disks_io={}, temps={"Package id 0": 52.0},
            unwired_temps={"SYSTIN": (115.0, held), "AUXTIN0": (110.0, held), "temp3": (62.0, falls)},
            mounts={})
    page = " ".join(page.split())
    assert (f"Not counted, likely nothing wired to them: temp3 62°C ({falls}); "
            f"SYSTIN 115°C, AUXTIN0 110°C ({held}).") in page


async def test_an_nfs_mount_says_what_a_request_took_and_a_local_one_says_nothing(app):
    """#5996: the delay under each NFS mount's bar, graded at 100 and 1000 ms."""
    async with app.test_request_context("/plugins/host_agent/h1/metrics"):
        page = await render_plugin_template(
            app, "host_agent", "_host_metrics.html", host=SimpleNamespace(address="10.0.0.2"), stale=False,
            reg=SimpleNamespace(distro=None, kernel=None, arch=None, agent_version=None,
                                last_seen_at=datetime.now(timezone.utc)),
            hostlvl={"cpu_pct": 10.0}, cores=[], nets={}, disks_io={}, temps={},
            mounts={"/nfs/data": {"disk_used_pct": 40.0, "nfs_ops_per_sec": 4.0, "nfs_exe_ms": 250.0,
                                  "nfs_retrans": 3.0},
                    "/nfs/media": {"disk_used_pct": 30.0, "nfs_ops_per_sec": 0.0},
                    "/": {"disk_used_pct": 20.0}})
    page = re.sub(r">\s+<", "><", " ".join(page.split()))
    assert '<span class="tone-warn">250.0 ms a request</span> · 4/s · <span class="tone-warn">3 sent again</span>' in page
    assert "NFS idle" in page and page.count("NFS ") == 2
    assert "timed out" not in page
    assert not hex_colours(page), hex_colours(page)