diff --git a/steward/hosts/routes.py b/steward/hosts/routes.py index cced3b9..7ac08cb 100644 --- a/steward/hosts/routes.py +++ b/steward/hosts/routes.py @@ -27,21 +27,15 @@ def _ansible_source_names() -> list[str]: @hosts_bp.get("/overview/widget") @require_role(UserRole.viewer) async def overview_widget(): - """Dashboard widget: unified per-host monitor + agent glance, linking to the hub.""" + """Dashboard widget: each host's resources, worst first (#5251). Uptime and + latency belong to the Monitors widget, so this one doesn't read them.""" async with current_app.db_sessionmaker() as db: hosts = (await db.execute(select(Host).order_by(Host.name))).scalars().all() host_status = await host_svc.host_status(db, [h.id for h in hosts]) - uptime = await host_svc.compute_uptime(db) - host_names = [h.name for h in hosts] agent = await host_svc.agent_overview_by_host( - db, host_names, host_svc.agent_stale_after_seconds(current_app.config)) - cpu_sparks = await host_svc.agent_cpu_sparks_by_host(db, host_names) - - return await render_template( - "hosts/overview_widget.html", - hosts=hosts, host_status=host_status, - uptime=uptime, agent=agent, cpu_sparks=cpu_sparks, - ) + db, [h.name for h in hosts], host_svc.agent_stale_after_seconds(current_app.config)) + rows = host_svc.host_glance(hosts, host_status, agent, current_app.config.get("THRESHOLDS", {})) + return await render_template("hosts/overview_widget.html", rows=rows) @hosts_bp.get("/") diff --git a/steward/hosts/service.py b/steward/hosts/service.py index d7ff4a0..46fb4bc 100644 --- a/steward/hosts/service.py +++ b/steward/hosts/service.py @@ -170,10 +170,13 @@ async def host_monitors(db, host_id: str) -> list[dict]: async def agent_overview_by_host( db, host_names: list[str], stale_after_seconds: int = 180, ) -> dict[str, dict]: - """Per-host agent glance (cpu/mem/disk + freshness) from plugin_metric_latest. + """Per-host agent glance (cpu/mem/disk/load + freshness) from plugin_metric_latest. {host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root, load_1m, - last_at, fresh}} for each host with any agent samples. + cores, load_per_core, last_at, fresh}} for each host with any + agent samples. Cores are counted from the per-core CPU sub-resources + (":core"), as on the host page; load_per_core is load_1m as a % + of them, the figure Settings → Thresholds grades "load" by. Core-safe (no host_agent import). Freshness compares the latest host-level sample against the host_agent plugin's stale window (stale_after_seconds; @@ -188,6 +191,16 @@ async def agent_overview_by_host( latest = await latest_by_resource( db, "host_agent", list(host_names) + [h + ":/" for h in host_names], set(AGENT_GLANCE_METRICS) | {"disk_used_pct"}) + from steward.models.metrics import PluginMetricLatest + cores: dict[str, int] = {} + for res in (await db.execute( + select(PluginMetricLatest.resource_name).where( + PluginMetricLatest.source_module == "host_agent", + PluginMetricLatest.metric_name == "cpu_pct", + PluginMetricLatest.resource_name.like("%:core%")) + )).scalars(): + host = res.rsplit(":core", 1)[0] + cores[host] = cores.get(host, 0) + 1 out: dict[str, dict] = {} for name in host_names: own = {m: r for m, r in latest.get(name, {}).items() if m in AGENT_GLANCE_METRICS} @@ -200,35 +213,44 @@ async def agent_overview_by_host( ts = max((r.recorded_at for r in own.values()), default=None) d["last_at"] = ts d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after) + d["cores"] = cores.get(name) + load = d.get("load_1m") + d["load_per_core"] = round(load / d["cores"] * 100) if (load is not None and d["cores"]) else None out[name] = d return out -async def agent_cpu_sparks_by_host(db, host_names: list[str], hours: int = 1) -> dict[str, str]: - """{host_name: inline-SVG cpu sparkline} over the last `hours` (core-safe). +GLANCE_ORDER = {"down": 0, "crit": 1, "stale": 2, "warn": 3, "up": 4, "unknown": 5} - A tiny at-a-glance trend in each row — the host-view sparkline, on the widget. - """ - from steward.models.metrics import PluginMetric - from steward.core.status import sparkline_svg - if not host_names: - return {} - cutoff = datetime.now(timezone.utc) - timedelta(hours=hours) - rows = (await db.execute( - select(PluginMetric.resource_name, PluginMetric.value, PluginMetric.recorded_at) - .where( - PluginMetric.source_module == "host_agent", - PluginMetric.metric_name == "cpu_pct", - PluginMetric.resource_name.in_(host_names), - PluginMetric.recorded_at >= cutoff, - ) - .order_by(PluginMetric.recorded_at) - )).all() - by_host: dict[str, list[float]] = {} - for r in rows: - by_host.setdefault(r.resource_name, []).append(r.value) - # Amber line to read as "CPU"; only draw when there are a couple of points. - return { - name: sparkline_svg(vals[-40:], width=64, height=18, stroke="#c8a840") - for name, vals in by_host.items() if len(vals) >= 2 - } + +def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict]: + """The Hosts — Overview rows, worst first (#5251). + + Each is {host, state, worst, agent}. State is "down" when a linked monitor + is down, "stale" when the agent has stopped reporting, else the grade of + the reading furthest past its cutoff ("crit" / "warn": CPU, memory, disk / + or load per core, against Settings → Thresholds), else "up"; "unknown" for + a host with neither monitors nor agent. `worst` is (label, value, grade) + or None, from formatting.worst_reading.""" + from steward.core.formatting import worst_reading + rows = [] + for h in hosts: + st, a = status.get(h.id), agent.get(h.name) + worst = None + if a and a.get("fresh"): + worst = worst_reading(( + ("CPU", a.get("cpu_pct"), "cpu"), ("Memory", a.get("mem_used_pct"), "mem"), + ("Disk /", a.get("disk_root"), "disk"), ("Load", a.get("load_per_core"), "load")), + thresholds) + if st and st["state"] == "down": + state = "down" + elif a and not a.get("fresh"): + state = "stale" + elif worst: + state = worst[2] + elif st or a: + state = "up" + else: + state = "unknown" + rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}}) + return sorted(rows, key=lambda r: (GLANCE_ORDER[r["state"]], r["host"].name.lower())) diff --git a/steward/templates/hosts/overview_widget.html b/steward/templates/hosts/overview_widget.html index 53ee2f6..1c42369 100644 --- a/steward/templates/hosts/overview_widget.html +++ b/steward/templates/hosts/overview_widget.html @@ -1,50 +1,43 @@ -{# Unified Hosts widget — monitor status + agent glance per host, links to the hub. - Name sits on its own line (never truncated); its data wraps beneath it, grouped - under the host. Prefer vertical growth over cramming a row. #} -{% if hosts %} -
- {% for host in hosts %} - {% set hs = host_status.get(host.id) %} - {% set ut = uptime.get(host.id, {}) %} - {% set a = agent.get(host.name, {}) %} - {% set down = hs and hs.state == 'down' %} - {% set spark = cpu_sparks.get(host.name) %} -
- {# Line 1 — status dot + full host name (own line) + cpu trend #} -
- - {% if session.user_id %} - {{ host.name }} - {% else %} - {{ host.name }} - {% endif %} -
- {# Line 2 — monitors + agent metrics; wraps instead of truncating #} -
- {% if hs and hs.latency_ms is not none %} - ping {{ "%.0f"|format(hs.latency_ms) }}ms - {% endif %} - {% if ut.get('24h') is not none %} - 24h {{ "%.1f"|format(ut['24h']) }}% - {% endif %} - {% if a %} - {% if a.get('cpu_pct') is not none %}cpu {{ "%.0f"|format(a.cpu_pct) }}%{% if spark %}{{ spark | safe }}{% endif %}{% endif %} - {% if a.get('mem_used_pct') is not none %}mem {{ "%.0f"|format(a.mem_used_pct) }}%{% endif %} - {% if a.get('disk_root') is not none %}disk / {{ "%.0f"|format(a.disk_root) }}%{% endif %} - {% if a.get('load_1m') is not none %}load {{ "%.2f"|format(a.load_1m) }}{% endif %} - {% if not a.fresh %}stale{% endif %} - {% else %} - no agent - {% endif %} -
-
- {% endfor %} +{# hosts/overview_widget.html — the dashboard Hosts widget (#5251): a verdict, + then one row per host, worst first, with its resources under one header. + Load is shown against the host's cores and graded per core. Reachability + and uptime belong to the Monitors widget, so they aren't repeated here. #} +{% from "_macros.html" import status_dot %} +{% if not rows %} +
+ No hosts yet.{% if session.user_id %} Add one →{% endif %}
{% else %} -
- No hosts yet. Add one. +{% set problems = rows | selectattr("state", "in", ["down", "crit", "stale", "warn"]) | list %} +
+ {%- if not problems %}All {{ rows|length }} host{{ 's' if rows|length != 1 }} healthy. + {%- elif problems|length == 1 %}{% set p = problems[0] %}{{ p.host.name }}: + {% if p.state == 'down' %}a monitor is down.{% elif p.state == 'stale' %}its agent stopped reporting.{% else %}{{ p.worst[0] }} high.{% endif %} + {%- else %}{{ problems|length }} of {{ rows|length }} hosts need a look.{% endif -%}
+ + + + + {% for r in rows %} + {% set a = r.agent %}{% set fresh = a.get('fresh') %} + + + {% for v, kind in ((a.get('cpu_pct'), 'cpu'), (a.get('mem_used_pct'), 'mem'), (a.get('disk_root'), 'disk')) %} + + {% endfor %} + + + {% endfor %} + +
HostCPUMemDisk /Load
+ {{- status_dot(r.state) -}} + {% if session.user_id %}{{ r.host.name }}{% else %}{{ r.host.name }}{% endif %} + {%- if r.state == 'down' %}a monitor is down + {%- elif r.state == 'stale' %}agent last reported {{ a.last_at|ago }}; figures are from then + {%- elif not a %}no agent{% endif %} + {% if v is none %}—{% else %}{{ "%.0f"|format(v) }}%{% endif %} + {%- if a.get('load_1m') is none %}—{% elif a.get('cores') %}{{ "%.1f"|format(a.load_1m) }} / {{ a.cores }} cores{% else %}{{ "%.1f"|format(a.load_1m) }}{% endif -%} +
{% endif %} diff --git a/tests/core/test_dashboard_defaults.py b/tests/core/test_dashboard_defaults.py index 66afb01..9a2cdb5 100644 --- a/tests/core/test_dashboard_defaults.py +++ b/tests/core/test_dashboard_defaults.py @@ -84,13 +84,59 @@ async def test_a_quiet_overview_says_so(app): assert ">host<" in page and 'href="/alerts/"' not in page # the share view has no links -async def test_the_hosts_widget_shows_load(app): - host = SimpleNamespace(id="h1", name="web-1") - agent = {"web-1": {"cpu_pct": 12.0, "mem_used_pct": 40.0, "load_1m": 0.734, "fresh": True}} +def _host(hid, name): + return SimpleNamespace(id=hid, name=name) + + +def test_hosts_sort_worst_first(): + """#5251: a down monitor, then a reading past critical, then a quiet agent, + then a warning, then the healthy, by name; a host with nothing last.""" + from steward.core.settings import to_thresholds_cfg + from steward.hosts.service import host_glance + th = to_thresholds_cfg({}) + hosts = [_host("a", "alpha"), _host("b", "busy"), _host("d", "dead"), _host("q", "quiet"), + _host("w", "warm"), _host("n", "nothing")] + status = {"d": {"state": "down"}, "a": {"state": "up"}} + agent = {"alpha": {"cpu_pct": 10.0, "fresh": True}, "busy": {"cpu_pct": 99.0, "fresh": True}, + "quiet": {"cpu_pct": 99.0, "fresh": False}, + "warm": {"cpu_pct": 10.0, "load_per_core": 85, "fresh": True}} + rows = host_glance(hosts, status, agent, th) + assert [(r["host"].name, r["state"]) for r in rows] == [ + ("dead", "down"), ("busy", "crit"), ("quiet", "stale"), ("warm", "warn"), + ("alpha", "up"), ("nothing", "unknown")] + assert rows[1]["worst"][0] == "CPU" and rows[3]["worst"][0] == "Load" + assert rows[2]["worst"] is None # a quiet agent's old figures aren't graded + + +async def test_the_hosts_widget_gives_a_verdict_and_load_against_cores(app): + from datetime import datetime, timezone + rows = [ + {"host": _host("h2", "VDNT-Util01"), "state": "crit", "worst": ("CPU", 99.0, "crit"), + "agent": {"cpu_pct": 99.0, "mem_used_pct": 28.0, "disk_root": 52.0, "load_1m": 16.77, + "cores": 8, "load_per_core": 210, "fresh": True}}, + {"host": _host("h1", "VDNT-Docker02"), "state": "up", "worst": None, + "agent": {"cpu_pct": 1.0, "mem_used_pct": 4.0, "disk_root": 44.0, "load_1m": 0.92, + "cores": 4, "load_per_core": 23, "fresh": True}}, + {"host": _host("h3", "nas"), "state": "stale", "worst": None, + "agent": {"cpu_pct": 5.0, "fresh": False, "last_at": datetime(2026, 10, 7, tzinfo=timezone.utc)}}, + ] async with app.test_request_context("/hosts/overview/widget"): - page = await render_template("hosts/overview_widget.html", hosts=[host], host_status={}, - uptime={}, agent=agent, cpu_sparks={}) - assert ">load 0.73" in page + session["user_id"] = "u1" + page = await render_template("hosts/overview_widget.html", rows=rows) + assert "2 of 3 hosts need a look." in page + assert page.index("VDNT-Util01") < page.index("VDNT-Docker02") + assert "16.8 / 8 cores" in page and "0.9 / 4 cores" in page + assert "agent last reported" in page + assert page.count("") == 5 and ">ping<" not in page and "24h" not in page + assert not hex_colours(page), hex_colours(page) + + +async def test_one_busy_host_is_named_in_the_verdict(app): + rows = [{"host": _host("h2", "web-1"), "state": "crit", "worst": ("CPU", 99.0, "crit"), + "agent": {"cpu_pct": 99.0, "fresh": True}}] + async with app.test_request_context("/hosts/overview/widget"): + page = await render_template("hosts/overview_widget.html", rows=rows) + assert "web-1:" in page and "CPU high." in page and "tone-crit" in page async def test_a_widget_polls_at_its_own_pace(app): diff --git a/tests/integration/test_metric_latest.py b/tests/integration/test_metric_latest.py index 759a72d..24e4249 100644 --- a/tests/integration/test_metric_latest.py +++ b/tests/integration/test_metric_latest.py @@ -98,6 +98,29 @@ def test_a_quiet_host_keeps_its_last_values_and_reads_stale_not_missing(app): assert "never-" + quiet not in overview # no samples at all: absent + +def test_load_is_read_against_the_hosts_cores(app): + """#5251: cores are counted from the per-core CPU sub-resources, and load + per core is load_1m as a % of them, the figure "load" thresholds grade.""" + from steward.core.alerts import record_metrics + from steward.hosts.service import agent_overview_by_host + + now = datetime.now(timezone.utc) + host = "cores-" + uuid.uuid4().hex[:8] + + async def go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + await record_metrics(s, [ + _sample(host, "load_1m", 3.0, now), + _sample(host + ":core0", "cpu_pct", 80.0, now), + _sample(host + ":core1", "cpu_pct", 90.0, now), + ]) + return await agent_overview_by_host(s, [host], 180) + + h = asyncio.run(go())[host] + assert (h["cores"], h["load_per_core"]) == (2, 150) + def test_retention_drops_a_latest_value_with_its_raw_history(app): from steward.core.alerts import record_metrics from steward.core.metrics_retention import rollup_plugin_metrics