feat(dashboard): Hosts widget worst first, load against cores, one home per fact (#5251)

The Hosts — Overview widget opens with a verdict ("VDNT-Util01: CPU high."
or "All 3 hosts healthy."), then one row per host under one header: CPU,
memory, disk / and load. Rows sort by host_glance: a down monitor, a
reading past critical, a quiet agent, a warning, then the healthy. Load
reads "16.8 / 8 cores", graded per core as Settings → Thresholds intends
(cores counted from the per-core CPU sub-resources, as on the host page).
A quiet agent's last figures are shown uncoloured, with when it last reported.

Ping and 24h uptime are gone from this widget: they belong to Monitors. So
are the unlabelled CPU sparkline and the 30-day uptime query it no longer
needs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/hosts/routes.py

@@ -27,21 +27,15 @@ def _ansible_source_names() -> list[str]:
 @hosts_bp.get("/overview/widget")
 @require_role(UserRole.viewer)
 async def overview_widget():
-    """Dashboard widget: unified per-host monitor + agent glance, linking to the hub."""
+    """Dashboard widget: each host's resources, worst first (#5251). Uptime and
+    latency belong to the Monitors widget, so this one doesn't read them."""
     async with current_app.db_sessionmaker() as db:
         hosts = (await db.execute(select(Host).order_by(Host.name))).scalars().all()
         host_status = await host_svc.host_status(db, [h.id for h in hosts])
-        uptime = await host_svc.compute_uptime(db)
-        host_names = [h.name for h in hosts]
         agent = await host_svc.agent_overview_by_host(
-            db, host_names, host_svc.agent_stale_after_seconds(current_app.config))
-        cpu_sparks = await host_svc.agent_cpu_sparks_by_host(db, host_names)
-
-    return await render_template(
-        "hosts/overview_widget.html",
-        hosts=hosts, host_status=host_status,
-        uptime=uptime, agent=agent, cpu_sparks=cpu_sparks,
-    )
+            db, [h.name for h in hosts], host_svc.agent_stale_after_seconds(current_app.config))
+    rows = host_svc.host_glance(hosts, host_status, agent, current_app.config.get("THRESHOLDS", {}))
+    return await render_template("hosts/overview_widget.html", rows=rows)
 
 
 @hosts_bp.get("/")

steward/hosts/service.py

@@ -170,10 +170,13 @@ async def host_monitors(db, host_id: str) -> list[dict]:
 async def agent_overview_by_host(
     db, host_names: list[str], stale_after_seconds: int = 180,
 ) -> dict[str, dict]:
-    """Per-host agent glance (cpu/mem/disk + freshness) from plugin_metric_latest.
+    """Per-host agent glance (cpu/mem/disk/load + freshness) from plugin_metric_latest.
 
     {host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root, load_1m,
-                 last_at, fresh}} for each host with any agent samples.
+                 cores, load_per_core, last_at, fresh}} for each host with any
+    agent samples. Cores are counted from the per-core CPU sub-resources
+    ("<host>:core<N>"), as on the host page; load_per_core is load_1m as a %
+    of them, the figure Settings → Thresholds grades "load" by.
 
     Core-safe (no host_agent import). Freshness compares the latest host-level
     sample against the host_agent plugin's stale window (stale_after_seconds;
@@ -188,6 +191,16 @@ async def agent_overview_by_host(
     latest = await latest_by_resource(
         db, "host_agent", list(host_names) + [h + ":/" for h in host_names],
         set(AGENT_GLANCE_METRICS) | {"disk_used_pct"})
+    from steward.models.metrics import PluginMetricLatest
+    cores: dict[str, int] = {}
+    for res in (await db.execute(
+        select(PluginMetricLatest.resource_name).where(
+            PluginMetricLatest.source_module == "host_agent",
+            PluginMetricLatest.metric_name == "cpu_pct",
+            PluginMetricLatest.resource_name.like("%:core%"))
+    )).scalars():
+        host = res.rsplit(":core", 1)[0]
+        cores[host] = cores.get(host, 0) + 1
     out: dict[str, dict] = {}
     for name in host_names:
         own = {m: r for m, r in latest.get(name, {}).items() if m in AGENT_GLANCE_METRICS}
@@ -200,35 +213,44 @@ async def agent_overview_by_host(
         ts = max((r.recorded_at for r in own.values()), default=None)
         d["last_at"] = ts
         d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
+        d["cores"] = cores.get(name)
+        load = d.get("load_1m")
+        d["load_per_core"] = round(load / d["cores"] * 100) if (load is not None and d["cores"]) else None
         out[name] = d
     return out
 
 
-async def agent_cpu_sparks_by_host(db, host_names: list[str], hours: int = 1) -> dict[str, str]:
-    """{host_name: inline-SVG cpu sparkline} over the last `hours` (core-safe).
+GLANCE_ORDER = {"down": 0, "crit": 1, "stale": 2, "warn": 3, "up": 4, "unknown": 5}
 
-    A tiny at-a-glance trend in each row — the host-view sparkline, on the widget.
-    """
-    from steward.models.metrics import PluginMetric
-    from steward.core.status import sparkline_svg
-    if not host_names:
-        return {}
-    cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
-    rows = (await db.execute(
-        select(PluginMetric.resource_name, PluginMetric.value, PluginMetric.recorded_at)
-        .where(
-            PluginMetric.source_module == "host_agent",
-            PluginMetric.metric_name == "cpu_pct",
-            PluginMetric.resource_name.in_(host_names),
-            PluginMetric.recorded_at >= cutoff,
-        )
-        .order_by(PluginMetric.recorded_at)
-    )).all()
-    by_host: dict[str, list[float]] = {}
-    for r in rows:
-        by_host.setdefault(r.resource_name, []).append(r.value)
-    # Amber line to read as "CPU"; only draw when there are a couple of points.
-    return {
-        name: sparkline_svg(vals[-40:], width=64, height=18, stroke="#c8a840")
-        for name, vals in by_host.items() if len(vals) >= 2
-    }
+
+def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict]:
+    """The Hosts — Overview rows, worst first (#5251).
+
+    Each is {host, state, worst, agent}. State is "down" when a linked monitor
+    is down, "stale" when the agent has stopped reporting, else the grade of
+    the reading furthest past its cutoff ("crit" / "warn": CPU, memory, disk /
+    or load per core, against Settings → Thresholds), else "up"; "unknown" for
+    a host with neither monitors nor agent. `worst` is (label, value, grade)
+    or None, from formatting.worst_reading."""
+    from steward.core.formatting import worst_reading
+    rows = []
+    for h in hosts:
+        st, a = status.get(h.id), agent.get(h.name)
+        worst = None
+        if a and a.get("fresh"):
+            worst = worst_reading((
+                ("CPU", a.get("cpu_pct"), "cpu"), ("Memory", a.get("mem_used_pct"), "mem"),
+                ("Disk /", a.get("disk_root"), "disk"), ("Load", a.get("load_per_core"), "load")),
+                thresholds)
+        if st and st["state"] == "down":
+            state = "down"
+        elif a and not a.get("fresh"):
+            state = "stale"
+        elif worst:
+            state = worst[2]
+        elif st or a:
+            state = "up"
+        else:
+            state = "unknown"
+        rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}})
+    return sorted(rows, key=lambda r: (GLANCE_ORDER[r["state"]], r["host"].name.lower()))

steward/templates/hosts/overview_widget.html

@@ -1,50 +1,43 @@
-{# Unified Hosts widget — monitor status + agent glance per host, links to the hub.
-   Name sits on its own line (never truncated); its data wraps beneath it, grouped
-   under the host. Prefer vertical growth over cramming a row. #}
-{% if hosts %}
-<div class="host-blocks">
-  {% for host in hosts %}
-  {% set hs = host_status.get(host.id) %}
-  {% set ut = uptime.get(host.id, {}) %}
-  {% set a = agent.get(host.name, {}) %}
-  {% set down = hs and hs.state == 'down' %}
-  {% set spark = cpu_sparks.get(host.name) %}
-  <div style="padding:0.5rem 0;border-bottom:1px solid var(--border);">
-    {# Line 1 — status dot + full host name (own line) + cpu trend #}
-    <div style="display:flex;align-items:center;gap:0.5rem;">
-      <span class="dot {% if down %}dot-down{% elif hs and hs.state == 'up' %}dot-up{% else %}dot-dim{% endif %}"
-            style="flex-shrink:0;"
-            title="{% if not hs %}no monitors{% else %}{{ hs.state }}{% endif %}"></span>
-      {% if session.user_id %}
-      <a href="/hosts/{{ host.id }}" style="font-weight:500;font-size:0.85rem;overflow-wrap:anywhere;">{{ host.name }}</a>
-      {% else %}
-      <span style="font-weight:500;font-size:0.85rem;overflow-wrap:anywhere;">{{ host.name }}</span>
-      {% endif %}
-    </div>
-    {# Line 2 — monitors + agent metrics; wraps instead of truncating #}
-    <div style="display:flex;flex-wrap:wrap;gap:0.25rem 0.9rem;margin-top:0.25rem;padding-left:1.1rem;
-                font-variant-numeric:tabular-nums;color:var(--text-muted);font-size:0.75rem;">
-      {% if hs and hs.latency_ms is not none %}
-      <span title="Latency" style="{{ threshold_style(hs.latency_ms, 'latency') }}"><span style="color:var(--text-dim);">ping</span> {{ "%.0f"|format(hs.latency_ms) }}ms</span>
-      {% endif %}
-      {% if ut.get('24h') is not none %}
-      <span title="Uptime 24h" style="{{ threshold_style(ut['24h'], 'uptime') }}"><span style="color:var(--text-dim);">24h</span> {{ "%.1f"|format(ut['24h']) }}%</span>
-      {% endif %}
-      {% if a %}
-      {% if a.get('cpu_pct') is not none %}<span title="CPU — value + last-hour trend" style="display:inline-flex;align-items:center;gap:0.3rem;{{ threshold_style(a.cpu_pct, 'cpu') }}"><span style="color:var(--text-dim);">cpu</span> {{ "%.0f"|format(a.cpu_pct) }}%{% if spark %}<span style="line-height:0;opacity:0.85;">{{ spark | safe }}</span>{% endif %}</span>{% endif %}
-      {% if a.get('mem_used_pct') is not none %}<span title="Memory" style="{{ threshold_style(a.mem_used_pct, 'mem') }}"><span style="color:var(--text-dim);">mem</span> {{ "%.0f"|format(a.mem_used_pct) }}%</span>{% endif %}
-      {% if a.get('disk_root') is not none %}<span title="Root filesystem (/)" style="{{ threshold_style(a.disk_root, 'disk') }}"><span style="color:var(--text-dim);">disk /</span> {{ "%.0f"|format(a.disk_root) }}%</span>{% endif %}
-      {% if a.get('load_1m') is not none %}<span title="Load average over 1 minute: processes running or waiting to run"><span style="color:var(--text-dim);">load</span> {{ "%.2f"|format(a.load_1m) }}</span>{% endif %}
-      {% if not a.fresh %}<span title="Agent data is stale" style="color:var(--yellow);">stale</span>{% endif %}
-      {% else %}
-      <span style="color:var(--text-dim);" title="No agent reporting">no agent</span>
-      {% endif %}
-    </div>
-  </div>
-  {% endfor %}
+{# hosts/overview_widget.html — the dashboard Hosts widget (#5251): a verdict,
+   then one row per host, worst first, with its resources under one header.
+   Load is shown against the host's cores and graded per core. Reachability
+   and uptime belong to the Monitors widget, so they aren't repeated here. #}
+{% from "_macros.html" import status_dot %}
+{% if not rows %}
+<div class="w-verdict" style="color:var(--text-muted);">
+  No hosts yet.{% if session.user_id %} <a href="/hosts/new">Add one →</a>{% endif %}
 </div>
 {% else %}
-<div class="empty" style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">
-  No hosts yet. <a href="/hosts/new">Add one.</a>
+{% set problems = rows | selectattr("state", "in", ["down", "crit", "stale", "warn"]) | list %}
+<div class="w-verdict {{ 'tone-crit' if problems and problems[0].state in ['down', 'crit'] else ('tone-warn' if problems else 'tone-ok') }}">
+  {%- if not problems %}All {{ rows|length }} host{{ 's' if rows|length != 1 }} healthy.
+  {%- elif problems|length == 1 %}{% set p = problems[0] %}{{ p.host.name }}:
+    {% if p.state == 'down' %}a monitor is down.{% elif p.state == 'stale' %}its agent stopped reporting.{% else %}{{ p.worst[0] }} high.{% endif %}
+  {%- else %}{{ problems|length }} of {{ rows|length }} hosts need a look.{% endif -%}
 </div>
+<table class="w-table">
+  <colgroup><col><col style="width:3.6rem"><col style="width:3.6rem"><col style="width:3.6rem"><col style="width:6.5rem"></colgroup>
+  <thead><tr><th>Host</th><th>CPU</th><th>Mem</th><th>Disk /</th><th>Load</th></tr></thead>
+  <tbody>
+  {% for r in rows %}
+  {% set a = r.agent %}{% set fresh = a.get('fresh') %}
+  <tr>
+    <td class="w-name" title="{{ r.host.name }}">
+      {{- status_dot(r.state) -}}
+      {% if session.user_id %}<a href="/hosts/{{ r.host.id }}">{{ r.host.name }}</a>{% else %}{{ r.host.name }}{% endif %}
+      {%- if r.state == 'down' %}<span class="w-sub tone-crit">a monitor is down</span>
+      {%- elif r.state == 'stale' %}<span class="w-sub tone-warn">agent last reported {{ a.last_at|ago }}; figures are from then</span>
+      {%- elif not a %}<span class="w-sub">no agent</span>{% endif %}
+    </td>
+    {% for v, kind in ((a.get('cpu_pct'), 'cpu'), (a.get('mem_used_pct'), 'mem'), (a.get('disk_root'), 'disk')) %}
+    <td style="{{ threshold_style(v, kind) if fresh and v is not none }}">{% if v is none %}—{% else %}{{ "%.0f"|format(v) }}%{% endif %}</td>
+    {% endfor %}
+    <td style="{{ threshold_style(a.get('load_per_core'), 'load') if fresh and a.get('load_per_core') is not none }}"
+        title="{% if a.get('cores') %}Load over 1 minute on {{ a.cores }} cores: {{ a.load_per_core }}% of them busy or waiting{% endif %}">
+      {%- if a.get('load_1m') is none %}—{% elif a.get('cores') %}{{ "%.1f"|format(a.load_1m) }} / {{ a.cores }} cores{% else %}{{ "%.1f"|format(a.load_1m) }}{% endif -%}
+    </td>
+  </tr>
+  {% endfor %}
+  </tbody>
+</table>
 {% endif %}

tests/core/test_dashboard_defaults.py

@@ -84,13 +84,59 @@ async def test_a_quiet_overview_says_so(app):
     assert ">host<" in page and 'href="/alerts/"' not in page   # the share view has no links
 
 
-async def test_the_hosts_widget_shows_load(app):
-    host = SimpleNamespace(id="h1", name="web-1")
-    agent = {"web-1": {"cpu_pct": 12.0, "mem_used_pct": 40.0, "load_1m": 0.734, "fresh": True}}
+def _host(hid, name):
+    return SimpleNamespace(id=hid, name=name)
+
+
+def test_hosts_sort_worst_first():
+    """#5251: a down monitor, then a reading past critical, then a quiet agent,
+    then a warning, then the healthy, by name; a host with nothing last."""
+    from steward.core.settings import to_thresholds_cfg
+    from steward.hosts.service import host_glance
+    th = to_thresholds_cfg({})
+    hosts = [_host("a", "alpha"), _host("b", "busy"), _host("d", "dead"), _host("q", "quiet"),
+             _host("w", "warm"), _host("n", "nothing")]
+    status = {"d": {"state": "down"}, "a": {"state": "up"}}
+    agent = {"alpha": {"cpu_pct": 10.0, "fresh": True}, "busy": {"cpu_pct": 99.0, "fresh": True},
+             "quiet": {"cpu_pct": 99.0, "fresh": False},
+             "warm": {"cpu_pct": 10.0, "load_per_core": 85, "fresh": True}}
+    rows = host_glance(hosts, status, agent, th)
+    assert [(r["host"].name, r["state"]) for r in rows] == [
+        ("dead", "down"), ("busy", "crit"), ("quiet", "stale"), ("warm", "warn"),
+        ("alpha", "up"), ("nothing", "unknown")]
+    assert rows[1]["worst"][0] == "CPU" and rows[3]["worst"][0] == "Load"
+    assert rows[2]["worst"] is None          # a quiet agent's old figures aren't graded
+
+
+async def test_the_hosts_widget_gives_a_verdict_and_load_against_cores(app):
+    from datetime import datetime, timezone
+    rows = [
+        {"host": _host("h2", "VDNT-Util01"), "state": "crit", "worst": ("CPU", 99.0, "crit"),
+         "agent": {"cpu_pct": 99.0, "mem_used_pct": 28.0, "disk_root": 52.0, "load_1m": 16.77,
+                   "cores": 8, "load_per_core": 210, "fresh": True}},
+        {"host": _host("h1", "VDNT-Docker02"), "state": "up", "worst": None,
+         "agent": {"cpu_pct": 1.0, "mem_used_pct": 4.0, "disk_root": 44.0, "load_1m": 0.92,
+                   "cores": 4, "load_per_core": 23, "fresh": True}},
+        {"host": _host("h3", "nas"), "state": "stale", "worst": None,
+         "agent": {"cpu_pct": 5.0, "fresh": False, "last_at": datetime(2026, 10, 7, tzinfo=timezone.utc)}},
+    ]
     async with app.test_request_context("/hosts/overview/widget"):
-        page = await render_template("hosts/overview_widget.html", hosts=[host], host_status={},
-                                     uptime={}, agent=agent, cpu_sparks={})
-    assert ">load</span> 0.73" in page
+        session["user_id"] = "u1"
+        page = await render_template("hosts/overview_widget.html", rows=rows)
+    assert "2 of 3 hosts need a look." in page
+    assert page.index("VDNT-Util01") < page.index("VDNT-Docker02")
+    assert "16.8 / 8 cores" in page and "0.9 / 4 cores" in page
+    assert "agent last reported" in page
+    assert page.count("<th>") == 5 and ">ping<" not in page and "24h" not in page
+    assert not hex_colours(page), hex_colours(page)
+
+
+async def test_one_busy_host_is_named_in_the_verdict(app):
+    rows = [{"host": _host("h2", "web-1"), "state": "crit", "worst": ("CPU", 99.0, "crit"),
+             "agent": {"cpu_pct": 99.0, "fresh": True}}]
+    async with app.test_request_context("/hosts/overview/widget"):
+        page = await render_template("hosts/overview_widget.html", rows=rows)
+    assert "web-1:" in page and "CPU high." in page and "tone-crit" in page
 
 
 async def test_a_widget_polls_at_its_own_pace(app):

tests/integration/test_metric_latest.py

@@ -98,6 +98,29 @@ def test_a_quiet_host_keeps_its_last_values_and_reads_stale_not_missing(app):
     assert "never-" + quiet not in overview          # no samples at all: absent
 
 
+
+def test_load_is_read_against_the_hosts_cores(app):
+    """#5251: cores are counted from the per-core CPU sub-resources, and load
+    per core is load_1m as a % of them, the figure "load" thresholds grade."""
+    from steward.core.alerts import record_metrics
+    from steward.hosts.service import agent_overview_by_host
+
+    now = datetime.now(timezone.utc)
+    host = "cores-" + uuid.uuid4().hex[:8]
+
+    async def go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await record_metrics(s, [
+                    _sample(host, "load_1m", 3.0, now),
+                    _sample(host + ":core0", "cpu_pct", 80.0, now),
+                    _sample(host + ":core1", "cpu_pct", 90.0, now),
+                ])
+            return await agent_overview_by_host(s, [host], 180)
+
+    h = asyncio.run(go())[host]
+    assert (h["cores"], h["load_per_core"]) == (2, 150)
+
 def test_retention_drops_a_latest_value_with_its_raw_history(app):
     from steward.core.alerts import record_metrics
     from steward.core.metrics_retention import rollup_plugin_metrics