feat(widgets): Overview verdict, pending monitors' age, fullest filesystems first (#5810)
- Overview: a verdict line when something is wrong ('1 alert firing, 1
monitor down.'); its picker description now says what it lists (alerts,
monitors down, anomalies). Host figures have their home in Hosts.
Comments no longer name the Status page that was folded into Monitors.
- Monitors widget: a monitor not checked yet says when it was added.
- Host page: filesystems fullest first, by the worse of space and inodes.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -864,6 +864,10 @@ def _split_host_metrics(host_name: str, latest: dict) -> tuple:
|
||||
else:
|
||||
mounts[suffix] = metrics
|
||||
cores.sort(key=lambda c: c[0])
|
||||
# Fullest first, by the worse of space and inodes (#5810): a full
|
||||
# /var/lib/docker must not sort after /boot.
|
||||
mounts = dict(sorted(mounts.items(), key=lambda kv: (
|
||||
-max(kv[1].get("disk_used_pct") or 0, kv[1].get("inodes_used_pct") or 0), kv[0])))
|
||||
return hostlvl, cores, nets, disks_io, temps, mounts
|
||||
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# steward/core/status.py
|
||||
"""Unified monitor-status aggregation.
|
||||
|
||||
A single readable "is everything up?" surface (the Status page + dashboard
|
||||
A single readable "is everything up?" surface (the dashboard Overview
|
||||
widget) is assembled from heterogeneous monitors. The core unified Monitor
|
||||
entity (ping/dns/http) contributes via `monitor_status_source`; plugins may
|
||||
register their own sources from setup(). Each source is an async callable(db)
|
||||
returning normalised StatusEntry objects so the Status page never imports
|
||||
returning normalised StatusEntry objects so the Overview never imports
|
||||
plugin tables directly.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
@@ -62,7 +62,7 @@ async def collect_status(db) -> list[StatusEntry]:
|
||||
"""Gather entries from every registered source, down-first then by name.
|
||||
|
||||
A failing source is logged and skipped so one broken plugin can't blank
|
||||
the whole Status page.
|
||||
the whole Overview.
|
||||
"""
|
||||
entries: list[StatusEntry] = []
|
||||
for src in list(_SOURCES):
|
||||
@@ -210,7 +210,7 @@ def _heartbeat_title(mtype: str, r) -> str:
|
||||
|
||||
|
||||
async def monitor_status_source(db) -> list[StatusEntry]:
|
||||
"""Contribute every enabled Monitor (all types) to the Status page."""
|
||||
"""Contribute every enabled Monitor (all types) to the Overview."""
|
||||
from steward.models.monitors import Monitor
|
||||
|
||||
monitors = list((await db.execute(
|
||||
|
||||
@@ -65,7 +65,7 @@ WIDGET_REGISTRY: dict[str, dict] = {
|
||||
"status_overview": {
|
||||
"key": "status_overview",
|
||||
"label": "Overview",
|
||||
"description": "What needs attention across Steward: hosts, monitors and firing alerts, problems listed with since-times",
|
||||
"description": "What needs attention across Steward: firing alerts, monitors down and metric anomalies, each with since when",
|
||||
"hx_url": "/status/widget",
|
||||
"detail_url": "/monitors/",
|
||||
"plugin": None,
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
{% if session.user_id %}<a href="/monitors/{{ m.id }}">{{ m.name }}</a>{% else %}{{ m.name }}{% endif %}
|
||||
{%- if d.state == 'down' %}<span class="w-sub tone-crit">went down {{ d.down_since|ago if d.down_since else '' }}{% if d.down_reason %} · {{ d.down_reason }}{% endif %}</span>
|
||||
{%- elif d.state == 'degraded' %}<span class="w-sub tone-warn">{% if (d.latency or status_class(latest.response_ms, 'latency') if latest else none) == 'crit' %}slow replies{% else %}checks failing now and then{% endif %}</span>
|
||||
{%- elif d.state == 'pending' %}<span class="w-sub">not checked yet</span>{% endif %}
|
||||
{%- elif d.state == 'pending' %}<span class="w-sub">not checked yet{% if m.created_at %} · added {{ m.created_at|ago }}{% endif %}</span>{% endif %}
|
||||
</td>
|
||||
{% set lat = (d.latency or status_class(latest.response_ms, 'latency')) if latest and latest.is_up and latest.response_ms is not none else none %}
|
||||
<td class="{{ 'tone-' ~ lat if lat in ('warn', 'crit') }}"{% if d.latency_usual %} title="{{ d.latency_usual|capitalize }}"{% endif %}>
|
||||
|
||||
@@ -30,6 +30,13 @@
|
||||
{% if entries %}✓ All {{ up }} monitors are up and no alert is firing.{% else %}No monitors configured, and no alert is firing.{% endif %}
|
||||
</div>
|
||||
{% else %}
|
||||
{# The verdict when something is wrong (#5810): what, in counts, worst first. #}
|
||||
{% set said = [] %}
|
||||
{% if firing %}{% set _ = said.append(firing|length ~ " alert" ~ ('' if firing|length == 1 else 's') ~ " firing") %}{% endif %}
|
||||
{% if down %}{% set _ = said.append(down ~ " monitor" ~ ('' if down == 1 else 's') ~ " down") %}{% endif %}
|
||||
{% if anomalies %}{% set _ = said.append(anomalies|length ~ " metric" ~ ('' if anomalies|length == 1 else 's') ~ " unusual") %}{% endif %}
|
||||
{% if pending %}{% set _ = said.append(pending ~ " monitor" ~ ('' if pending == 1 else 's') ~ " waiting for a first check") %}{% endif %}
|
||||
{% if said %}<div class="w-verdict {{ 'tone-crit' if firing or down else 'tone-warn' }}">{{ said|join(', ')|capitalize }}.</div>{% endif %}
|
||||
<div style="display:grid;gap:1px;">
|
||||
{% for a in alerts[:6] %}
|
||||
<div style="display:flex;align-items:center;gap:0.5rem;padding:0.2rem 0;font-size:0.82rem;">
|
||||
|
||||
@@ -77,6 +77,9 @@ async def test_the_overview_labels_its_counts_and_lists_what_is_wrong(app):
|
||||
assert '<span class="badge badge-crit">Down</span>' in page and "2h ago" in page
|
||||
assert page.index("nas disk full") < page.index("ssh-nas") # alerts, then monitors
|
||||
assert ">pending<" not in page # a zero pending count is not shown
|
||||
# #5810: a verdict says what is wrong before the list says where.
|
||||
assert '<div class="w-verdict tone-crit">1 alert firing, 1 monitor down.</div>' in page
|
||||
assert page.index("1 alert firing, 1 monitor down.") < page.index("nas disk full")
|
||||
assert not hex_colours(page), hex_colours(page)
|
||||
|
||||
|
||||
|
||||
@@ -181,3 +181,14 @@ async def test_the_widget_says_when_all_is_well(app):
|
||||
up = {**_row(_mon("web", mid="m1"), "up", _check()), "uptime": {"24h": 100.0}}
|
||||
page = await _widget(app, [], [up])
|
||||
assert "All 1 monitor up." in page and "tone-ok" in page
|
||||
|
||||
|
||||
async def test_a_pending_monitor_says_when_it_was_added(app):
|
||||
"""#5810: a monitor not yet checked says how long it has waited."""
|
||||
m = _mon("new-api", mid="m7")
|
||||
m.created_at = datetime.now(timezone.utc) - timedelta(minutes=6)
|
||||
pending = {**_row(m, "pending", None), "uptime": {}}
|
||||
page = await _widget(app, [pending], [], counts={"up": 0, "degraded": 0, "down": 0,
|
||||
"pending": 1, "paused": 0})
|
||||
page = " ".join(page.split())
|
||||
assert "not checked yet · added <time" in page and "6m ago" in page
|
||||
|
||||
@@ -167,3 +167,17 @@ def test_the_page_lists_unwired_sensors_apart_hottest_first():
|
||||
assert list(real) == ["Package id 0", "Composite", "CPUTIN", "gone"]
|
||||
assert list(unwired) == ["SYSTIN", "AUXTIN0", "PCH_CHIP_TEMP"]
|
||||
assert unwired["SYSTIN"] == (115.0, "reads a value no wired sensor gives")
|
||||
|
||||
|
||||
def test_the_host_page_lists_filesystems_fullest_first():
|
||||
"""#5810: by the worse of space and inodes, so a full /var/lib/docker
|
||||
doesn't sort after /boot; ties by name."""
|
||||
from plugins.host_agent.routes import _split_host_metrics
|
||||
latest = {"h": {"cpu_pct": 1.0},
|
||||
"h:/": {"disk_used_pct": 40.0, "inodes_used_pct": 5.0},
|
||||
"h:/boot": {"disk_used_pct": 30.0},
|
||||
"h:/var/lib/docker": {"disk_used_pct": 91.0},
|
||||
"h:/srv": {"disk_used_pct": 20.0, "inodes_used_pct": 97.0},
|
||||
"h:/data": {"disk_used_pct": 30.0}}
|
||||
*_, mounts = _split_host_metrics("h", latest)
|
||||
assert list(mounts) == ["/srv", "/var/lib/docker", "/", "/boot", "/data"]
|
||||
|
||||
Reference in New Issue
Block a user