Compare commits
2
Commits
4dcf958022
...
17abe450cc
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
17abe450cc | ||
|
|
b5fcccba5f |
+3
-15
@@ -18,7 +18,7 @@ WIDGET_REGISTRY: dict[str, dict] = {
|
||||
"monitors": {
|
||||
"key": "monitors",
|
||||
"label": "Monitors",
|
||||
"description": "Uptime checks of every type — ping, DNS, and HTTP — with up/down status and latency",
|
||||
"description": "Every uptime check, problems first: latency and uptime over 24h, 7d and 30d",
|
||||
"hx_url": "/monitors/widget",
|
||||
"detail_url": "/monitors/",
|
||||
"plugin": None,
|
||||
@@ -80,18 +80,6 @@ WIDGET_REGISTRY: dict[str, dict] = {
|
||||
"poll": True,
|
||||
"params": [],
|
||||
},
|
||||
"uptime_summary": {
|
||||
"key": "uptime_summary",
|
||||
"label": "Uptime / SLA",
|
||||
"description": "Per-host rolling uptime % for 24h, 7d, and 30d windows",
|
||||
"hx_url": "/hosts/uptime/widget",
|
||||
"detail_url": "/hosts/uptime",
|
||||
"plugin": None,
|
||||
"poll": True,
|
||||
# Uptime moves slowly and reads 30 days of checks: every 5 minutes, not every poll.
|
||||
"poll_seconds": 300,
|
||||
"params": [],
|
||||
},
|
||||
"traefik_routers": {
|
||||
"key": "traefik_routers",
|
||||
"label": "Traefik — Routers",
|
||||
@@ -278,6 +266,7 @@ WIDGET_REGISTRY: dict[str, dict] = {
|
||||
RETIRED = {
|
||||
"host_resources": "hosts_overview", # Hosts — Overview carries cpu/mem/disk/load per host
|
||||
"unifi_devices": "unifi_overview", # UniFi — Overview lists the offline devices
|
||||
"uptime_summary": "monitors", # Monitors shows 24h/7d/30d uptime per monitor (#5250)
|
||||
}
|
||||
|
||||
# The recommended dashboard (#5213): what needs attention first, then the
|
||||
@@ -291,7 +280,6 @@ RECOMMENDED: list[tuple[str, int, int, dict]] = [
|
||||
("monitors", 6, 7, {"show_down_only": "no", "limit": 10}),
|
||||
("docker_containers", 6, 6, {}),
|
||||
("docker_resources", 6, 6, {"limit": 10}),
|
||||
("uptime_summary", 6, 5, {}),
|
||||
("traefik_routers", 6, 5, {}),
|
||||
("unifi_overview", 6, 4, {}),
|
||||
("snmp_devices", 6, 4, {}),
|
||||
@@ -318,7 +306,7 @@ def recommended_layout(plugins_cfg: dict) -> list[dict]:
|
||||
# Group each widget for the picker (Core monitors / Monitoring capabilities /
|
||||
# Integrations), mirroring the Settings → Plugins taxonomy. Capability plugins
|
||||
# are host/monitoring facets; traefik/unifi are discrete vendor integrations.
|
||||
_CORE_WIDGETS = {"monitors", "status_overview", "uptime_summary", "hosts_overview"}
|
||||
_CORE_WIDGETS = {"monitors", "status_overview", "hosts_overview"}
|
||||
_INTEGRATION_PLUGINS = {"traefik", "unifi"}
|
||||
GROUP_LABELS = {
|
||||
"core": "Core monitors",
|
||||
|
||||
+5
-25
@@ -27,21 +27,15 @@ def _ansible_source_names() -> list[str]:
|
||||
@hosts_bp.get("/overview/widget")
|
||||
@require_role(UserRole.viewer)
|
||||
async def overview_widget():
|
||||
"""Dashboard widget: unified per-host monitor + agent glance, linking to the hub."""
|
||||
"""Dashboard widget: each host's resources, worst first (#5251). Uptime and
|
||||
latency belong to the Monitors widget, so this one doesn't read them."""
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
hosts = (await db.execute(select(Host).order_by(Host.name))).scalars().all()
|
||||
host_status = await host_svc.host_status(db, [h.id for h in hosts])
|
||||
uptime = await host_svc.compute_uptime(db)
|
||||
host_names = [h.name for h in hosts]
|
||||
agent = await host_svc.agent_overview_by_host(
|
||||
db, host_names, host_svc.agent_stale_after_seconds(current_app.config))
|
||||
cpu_sparks = await host_svc.agent_cpu_sparks_by_host(db, host_names)
|
||||
|
||||
return await render_template(
|
||||
"hosts/overview_widget.html",
|
||||
hosts=hosts, host_status=host_status,
|
||||
uptime=uptime, agent=agent, cpu_sparks=cpu_sparks,
|
||||
)
|
||||
db, [h.name for h in hosts], host_svc.agent_stale_after_seconds(current_app.config))
|
||||
rows = host_svc.host_glance(hosts, host_status, agent, current_app.config.get("THRESHOLDS", {}))
|
||||
return await render_template("hosts/overview_widget.html", rows=rows)
|
||||
|
||||
|
||||
@hosts_bp.get("/")
|
||||
@@ -314,17 +308,3 @@ async def uptime_page():
|
||||
uptime = await host_svc.compute_uptime(db)
|
||||
hosts = host_svc.worst_uptime_first(result.scalars().all(), uptime)
|
||||
return await render_template("hosts/uptime.html", hosts=hosts, uptime=uptime)
|
||||
|
||||
|
||||
@hosts_bp.get("/uptime/widget")
|
||||
@require_role(UserRole.viewer)
|
||||
async def uptime_widget():
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
result = await db.execute(select(Host).order_by(Host.name))
|
||||
uptime = await host_svc.compute_uptime(db)
|
||||
hosts = host_svc.worst_uptime_first(result.scalars().all(), uptime)
|
||||
return await render_template(
|
||||
"hosts/uptime_widget.html",
|
||||
hosts=hosts,
|
||||
uptime=uptime,
|
||||
)
|
||||
|
||||
+51
-29
@@ -170,10 +170,13 @@ async def host_monitors(db, host_id: str) -> list[dict]:
|
||||
async def agent_overview_by_host(
|
||||
db, host_names: list[str], stale_after_seconds: int = 180,
|
||||
) -> dict[str, dict]:
|
||||
"""Per-host agent glance (cpu/mem/disk + freshness) from plugin_metric_latest.
|
||||
"""Per-host agent glance (cpu/mem/disk/load + freshness) from plugin_metric_latest.
|
||||
|
||||
{host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root, load_1m,
|
||||
last_at, fresh}} for each host with any agent samples.
|
||||
cores, load_per_core, last_at, fresh}} for each host with any
|
||||
agent samples. Cores are counted from the per-core CPU sub-resources
|
||||
("<host>:core<N>"), as on the host page; load_per_core is load_1m as a %
|
||||
of them, the figure Settings → Thresholds grades "load" by.
|
||||
|
||||
Core-safe (no host_agent import). Freshness compares the latest host-level
|
||||
sample against the host_agent plugin's stale window (stale_after_seconds;
|
||||
@@ -188,6 +191,16 @@ async def agent_overview_by_host(
|
||||
latest = await latest_by_resource(
|
||||
db, "host_agent", list(host_names) + [h + ":/" for h in host_names],
|
||||
set(AGENT_GLANCE_METRICS) | {"disk_used_pct"})
|
||||
from steward.models.metrics import PluginMetricLatest
|
||||
cores: dict[str, int] = {}
|
||||
for res in (await db.execute(
|
||||
select(PluginMetricLatest.resource_name).where(
|
||||
PluginMetricLatest.source_module == "host_agent",
|
||||
PluginMetricLatest.metric_name == "cpu_pct",
|
||||
PluginMetricLatest.resource_name.like("%:core%"))
|
||||
)).scalars():
|
||||
host = res.rsplit(":core", 1)[0]
|
||||
cores[host] = cores.get(host, 0) + 1
|
||||
out: dict[str, dict] = {}
|
||||
for name in host_names:
|
||||
own = {m: r for m, r in latest.get(name, {}).items() if m in AGENT_GLANCE_METRICS}
|
||||
@@ -200,35 +213,44 @@ async def agent_overview_by_host(
|
||||
ts = max((r.recorded_at for r in own.values()), default=None)
|
||||
d["last_at"] = ts
|
||||
d["fresh"] = bool(ts and (now - ts).total_seconds() <= stale_after)
|
||||
d["cores"] = cores.get(name)
|
||||
load = d.get("load_1m")
|
||||
d["load_per_core"] = round(load / d["cores"] * 100) if (load is not None and d["cores"]) else None
|
||||
out[name] = d
|
||||
return out
|
||||
|
||||
|
||||
async def agent_cpu_sparks_by_host(db, host_names: list[str], hours: int = 1) -> dict[str, str]:
|
||||
"""{host_name: inline-SVG cpu sparkline} over the last `hours` (core-safe).
|
||||
GLANCE_ORDER = {"down": 0, "crit": 1, "stale": 2, "warn": 3, "up": 4, "unknown": 5}
|
||||
|
||||
A tiny at-a-glance trend in each row — the host-view sparkline, on the widget.
|
||||
"""
|
||||
from steward.models.metrics import PluginMetric
|
||||
from steward.core.status import sparkline_svg
|
||||
if not host_names:
|
||||
return {}
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
|
||||
rows = (await db.execute(
|
||||
select(PluginMetric.resource_name, PluginMetric.value, PluginMetric.recorded_at)
|
||||
.where(
|
||||
PluginMetric.source_module == "host_agent",
|
||||
PluginMetric.metric_name == "cpu_pct",
|
||||
PluginMetric.resource_name.in_(host_names),
|
||||
PluginMetric.recorded_at >= cutoff,
|
||||
)
|
||||
.order_by(PluginMetric.recorded_at)
|
||||
)).all()
|
||||
by_host: dict[str, list[float]] = {}
|
||||
for r in rows:
|
||||
by_host.setdefault(r.resource_name, []).append(r.value)
|
||||
# Amber line to read as "CPU"; only draw when there are a couple of points.
|
||||
return {
|
||||
name: sparkline_svg(vals[-40:], width=64, height=18, stroke="#c8a840")
|
||||
for name, vals in by_host.items() if len(vals) >= 2
|
||||
}
|
||||
|
||||
def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict]:
|
||||
"""The Hosts — Overview rows, worst first (#5251).
|
||||
|
||||
Each is {host, state, worst, agent}. State is "down" when a linked monitor
|
||||
is down, "stale" when the agent has stopped reporting, else the grade of
|
||||
the reading furthest past its cutoff ("crit" / "warn": CPU, memory, disk /
|
||||
or load per core, against Settings → Thresholds), else "up"; "unknown" for
|
||||
a host with neither monitors nor agent. `worst` is (label, value, grade)
|
||||
or None, from formatting.worst_reading."""
|
||||
from steward.core.formatting import worst_reading
|
||||
rows = []
|
||||
for h in hosts:
|
||||
st, a = status.get(h.id), agent.get(h.name)
|
||||
worst = None
|
||||
if a and a.get("fresh"):
|
||||
worst = worst_reading((
|
||||
("CPU", a.get("cpu_pct"), "cpu"), ("Memory", a.get("mem_used_pct"), "mem"),
|
||||
("Disk /", a.get("disk_root"), "disk"), ("Load", a.get("load_per_core"), "load")),
|
||||
thresholds)
|
||||
if st and st["state"] == "down":
|
||||
state = "down"
|
||||
elif a and not a.get("fresh"):
|
||||
state = "stale"
|
||||
elif worst:
|
||||
state = worst[2]
|
||||
elif st or a:
|
||||
state = "up"
|
||||
else:
|
||||
state = "unknown"
|
||||
rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}})
|
||||
return sorted(rows, key=lambda r: (GLANCE_ORDER[r["state"]], r["host"].name.lower()))
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Uptime / SLA folds into Monitors (#5250)
|
||||
|
||||
Revision ID: 0033_retire_uptime_widget
|
||||
Revises: 0032_retire_duplicate_widgets
|
||||
Create Date: 2026-10-07
|
||||
|
||||
The Monitors widget now shows each monitor's uptime over 24h, 7d and 30d, so
|
||||
the separate Uptime / SLA widget (uptime_summary) repeated it. A dashboard
|
||||
that already shows Monitors loses the Uptime placement; any other keeps the
|
||||
cell, now showing Monitors. The same steps as 0032.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0033_retire_uptime_widget"
|
||||
down_revision: Union[str, None] = "0032_retire_duplicate_widgets"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
_OLD, _NEW = "uptime_summary", "monitors"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(sa.text("""
|
||||
DELETE FROM dashboard_widgets d
|
||||
WHERE d.widget_key = :old
|
||||
AND EXISTS (SELECT 1 FROM dashboard_widgets k
|
||||
WHERE k.dashboard_id = d.dashboard_id AND k.widget_key = :new)
|
||||
""").bindparams(old=_OLD, new=_NEW))
|
||||
op.execute(sa.text("""
|
||||
DELETE FROM dashboard_widgets d
|
||||
WHERE d.widget_key = :old
|
||||
AND EXISTS (SELECT 1 FROM dashboard_widgets k
|
||||
WHERE k.dashboard_id = d.dashboard_id AND k.widget_key = :old
|
||||
AND k.id < d.id)
|
||||
""").bindparams(old=_OLD))
|
||||
op.execute(sa.text("""
|
||||
UPDATE dashboard_widgets SET widget_key = :new, config_json = NULL
|
||||
WHERE widget_key = :old
|
||||
""").bindparams(old=_OLD, new=_NEW))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# One-way, as 0032: the retired widget no longer exists to render.
|
||||
pass
|
||||
@@ -221,14 +221,16 @@ async def widget():
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
data, counts, _ = await monitor_svc.monitor_rows(
|
||||
db, DEFAULT_RANGE, current_app.config.get("THRESHOLDS", {}))
|
||||
up, down, pending = counts["up"], counts["down"], counts["pending"]
|
||||
if type_filter in MONITOR_TYPE_VALUES:
|
||||
data = [d for d in data if d["monitor"].type == type_filter]
|
||||
if show_down_only:
|
||||
data = [d for d in data if not (d["latest"] and d["latest"].is_up)]
|
||||
# Counted after the type filter, so the verdict speaks for what is listed.
|
||||
counts = {k: sum(d["state"] == k for d in data) for k in ("up", "degraded", "down", "pending", "paused")}
|
||||
problems = [d for d in data if d["state"] in ("down", "degraded", "pending")]
|
||||
rest = [] if show_down_only else [d for d in data if d["state"] not in ("down", "degraded", "pending")]
|
||||
shown = max(0, limit - len(problems))
|
||||
|
||||
return await render_template(
|
||||
"monitors/widget.html",
|
||||
monitor_data=data[:limit], up=up, down=down, pending=pending,
|
||||
show_down_only=show_down_only, widget_id=widget_id,
|
||||
problems=problems, rest=rest[:shown], more=len(rest) - min(len(rest), shown),
|
||||
counts=counts, total=len(data), show_down_only=show_down_only, widget_id=widget_id,
|
||||
)
|
||||
|
||||
@@ -204,6 +204,7 @@ async def monitor_rows(db, range_key: str, thresholds: dict):
|
||||
"heartbeat": [{"state": "up" if r.is_up else "down",
|
||||
"title": _heartbeat_title(m.type, r)} for r in rows],
|
||||
"uptime_pct": uptime_pct,
|
||||
"uptime": uptime.get(m.id, {}),
|
||||
"tls_days": tls_days,
|
||||
"down_since": None,
|
||||
"down_reason": down_reason(latest) if latest is not None and not latest.is_up else None,
|
||||
|
||||
@@ -381,6 +381,22 @@ body.dash-editing .widget-drawer { transform:translateY(0); }
|
||||
.cell-state { font-size:0.85rem; color:var(--text-muted); padding:0.25rem 0; }
|
||||
.cell-state-failed { display:flex; flex-direction:column; align-items:flex-start; gap:0.5rem; color:var(--yellow); }
|
||||
.widget-link:hover { color:var(--text); }
|
||||
/* Widget content, in the order of the widget-shape ruling (UI & Design):
|
||||
a verdict line, the exceptions, then the healthy rest in a compact table
|
||||
whose column labels appear once. Names cut off with … and show whole on
|
||||
hover; the table never scrolls sideways. */
|
||||
.w-verdict { font-size:0.85rem; margin-bottom:0.5rem; }
|
||||
.w-table { width:100%; border-collapse:collapse; table-layout:fixed; font-size:0.82rem; font-variant-numeric:tabular-nums; }
|
||||
.w-table th { text-align:right; font-weight:500; font-size:var(--fs-size-tiny); color:var(--text-dim); text-transform:uppercase; letter-spacing:var(--fs-tracking-tiny); padding:0 0 0.25rem 0.5rem; white-space:nowrap; }
|
||||
.w-table td { text-align:right; padding:0.2rem 0 0.2rem 0.5rem; border-top:1px solid var(--border); white-space:nowrap; overflow:hidden; text-overflow:ellipsis; }
|
||||
.w-table th:first-child, .w-table td:first-child { text-align:left; padding-left:0; }
|
||||
.w-table td.w-name { overflow:hidden; text-overflow:ellipsis; }
|
||||
.w-table td.w-name a { color:inherit; text-decoration:none; }
|
||||
.w-table td.w-name a:hover { text-decoration:underline; }
|
||||
.w-table .dot { margin-right:0.4rem; vertical-align:middle; }
|
||||
.w-sub { display:block; font-size:0.72rem; color:var(--text-muted); white-space:nowrap; overflow:hidden; text-overflow:ellipsis; }
|
||||
.w-more { font-size:0.75rem; color:var(--text-dim); padding-top:0.3rem; }
|
||||
|
||||
|
||||
/* Time range selector */
|
||||
.range-btn { background:none; border:1px solid var(--border-control); border-radius:4px; color:var(--text-muted); padding:0.18rem 0.5rem; cursor:pointer; font-size:0.75rem; font-family:inherit; transition:background .1s,color .1s,border-color .1s; }
|
||||
|
||||
@@ -1,50 +1,43 @@
|
||||
{# Unified Hosts widget — monitor status + agent glance per host, links to the hub.
|
||||
Name sits on its own line (never truncated); its data wraps beneath it, grouped
|
||||
under the host. Prefer vertical growth over cramming a row. #}
|
||||
{% if hosts %}
|
||||
<div class="host-blocks">
|
||||
{% for host in hosts %}
|
||||
{% set hs = host_status.get(host.id) %}
|
||||
{% set ut = uptime.get(host.id, {}) %}
|
||||
{% set a = agent.get(host.name, {}) %}
|
||||
{% set down = hs and hs.state == 'down' %}
|
||||
{% set spark = cpu_sparks.get(host.name) %}
|
||||
<div style="padding:0.5rem 0;border-bottom:1px solid var(--border);">
|
||||
{# Line 1 — status dot + full host name (own line) + cpu trend #}
|
||||
<div style="display:flex;align-items:center;gap:0.5rem;">
|
||||
<span class="dot {% if down %}dot-down{% elif hs and hs.state == 'up' %}dot-up{% else %}dot-dim{% endif %}"
|
||||
style="flex-shrink:0;"
|
||||
title="{% if not hs %}no monitors{% else %}{{ hs.state }}{% endif %}"></span>
|
||||
{% if session.user_id %}
|
||||
<a href="/hosts/{{ host.id }}" style="font-weight:500;font-size:0.85rem;overflow-wrap:anywhere;">{{ host.name }}</a>
|
||||
{% else %}
|
||||
<span style="font-weight:500;font-size:0.85rem;overflow-wrap:anywhere;">{{ host.name }}</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
{# Line 2 — monitors + agent metrics; wraps instead of truncating #}
|
||||
<div style="display:flex;flex-wrap:wrap;gap:0.25rem 0.9rem;margin-top:0.25rem;padding-left:1.1rem;
|
||||
font-variant-numeric:tabular-nums;color:var(--text-muted);font-size:0.75rem;">
|
||||
{% if hs and hs.latency_ms is not none %}
|
||||
<span title="Latency" style="{{ threshold_style(hs.latency_ms, 'latency') }}"><span style="color:var(--text-dim);">ping</span> {{ "%.0f"|format(hs.latency_ms) }}ms</span>
|
||||
{% endif %}
|
||||
{% if ut.get('24h') is not none %}
|
||||
<span title="Uptime 24h" style="{{ threshold_style(ut['24h'], 'uptime') }}"><span style="color:var(--text-dim);">24h</span> {{ "%.1f"|format(ut['24h']) }}%</span>
|
||||
{% endif %}
|
||||
{% if a %}
|
||||
{% if a.get('cpu_pct') is not none %}<span title="CPU — value + last-hour trend" style="display:inline-flex;align-items:center;gap:0.3rem;{{ threshold_style(a.cpu_pct, 'cpu') }}"><span style="color:var(--text-dim);">cpu</span> {{ "%.0f"|format(a.cpu_pct) }}%{% if spark %}<span style="line-height:0;opacity:0.85;">{{ spark | safe }}</span>{% endif %}</span>{% endif %}
|
||||
{% if a.get('mem_used_pct') is not none %}<span title="Memory" style="{{ threshold_style(a.mem_used_pct, 'mem') }}"><span style="color:var(--text-dim);">mem</span> {{ "%.0f"|format(a.mem_used_pct) }}%</span>{% endif %}
|
||||
{% if a.get('disk_root') is not none %}<span title="Root filesystem (/)" style="{{ threshold_style(a.disk_root, 'disk') }}"><span style="color:var(--text-dim);">disk /</span> {{ "%.0f"|format(a.disk_root) }}%</span>{% endif %}
|
||||
{% if a.get('load_1m') is not none %}<span title="Load average over 1 minute: processes running or waiting to run"><span style="color:var(--text-dim);">load</span> {{ "%.2f"|format(a.load_1m) }}</span>{% endif %}
|
||||
{% if not a.fresh %}<span title="Agent data is stale" style="color:var(--yellow);">stale</span>{% endif %}
|
||||
{% else %}
|
||||
<span style="color:var(--text-dim);" title="No agent reporting">no agent</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
{# hosts/overview_widget.html — the dashboard Hosts widget (#5251): a verdict,
|
||||
then one row per host, worst first, with its resources under one header.
|
||||
Load is shown against the host's cores and graded per core. Reachability
|
||||
and uptime belong to the Monitors widget, so they aren't repeated here. #}
|
||||
{% from "_macros.html" import status_dot %}
|
||||
{% if not rows %}
|
||||
<div class="w-verdict" style="color:var(--text-muted);">
|
||||
No hosts yet.{% if session.user_id %} <a href="/hosts/new">Add one →</a>{% endif %}
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="empty" style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">
|
||||
No hosts yet. <a href="/hosts/new">Add one.</a>
|
||||
{% set problems = rows | selectattr("state", "in", ["down", "crit", "stale", "warn"]) | list %}
|
||||
<div class="w-verdict {{ 'tone-crit' if problems and problems[0].state in ['down', 'crit'] else ('tone-warn' if problems else 'tone-ok') }}">
|
||||
{%- if not problems %}All {{ rows|length }} host{{ 's' if rows|length != 1 }} healthy.
|
||||
{%- elif problems|length == 1 %}{% set p = problems[0] %}{{ p.host.name }}:
|
||||
{% if p.state == 'down' %}a monitor is down.{% elif p.state == 'stale' %}its agent stopped reporting.{% else %}{{ p.worst[0] }} high.{% endif %}
|
||||
{%- else %}{{ problems|length }} of {{ rows|length }} hosts need a look.{% endif -%}
|
||||
</div>
|
||||
<table class="w-table">
|
||||
<colgroup><col><col style="width:3.6rem"><col style="width:3.6rem"><col style="width:3.6rem"><col style="width:6.5rem"></colgroup>
|
||||
<thead><tr><th>Host</th><th>CPU</th><th>Mem</th><th>Disk /</th><th>Load</th></tr></thead>
|
||||
<tbody>
|
||||
{% for r in rows %}
|
||||
{% set a = r.agent %}{% set fresh = a.get('fresh') %}
|
||||
<tr>
|
||||
<td class="w-name" title="{{ r.host.name }}">
|
||||
{{- status_dot(r.state) -}}
|
||||
{% if session.user_id %}<a href="/hosts/{{ r.host.id }}">{{ r.host.name }}</a>{% else %}{{ r.host.name }}{% endif %}
|
||||
{%- if r.state == 'down' %}<span class="w-sub tone-crit">a monitor is down</span>
|
||||
{%- elif r.state == 'stale' %}<span class="w-sub tone-warn">agent last reported {{ a.last_at|ago }}; figures are from then</span>
|
||||
{%- elif not a %}<span class="w-sub">no agent</span>{% endif %}
|
||||
</td>
|
||||
{% for v, kind in ((a.get('cpu_pct'), 'cpu'), (a.get('mem_used_pct'), 'mem'), (a.get('disk_root'), 'disk')) %}
|
||||
<td style="{{ threshold_style(v, kind) if fresh and v is not none }}">{% if v is none %}—{% else %}{{ "%.0f"|format(v) }}%{% endif %}</td>
|
||||
{% endfor %}
|
||||
<td style="{{ threshold_style(a.get('load_per_core'), 'load') if fresh and a.get('load_per_core') is not none }}"
|
||||
title="{% if a.get('cores') %}Load over 1 minute on {{ a.cores }} cores: {{ a.load_per_core }}% of them busy or waiting{% endif %}">
|
||||
{%- if a.get('load_1m') is none %}—{% elif a.get('cores') %}{{ "%.1f"|format(a.load_1m) }} / {{ a.cores }} cores{% else %}{{ "%.1f"|format(a.load_1m) }}{% endif -%}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% endif %}
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
{# hosts/uptime_widget.html — dashboard widget: SLA uptime summary #}
|
||||
{% if not hosts %}
|
||||
<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">
|
||||
No hosts configured.
|
||||
</div>
|
||||
{% else %}
|
||||
|
||||
<div style="display:grid;gap:2px;">
|
||||
<div style="display:grid;grid-template-columns:1fr 52px 52px 52px;gap:0.35rem;font-size:0.72rem;color:var(--text-dim);padding:0 0 0.25rem;border-bottom:1px solid var(--border);">
|
||||
<span>Host</span>
|
||||
<span style="text-align:center;">24h</span>
|
||||
<span style="text-align:center;">7d</span>
|
||||
<span style="text-align:center;">30d</span>
|
||||
</div>
|
||||
{% for host in hosts %}
|
||||
{% set ut = uptime.get(host.id, {}) %}
|
||||
<div style="display:grid;grid-template-columns:1fr 52px 52px 52px;gap:0.35rem;align-items:center;padding:0.15rem 0;font-size:0.82rem;">
|
||||
{% if session.user_id %}
|
||||
<a href="/hosts/{{ host.id }}" title="{{ host.name }}"
|
||||
style="overflow-wrap:anywhere;color:inherit;text-decoration:none;">{{ host.name }}</a>
|
||||
{% else %}
|
||||
<span style="overflow-wrap:anywhere;" title="{{ host.name }}">{{ host.name }}</span>
|
||||
{% endif %}
|
||||
{% for window in ["24h", "7d", "30d"] %}
|
||||
{% set pct = ut.get(window) %}
|
||||
<span style="text-align:center;font-family:ui-monospace,monospace;font-size:0.78rem;">
|
||||
{% if pct is none %}
|
||||
<span style="color:var(--text-dim);">—</span>
|
||||
{% else %}
|
||||
{# Configurable uptime thresholds (Settings → Thresholds); a normal value
|
||||
(no degraded style) stays green to read as healthy at a glance. #}
|
||||
{% set us = threshold_style(pct, 'uptime') %}
|
||||
<span style="{{ us if us else 'color:var(--green);' }}">{{ "%.1f"|format(pct) }}%</span>
|
||||
{% endif %}
|
||||
</span>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
{% if session.user_id %}
|
||||
<div style="margin-top:0.5rem;text-align:right;">
|
||||
<a href="/hosts/uptime" style="font-size:0.75rem;color:var(--text-muted);">Full SLA →</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% endif %}
|
||||
@@ -1,44 +1,44 @@
|
||||
{# monitors/widget.html — dashboard widget: unified monitor summary #}
|
||||
{% if not monitor_data and up == 0 and down == 0 and pending == 0 %}
|
||||
<div style="color:var(--text-muted);font-size:0.85rem;padding:0.5rem 0;">
|
||||
No monitors configured. <a href="/monitors/new">Add a monitor →</a>
|
||||
{# monitors/widget.html — the dashboard Monitors widget (#5250): a verdict,
|
||||
then one row per monitor, problems first with why and since when, under one
|
||||
header of latency and uptime over 24h / 7d / 30d. It replaced the separate
|
||||
Uptime / SLA widget, so uptime has one home. #}
|
||||
{% from "_macros.html" import status_dot %}
|
||||
{% if not total %}
|
||||
<div class="w-verdict" style="color:var(--text-muted);">
|
||||
No monitors configured.{% if session.user_id %} <a href="/monitors/new">Add a monitor →</a>{% endif %}
|
||||
</div>
|
||||
{% else %}
|
||||
|
||||
<div style="display:flex;gap:1rem;margin-bottom:0.65rem;flex-wrap:wrap;">
|
||||
{% if up %}
|
||||
<div><span style="font-size:1.4rem;font-weight:500;line-height:1;color:var(--green);">{{ up }}</span>
|
||||
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">up</span></div>
|
||||
{% endif %}
|
||||
{% if down %}
|
||||
<div><span style="font-size:1.4rem;font-weight:500;line-height:1;color:var(--red);">{{ down }}</span>
|
||||
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">down</span></div>
|
||||
{% endif %}
|
||||
{% if pending %}
|
||||
<div><span style="font-size:1.4rem;font-weight:500;line-height:1;color:var(--text-dim);">{{ pending }}</span>
|
||||
<span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">pending</span></div>
|
||||
{% endif %}
|
||||
<div class="w-verdict {{ 'tone-crit' if counts.down else ('tone-warn' if counts.degraded or counts.pending else 'tone-ok') }}">
|
||||
{%- if counts.down %}{{ counts.down }} of {{ total }} monitor{{ 's' if total != 1 }} down{% if counts.degraded %}, {{ counts.degraded }} degraded{% endif %}.
|
||||
{%- elif counts.degraded %}{{ counts.degraded }} of {{ total }} monitor{{ 's' if total != 1 }} degraded, the rest up.
|
||||
{%- elif counts.pending %}{{ counts.pending }} not checked yet, {{ counts.up }} up.
|
||||
{%- else %}All {{ counts.up }} monitor{{ 's' if counts.up != 1 }} up{% if counts.paused %}, {{ counts.paused }} paused{% endif %}.{% endif -%}
|
||||
</div>
|
||||
|
||||
<div style="display:grid;gap:3px;">
|
||||
{% for d in monitor_data %}
|
||||
{% set latest = d.latest %}
|
||||
<div style="display:flex;align-items:center;gap:0.5rem;font-size:0.82rem;padding:0.15rem 0;">
|
||||
{% if not latest %}<span class="dot dot-dim"></span>
|
||||
{% elif latest.is_up %}<span class="dot dot-up"></span>
|
||||
{% else %}<span class="dot dot-down"></span>{% endif %}
|
||||
{% if d.monitor.host_id %}
|
||||
<a href="/hosts/{{ d.monitor.host_id }}" style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:inherit;text-decoration:none;">{{ d.monitor.name }}</a>
|
||||
{% else %}
|
||||
<span style="flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ d.monitor.name }}</span>
|
||||
{% endif %}
|
||||
{% if latest and latest.response_ms is not none %}
|
||||
<span style="font-size:0.72rem;font-family:ui-monospace,monospace;color:var(--text-muted);flex-shrink:0;">{{ "%.0f"|format(latest.response_ms) }}ms</span>
|
||||
{% elif latest and not latest.is_up %}
|
||||
<span style="font-size:0.72rem;color:var(--red);flex-shrink:0;">{{ latest.status_code or "down" }}</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if problems or rest %}
|
||||
<table class="w-table">
|
||||
<colgroup><col><col style="width:4.5rem"><col style="width:4.5rem"><col style="width:4.5rem"><col style="width:4.5rem"></colgroup>
|
||||
<thead><tr><th>Monitor</th><th>Latency</th><th>24h</th><th>7d</th><th>30d</th></tr></thead>
|
||||
<tbody>
|
||||
{% for d in problems + rest %}
|
||||
{% set m = d.monitor %}{% set latest = d.latest %}
|
||||
<tr>
|
||||
<td class="w-name" title="{{ m.name }}">
|
||||
{{- status_dot(d.state) -}}
|
||||
{% if session.user_id %}<a href="/monitors/{{ m.id }}">{{ m.name }}</a>{% else %}{{ m.name }}{% endif %}
|
||||
{%- if d.state == 'down' %}<span class="w-sub tone-crit">went down {{ d.down_since|ago if d.down_since else '' }}{% if d.down_reason %} · {{ d.down_reason }}{% endif %}</span>
|
||||
{%- elif d.state == 'degraded' %}<span class="w-sub tone-warn">{% if latest and status_class(latest.response_ms, 'latency') == 'crit' %}slow replies{% else %}checks failing now and then{% endif %}</span>
|
||||
{%- elif d.state == 'pending' %}<span class="w-sub">not checked yet</span>{% endif %}
|
||||
</td>
|
||||
<td style="{{ threshold_style(latest.response_ms, 'latency') if latest and latest.is_up and latest.response_ms is not none }}">
|
||||
{%- if latest and latest.is_up and latest.response_ms is not none %}{{ "%.0f"|format(latest.response_ms) }} ms{% else %}—{% endif -%}
|
||||
</td>
|
||||
{% for k in ("24h", "7d", "30d") %}{% set u = d.uptime.get(k) %}
|
||||
<td style="{{ threshold_style(u, 'uptime') if u is not none }}">{% if u is none %}—{% else %}{{ "%.2f"|format(u) }}%{% endif %}</td>
|
||||
{% endfor %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
{% endif %}
|
||||
{% if more %}<div class="w-more">+{{ more }} more up{% if session.user_id %} · <a href="/monitors/">all monitors</a>{% endif %}</div>{% endif %}
|
||||
{% endif %}
|
||||
|
||||
@@ -35,11 +35,9 @@ def test_the_recommended_layout_fills_rows_without_overlap():
|
||||
|
||||
|
||||
def test_the_recommended_layout_skips_what_is_not_enabled():
|
||||
keys = [i["key"] for i in recommended_layout({})]
|
||||
assert keys == ["status_overview", "hosts_overview", "monitors", "uptime_summary"]
|
||||
# uptime no longer fits beside the monitors, so it starts the next row
|
||||
items = recommended_layout({})
|
||||
assert (items[3]["x"], items[3]["y"]) == (0, 10)
|
||||
assert [i["key"] for i in items] == ["status_overview", "hosts_overview", "monitors"]
|
||||
assert [(i["x"], i["y"]) for i in items] == [(0, 0), (0, 3), (6, 3)]
|
||||
|
||||
|
||||
def test_overview_alerts_put_the_breaching_first():
|
||||
@@ -86,19 +84,65 @@ async def test_a_quiet_overview_says_so(app):
|
||||
assert ">host<" in page and 'href="/alerts/"' not in page # the share view has no links
|
||||
|
||||
|
||||
async def test_the_hosts_widget_shows_load(app):
|
||||
host = SimpleNamespace(id="h1", name="web-1")
|
||||
agent = {"web-1": {"cpu_pct": 12.0, "mem_used_pct": 40.0, "load_1m": 0.734, "fresh": True}}
|
||||
def _host(hid, name):
|
||||
return SimpleNamespace(id=hid, name=name)
|
||||
|
||||
|
||||
def test_hosts_sort_worst_first():
|
||||
"""#5251: a down monitor, then a reading past critical, then a quiet agent,
|
||||
then a warning, then the healthy, by name; a host with nothing last."""
|
||||
from steward.core.settings import to_thresholds_cfg
|
||||
from steward.hosts.service import host_glance
|
||||
th = to_thresholds_cfg({})
|
||||
hosts = [_host("a", "alpha"), _host("b", "busy"), _host("d", "dead"), _host("q", "quiet"),
|
||||
_host("w", "warm"), _host("n", "nothing")]
|
||||
status = {"d": {"state": "down"}, "a": {"state": "up"}}
|
||||
agent = {"alpha": {"cpu_pct": 10.0, "fresh": True}, "busy": {"cpu_pct": 99.0, "fresh": True},
|
||||
"quiet": {"cpu_pct": 99.0, "fresh": False},
|
||||
"warm": {"cpu_pct": 10.0, "load_per_core": 85, "fresh": True}}
|
||||
rows = host_glance(hosts, status, agent, th)
|
||||
assert [(r["host"].name, r["state"]) for r in rows] == [
|
||||
("dead", "down"), ("busy", "crit"), ("quiet", "stale"), ("warm", "warn"),
|
||||
("alpha", "up"), ("nothing", "unknown")]
|
||||
assert rows[1]["worst"][0] == "CPU" and rows[3]["worst"][0] == "Load"
|
||||
assert rows[2]["worst"] is None # a quiet agent's old figures aren't graded
|
||||
|
||||
|
||||
async def test_the_hosts_widget_gives_a_verdict_and_load_against_cores(app):
|
||||
from datetime import datetime, timezone
|
||||
rows = [
|
||||
{"host": _host("h2", "VDNT-Util01"), "state": "crit", "worst": ("CPU", 99.0, "crit"),
|
||||
"agent": {"cpu_pct": 99.0, "mem_used_pct": 28.0, "disk_root": 52.0, "load_1m": 16.77,
|
||||
"cores": 8, "load_per_core": 210, "fresh": True}},
|
||||
{"host": _host("h1", "VDNT-Docker02"), "state": "up", "worst": None,
|
||||
"agent": {"cpu_pct": 1.0, "mem_used_pct": 4.0, "disk_root": 44.0, "load_1m": 0.92,
|
||||
"cores": 4, "load_per_core": 23, "fresh": True}},
|
||||
{"host": _host("h3", "nas"), "state": "stale", "worst": None,
|
||||
"agent": {"cpu_pct": 5.0, "fresh": False, "last_at": datetime(2026, 10, 7, tzinfo=timezone.utc)}},
|
||||
]
|
||||
async with app.test_request_context("/hosts/overview/widget"):
|
||||
page = await render_template("hosts/overview_widget.html", hosts=[host], host_status={},
|
||||
uptime={}, agent=agent, cpu_sparks={})
|
||||
assert ">load</span> 0.73" in page
|
||||
session["user_id"] = "u1"
|
||||
page = await render_template("hosts/overview_widget.html", rows=rows)
|
||||
assert "2 of 3 hosts need a look." in page
|
||||
assert page.index("VDNT-Util01") < page.index("VDNT-Docker02")
|
||||
assert "16.8 / 8 cores" in page and "0.9 / 4 cores" in page
|
||||
assert "agent last reported" in page
|
||||
assert page.count("<th>") == 5 and ">ping<" not in page and "24h" not in page
|
||||
assert not hex_colours(page), hex_colours(page)
|
||||
|
||||
|
||||
async def test_one_busy_host_is_named_in_the_verdict(app):
|
||||
rows = [{"host": _host("h2", "web-1"), "state": "crit", "worst": ("CPU", 99.0, "crit"),
|
||||
"agent": {"cpu_pct": 99.0, "fresh": True}}]
|
||||
async with app.test_request_context("/hosts/overview/widget"):
|
||||
page = await render_template("hosts/overview_widget.html", rows=rows)
|
||||
assert "web-1:" in page and "CPU high." in page and "tone-crit" in page
|
||||
|
||||
|
||||
async def test_a_widget_polls_at_its_own_pace(app):
|
||||
defn = {**WIDGET_REGISTRY["uptime_summary"]}
|
||||
defn = {"poll": True, "poll_seconds": 300, "detail_url": "/x"}
|
||||
item = SimpleNamespace(db=SimpleNamespace(id=7, grid_x=0, grid_y=0, grid_w=6, grid_h=5),
|
||||
title="Uptime", defn=defn, hx_url="/uptime/widget")
|
||||
title="Slow", defn=defn, hx_url="/slow/widget")
|
||||
async with app.test_request_context("/"):
|
||||
page = await render_template("dashboard/_grid_item.html", item=item, can_edit=False, poll_interval=30)
|
||||
assert "every 300s" in page
|
||||
|
||||
@@ -137,3 +137,36 @@ async def test_a_down_monitor_says_since_when_and_why_in_the_list(app):
|
||||
range_key="24h")
|
||||
page = " ".join(page.split())
|
||||
assert re.search(r'Down <time [^>]*>12m</time> · connection refused', page), page
|
||||
|
||||
|
||||
async def _widget(app, problems, rest, more=0, counts=None):
|
||||
counts = counts or {"up": len(rest), "degraded": 0, "down": len(problems), "pending": 0, "paused": 0}
|
||||
async with app.test_request_context("/monitors/widget"):
|
||||
session["user_id"] = "u1"
|
||||
return await render_template("monitors/widget.html", problems=problems, rest=rest, more=more,
|
||||
counts=counts, total=len(problems) + len(rest) + more,
|
||||
show_down_only=False, widget_id="1")
|
||||
|
||||
|
||||
async def test_the_widget_gives_a_verdict_then_problems_with_uptime_columns(app):
|
||||
"""#5250: one home for uptime. A verdict, the down monitor first with why
|
||||
and since when, then each monitor's latency and 24h/7d/30d under one header."""
|
||||
down = {**_row(_mon("api", mid="m2"), "down", _check(False)),
|
||||
"uptime": {"24h": 90.0, "7d": 97.5, "30d": 99.12},
|
||||
"down_since": NOW - timedelta(minutes=12), "down_reason": "HTTP 503"}
|
||||
up = {**_row(_mon("web", mid="m1"), "up", _check(ms=31)), "uptime": {"24h": 100.0, "7d": 99.96}}
|
||||
page = await _widget(app, [down], [up], more=3)
|
||||
assert "1 of 5 monitors down." in page
|
||||
assert page.index(">api<") < page.index(">web<")
|
||||
assert 'href="/monitors/m2"' in page and "went down" in page and "12m ago" in page and "HTTP 503" in page
|
||||
assert page.count("<th>") == 5 and "<th>30d</th>" in page
|
||||
assert "99.96%" in page and "31 ms" in page
|
||||
assert ">—<" in page # web has no 30d yet: a dash, not 0
|
||||
assert "+3 more up" in page
|
||||
assert not hex_colours(page), hex_colours(page)
|
||||
|
||||
|
||||
async def test_the_widget_says_when_all_is_well(app):
|
||||
up = {**_row(_mon("web", mid="m1"), "up", _check()), "uptime": {"24h": 100.0}}
|
||||
page = await _widget(app, [], [up])
|
||||
assert "All 1 monitor up." in page and "tone-ok" in page
|
||||
|
||||
@@ -98,6 +98,29 @@ def test_a_quiet_host_keeps_its_last_values_and_reads_stale_not_missing(app):
|
||||
assert "never-" + quiet not in overview # no samples at all: absent
|
||||
|
||||
|
||||
|
||||
def test_load_is_read_against_the_hosts_cores(app):
|
||||
"""#5251: cores are counted from the per-core CPU sub-resources, and load
|
||||
per core is load_1m as a % of them, the figure "load" thresholds grade."""
|
||||
from steward.core.alerts import record_metrics
|
||||
from steward.hosts.service import agent_overview_by_host
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
host = "cores-" + uuid.uuid4().hex[:8]
|
||||
|
||||
async def go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await record_metrics(s, [
|
||||
_sample(host, "load_1m", 3.0, now),
|
||||
_sample(host + ":core0", "cpu_pct", 80.0, now),
|
||||
_sample(host + ":core1", "cpu_pct", 90.0, now),
|
||||
])
|
||||
return await agent_overview_by_host(s, [host], 180)
|
||||
|
||||
h = asyncio.run(go())[host]
|
||||
assert (h["cores"], h["load_per_core"]) == (2, 150)
|
||||
|
||||
def test_retention_drops_a_latest_value_with_its_raw_history(app):
|
||||
from steward.core.alerts import record_metrics
|
||||
from steward.core.metrics_retention import rollup_plugin_metrics
|
||||
|
||||
Reference in New Issue
Block a user