diff --git a/plugins/host_agent/templates/panel.html b/plugins/host_agent/templates/panel.html index d85904d..3feb62e 100644 --- a/plugins/host_agent/templates/panel.html +++ b/plugins/host_agent/templates/panel.html @@ -1,5 +1,4 @@ {# Per-host agent panel — embedded into /hosts/ via HTMX. Self-contained. #} -{% from "_macros.html" import metric_style %} {% set scope = "steward:target:" ~ target.id if target else "" %}
@@ -22,20 +21,20 @@
CPU
-
{{ '%.0f%%'|format(cpu) if cpu is not none else '—' }}
+
{{ '%.0f%%'|format(cpu) if cpu is not none else '—' }}
{{ sparks.cpu | safe }}
Memory
-
{{ '%.0f%%'|format(mem) if mem is not none else '—' }}
+
{{ '%.0f%%'|format(mem) if mem is not none else '—' }}
{{ sparks.mem | safe }}
Disk /
-
{{ '%.0f%%'|format(disk_root) if disk_root is not none else '—' }}
+
{{ '%.0f%%'|format(disk_root) if disk_root is not none else '—' }}
{{ sparks.disk | safe }}
{# Load /core: 100% = run queue matches CPU capacity, so warn 80 / crit 100. #}
Load /core
-
{{ '%d%%'|format(load_per_core) if load_per_core is not none else ('%.2f'|format(load1) if load1 is not none else '—') }}
+
{{ '%d%%'|format(load_per_core) if load_per_core is not none else ('%.2f'|format(load1) if load1 is not none else '—') }}
{{ sparks.load | safe }}
{% if psi.cpu is not none or psi.mem is not none or psi.io is not none %} diff --git a/plugins/host_agent/templates/widget_table.html b/plugins/host_agent/templates/widget_table.html index d543b8d..163823f 100644 --- a/plugins/host_agent/templates/widget_table.html +++ b/plugins/host_agent/templates/widget_table.html @@ -3,7 +3,6 @@ left, metric cells (cpu/mem/disk/load) each with a trend sparkline laid out horizontally on the right. The name wraps (never hard-truncated) but the row stays horizontal. #} -{% from "_macros.html" import metric_style %} {% if rows %} {% macro metric_cell(label, value, fmt, svg, style="", title="") %}
@@ -54,9 +53,9 @@
{# Right — metric cells with value + trend, laid out horizontally #}
- {{ metric_cell("cpu", r.cpu_pct, "%.0f%%", r.sparks.cpu, metric_style(r.cpu_pct), "Average CPU utilization") }} - {{ metric_cell("mem", r.mem_used_pct, "%.0f%%", r.sparks.mem, metric_style(r.mem_used_pct), "Memory in use") }} - {{ metric_cell("disk /", r.disk_root, "%.0f%%", r.sparks.disk, metric_style(r.disk_root), "Root filesystem (/) usage") }} + {{ metric_cell("cpu", r.cpu_pct, "%.0f%%", r.sparks.cpu, threshold_style(r.cpu_pct, 'cpu'), "Average CPU utilization") }} + {{ metric_cell("mem", r.mem_used_pct, "%.0f%%", r.sparks.mem, threshold_style(r.mem_used_pct, 'mem'), "Memory in use") }} + {{ metric_cell("disk /", r.disk_root, "%.0f%%", r.sparks.disk, threshold_style(r.disk_root, 'disk'), "Root filesystem (/) usage") }} {{ metric_cell("load", r.load_1m, "%.2f", r.sparks.load, "", "Load average (1m)") }} {# Extra agent metrics — shown only when the host reports them (VMs/containers often have no temp sensor, etc.) so absent data doesn't clutter the row. #} @@ -67,7 +66,7 @@ {{ io_cell("disk i/o", "rd", r.disk_read_bps, "wr", r.disk_write_bps, "Disk I/O (read / write)") }} {% endif %} {% if r.temp_max is not none %} - {{ metric_cell("temp", r.temp_max, "%.0f°C", "", metric_style(r.temp_max, warn=70, crit=85), "Hottest sensor") }} + {{ metric_cell("temp", r.temp_max, "%.0f°C", "", threshold_style(r.temp_max, 'temp'), "Hottest sensor") }} {% endif %}
diff --git a/steward/app.py b/steward/app.py index 92b1c2c..77d640e 100644 --- a/steward/app.py +++ b/steward/app.py @@ -59,6 +59,7 @@ def create_app( from .core.settings import ( load_settings_sync, to_smtp_cfg, to_webhook_cfg, to_ansible_cfg, to_plugins_cfg, to_oidc_cfg, to_ldap_cfg, + to_thresholds_cfg, ) _settings = load_settings_sync(app.config["DATABASE_URL"]) app.config.update( @@ -71,8 +72,10 @@ def create_app( PLUGINS=to_plugins_cfg(_settings), OIDC=to_oidc_cfg(_settings), LDAP=to_ldap_cfg(_settings), + THRESHOLDS=to_thresholds_cfg(_settings), ) else: + from .core.settings import to_thresholds_cfg app.config.update( SESSION_LIFETIME_HOURS=8, DATA_RETENTION_DAYS=90, @@ -83,8 +86,17 @@ def create_app( PLUGINS={}, OIDC={"enabled": False}, LDAP={"enabled": False}, + THRESHOLDS=to_thresholds_cfg({}), ) + # `threshold_style(value, kind)` — degraded-value coloring for templates, + # reading the configurable cutoffs from app.config["THRESHOLDS"]. Available + # in every template (core + plugin) via the jinja global. + from .core.settings import threshold_style_for as _ts_for + app.jinja_env.globals["threshold_style"] = ( + lambda value, kind: _ts_for(value, kind, app.config.get("THRESHOLDS", {})) + ) + # ── 5. Plugin migrations ─────────────────────────────────────────────────── # Step 3 already applied all plugin migrations via discover_all_plugin_migration_dirs, # so this is a no-op on normal startup. We still run it with all discovered dirs so diff --git a/steward/core/settings.py b/steward/core/settings.py index e636e2b..ef7f0bc 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -68,6 +68,16 @@ DEFAULTS: dict[str, Any] = { "ansible.max_concurrent_runs": 3, "ping.threshold.good_ms": 50, "ping.threshold.warn_ms": 200, + # Degraded/critical cutoffs for metric coloring (warn=amber, crit=red). + # cpu/mem/disk/load are percentages (load is load-per-core %); temp in °C. + # uptime is "higher is better" so its warn/crit are floors. Ping latency + # reuses ping.threshold.good_ms/warn_ms above. + "thresholds.cpu_warn": 80, "thresholds.cpu_crit": 90, + "thresholds.mem_warn": 80, "thresholds.mem_crit": 90, + "thresholds.disk_warn": 80, "thresholds.disk_crit": 90, + "thresholds.load_warn": 80, "thresholds.load_crit": 100, + "thresholds.temp_warn": 70, "thresholds.temp_crit": 85, + "thresholds.uptime_warn": 99.0, "thresholds.uptime_crit": 95.0, "plugins.index_url": "https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml", # Default-enabled plugins. These are the generic, non-vendor-specific # bundled plugins (protocols/standards, not a single product) — useful on @@ -224,6 +234,53 @@ def to_ldap_cfg(settings: dict[str, Any]) -> dict: return {k[len("ldap."):]: settings.get(k, DEFAULTS[k]) for k in DEFAULTS if k.startswith("ldap.")} +def to_thresholds_cfg(settings: dict[str, Any]) -> dict: + """Per-metric (warn, crit, direction) policy for degraded-value coloring. + + Centralizes "what counts as degraded" so templates just name a metric kind + via the threshold_style() jinja global. `dir` is "high" (higher is worse, + e.g. CPU) or "low" (lower is worse, e.g. uptime %). Latency reuses the + existing ping good/warn thresholds. + """ + g = settings.get + return { + "cpu": {"warn": g("thresholds.cpu_warn", 80), "crit": g("thresholds.cpu_crit", 90), "dir": "high"}, + "mem": {"warn": g("thresholds.mem_warn", 80), "crit": g("thresholds.mem_crit", 90), "dir": "high"}, + "disk": {"warn": g("thresholds.disk_warn", 80), "crit": g("thresholds.disk_crit", 90), "dir": "high"}, + "load": {"warn": g("thresholds.load_warn", 80), "crit": g("thresholds.load_crit", 100), "dir": "high"}, + "temp": {"warn": g("thresholds.temp_warn", 70), "crit": g("thresholds.temp_crit", 85), "dir": "high"}, + "latency": {"warn": g("ping.threshold.good_ms", 50), "crit": g("ping.threshold.warn_ms", 200), "dir": "high"}, + "uptime": {"warn": g("thresholds.uptime_warn", 99.0), "crit": g("thresholds.uptime_crit", 95.0), "dir": "low"}, + } + + +def threshold_style_for(value: Any, kind: str, thresholds: dict) -> str: + """Return an inline-style fragment (amber at warn, red at crit) for a metric. + + The Python twin of the old _macros.metric_style, but reading configurable + cutoffs from `thresholds` (see to_thresholds_cfg) and handling both + directions. Empty string when normal/unknown/None — drop straight into a + span's style="". + """ + if value is None: + return "" + th = thresholds.get(kind) + if not th: + return "" + warn, crit, direction = th["warn"], th["crit"], th.get("dir", "high") + if direction == "low": + if value < crit: + return "color:var(--red);font-weight:700;" + if value < warn: + return "color:var(--yellow);font-weight:600;" + else: + if value >= crit: + return "color:var(--red);font-weight:700;" + if value >= warn: + return "color:var(--yellow);font-weight:600;" + return "" + + def to_plugins_cfg(settings: dict[str, Any]) -> dict: """Assemble {plugin_name: {...config}} from all plugin.* keys.""" result = {} diff --git a/steward/settings/routes.py b/steward/settings/routes.py index 69e3c0e..8b1f0f6 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -10,7 +10,7 @@ from steward.models.users import UserRole from steward.core.settings import ( get_all_settings, set_setting, to_smtp_cfg, to_webhook_cfg, to_ansible_cfg, to_plugins_cfg, - to_oidc_cfg, to_ldap_cfg, + to_oidc_cfg, to_ldap_cfg, to_thresholds_cfg, ) settings_bp = Blueprint("settings", __name__, url_prefix="/settings") @@ -72,6 +72,7 @@ async def _reload_app_config() -> None: PLUGINS=to_plugins_cfg(fresh), OIDC=to_oidc_cfg(fresh), LDAP=to_ldap_cfg(fresh), + THRESHOLDS=to_thresholds_cfg(fresh), ) @@ -113,6 +114,49 @@ async def save_general(): return redirect(url_for("settings.general")) +# ── Monitoring thresholds ───────────────────────────────────────────────────── + +# (form field, setting key, is_float) for every tunable cutoff. +_THRESHOLD_FIELDS = [ + ("cpu_warn", "thresholds.cpu_warn", False), ("cpu_crit", "thresholds.cpu_crit", False), + ("mem_warn", "thresholds.mem_warn", False), ("mem_crit", "thresholds.mem_crit", False), + ("disk_warn", "thresholds.disk_warn", False), ("disk_crit", "thresholds.disk_crit", False), + ("load_warn", "thresholds.load_warn", False), ("load_crit", "thresholds.load_crit", False), + ("temp_warn", "thresholds.temp_warn", False), ("temp_crit", "thresholds.temp_crit", False), + ("uptime_warn", "thresholds.uptime_warn", True), ("uptime_crit", "thresholds.uptime_crit", True), + ("latency_warn", "ping.threshold.good_ms", False), ("latency_crit", "ping.threshold.warn_ms", False), +] + + +@settings_bp.get("/thresholds/") +@require_role(UserRole.admin) +async def thresholds(): + async with current_app.db_sessionmaker() as db: + settings = await get_all_settings(db) + return await render_template("settings/thresholds.html", settings=settings) + + +@settings_bp.post("/thresholds/") +@require_role(UserRole.admin) +async def save_thresholds(): + form = await request.form + async with current_app.db_sessionmaker() as db: + async with db.begin(): + for field, key, is_float in _THRESHOLD_FIELDS: + raw = form.get(field, "") + if raw == "": + continue + try: + val = float(raw) if is_float else int(raw) + except (TypeError, ValueError): + continue + await set_setting(db, key, val) + await _reload_app_config() + await log_audit(current_app, session.get("user_id"), session.get("username", ""), + "settings.saved", detail={"section": "thresholds"}) + return redirect(url_for("settings.thresholds")) + + # ── Notifications (SMTP + Webhook) ──────────────────────────────────────────── @settings_bp.get("/notifications/") diff --git a/steward/templates/_macros.html b/steward/templates/_macros.html index d5666f1..c97bcdc 100644 --- a/steward/templates/_macros.html +++ b/steward/templates/_macros.html @@ -17,24 +17,6 @@ {%- endmacro %} -{# Threshold emphasis for a numeric metric (CPU/mem/disk %, etc). - Returns an inline-style fragment to drop into a span's style="": amber + - semibold at >= warn, red + bold at >= crit, empty (inherit) when the value - is normal or None. This makes the FAILING METRIC ITSELF stand out — not just - the status dot. Defaults suit percentage gauges (warn 80 / crit 90); pass - explicit warn/crit for other scales. Keep thresholds here so they live in one - place rather than hardcoded per template. #} -{% macro metric_style(value, warn=80, crit=90) -%} -{%- if value is not none and value >= crit -%}color:var(--red);font-weight:700; -{%- elif value is not none and value >= warn -%}color:var(--yellow);font-weight:600; -{%- endif -%} -{%- endmacro %} - -{# Inverted emphasis for "higher is better" metrics like uptime %: amber below - `warn`, red below `crit`. The mirror of metric_style — same one-place-for- - thresholds discipline, opposite direction. Defaults suit SLA percentages. #} -{% macro uptime_style(value, warn=99.0, crit=95.0) -%} -{%- if value is not none and value < crit -%}color:var(--red);font-weight:700; -{%- elif value is not none and value < warn -%}color:var(--yellow);font-weight:600; -{%- endif -%} -{%- endmacro %} +{# Metric threshold coloring moved to the `threshold_style(value, kind)` jinja + global (steward/core/settings.py + app.py), which reads operator-configurable + cutoffs from Settings → Thresholds. Call it as threshold_style(v, 'cpu'). #} diff --git a/steward/templates/hosts/overview_widget.html b/steward/templates/hosts/overview_widget.html index 581d2db..e92a125 100644 --- a/steward/templates/hosts/overview_widget.html +++ b/steward/templates/hosts/overview_widget.html @@ -1,7 +1,6 @@ {# Unified Hosts widget — monitor status + agent glance per host, links to the hub. Name sits on its own line (never truncated); its data wraps beneath it, grouped under the host. Prefer vertical growth over cramming a row. #} -{% from "_macros.html" import metric_style, uptime_style %} {% if hosts %}
{% for host in hosts %} @@ -27,15 +26,15 @@
{% if host.ping_enabled and ping and ping.response_time_ms is not none %} - ping {{ "%.0f"|format(ping.response_time_ms) }}ms + ping {{ "%.0f"|format(ping.response_time_ms) }}ms {% endif %} {% if ut.get('24h') is not none %} - 24h {{ "%.1f"|format(ut['24h']) }}% + 24h {{ "%.1f"|format(ut['24h']) }}% {% endif %} {% if a %} - {% if a.get('cpu_pct') is not none %}cpu {{ "%.0f"|format(a.cpu_pct) }}%{% if spark %}{{ spark | safe }}{% endif %}{% endif %} - {% if a.get('mem_used_pct') is not none %}mem {{ "%.0f"|format(a.mem_used_pct) }}%{% endif %} - {% if a.get('disk_root') is not none %}disk / {{ "%.0f"|format(a.disk_root) }}%{% endif %} + {% if a.get('cpu_pct') is not none %}cpu {{ "%.0f"|format(a.cpu_pct) }}%{% if spark %}{{ spark | safe }}{% endif %}{% endif %} + {% if a.get('mem_used_pct') is not none %}mem {{ "%.0f"|format(a.mem_used_pct) }}%{% endif %} + {% if a.get('disk_root') is not none %}disk / {{ "%.0f"|format(a.disk_root) }}%{% endif %} {% if not a.fresh %}stale{% endif %} {% else %} no agent diff --git a/steward/templates/hosts/uptime_widget.html b/steward/templates/hosts/uptime_widget.html index 52675e4..fe31dc2 100644 --- a/steward/templates/hosts/uptime_widget.html +++ b/steward/templates/hosts/uptime_widget.html @@ -26,14 +26,11 @@ {% if pct is none %} — - {% elif pct >= 99.9 %} - {{ "%.1f"|format(pct) }}% - {% elif pct >= 99 %} - {{ "%.1f"|format(pct) }}% - {% elif pct >= 95 %} - {{ "%.1f"|format(pct) }}% {% else %} - {{ "%.1f"|format(pct) }}% + {# Configurable uptime thresholds (Settings → Thresholds); a normal value + (no degraded style) stays green to read as healthy at a glance. #} + {% set us = threshold_style(pct, 'uptime') %} + {{ "%.1f"|format(pct) }}% {% endif %} {% endfor %} diff --git a/steward/templates/ping/rows.html b/steward/templates/ping/rows.html index c20ee06..468fe2e 100644 --- a/steward/templates/ping/rows.html +++ b/steward/templates/ping/rows.html @@ -54,11 +54,8 @@
{% if uptime_pcts is defined %} {% set pct = uptime_pcts.get(host.id) %} -
+ {% set us = threshold_style(pct, 'uptime') if pct is not none else 'color:var(--text-dim);' %} +
{% if pct is not none %}{{ "%.1f"|format(pct) }}%{% else %}—{% endif %} {{ range_key }}
diff --git a/steward/templates/settings/_tabs.html b/steward/templates/settings/_tabs.html index 252391e..65bb7fd 100644 --- a/steward/templates/settings/_tabs.html +++ b/steward/templates/settings/_tabs.html @@ -3,6 +3,7 @@
{% set tabs = [ ("general", "General", "/settings/general/"), + ("thresholds", "Thresholds", "/settings/thresholds/"), ("notifications", "Notifications", "/settings/notifications/"), ("reports", "Reports", "/settings/reports/"), ("auth", "Auth", "/settings/auth/"), diff --git a/steward/templates/settings/thresholds.html b/steward/templates/settings/thresholds.html new file mode 100644 index 0000000..1a7509e --- /dev/null +++ b/steward/templates/settings/thresholds.html @@ -0,0 +1,50 @@ +{# steward/templates/settings/thresholds.html — tunable degraded/critical cutoffs #} +{% extends "base.html" %} +{% block title %}Settings — Thresholds — Steward{% endblock %} +{% block content %} +{% set active_tab = "thresholds" %} +{% include "settings/_tabs.html" %} + +{% macro pair(label, warn_field, warn_key, crit_field, crit_key, unit, hint, step="1") %} +
+ +
+ + warn + + + + crit + + +
+ {% if hint %}
{{ hint }}
{% endif %} +
+{% endmacro %} + +
+
+

Monitoring thresholds

+

+ Values at or beyond warn show amber; at or beyond + crit show red — across dashboard widgets and host views. + Higher is worse for everything except uptime, where the cutoffs are floors. +

+ + {{ pair("CPU usage", "cpu_warn", "thresholds.cpu_warn", "cpu_crit", "thresholds.cpu_crit", "%", "Average CPU utilization per host.") }} + {{ pair("Memory usage", "mem_warn", "thresholds.mem_warn", "mem_crit", "thresholds.mem_crit", "%", "Memory in use per host.") }} + {{ pair("Disk usage", "disk_warn", "thresholds.disk_warn", "disk_crit", "thresholds.disk_crit", "%", "Filesystem usage.") }} + {{ pair("Load", "load_warn", "thresholds.load_warn", "load_crit", "thresholds.load_crit", "% per core", "Load average normalized by core count (100% = one full core per core).") }} + {{ pair("Temperature", "temp_warn", "thresholds.temp_warn", "temp_crit", "thresholds.temp_crit", "°C", "Hottest reported sensor per host.") }} + {{ pair("Ping latency", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "Round-trip time. Shared with the Ping widget's good/warn colouring.") }} + {{ pair("Uptime / SLA", "uptime_warn", "thresholds.uptime_warn", "uptime_crit", "thresholds.uptime_crit", "%", "Lower is worse — amber below warn, red below crit.", step="0.1") }} +
+ +
+ + Takes effect immediately — no restart. +
+
+{% endblock %}