feat(settings): configurable monitoring thresholds

Move the hardcoded warn/crit cutoffs into Settings -> Thresholds (DB-backed,
live, no restart). New thresholds.* keys + to_thresholds_cfg() + a
threshold_style(value, kind) jinja global that reads them; latency reuses the
existing ping good/warn keys, uptime is direction-aware (floors).

Replace the _macros metric_style/uptime_style macros (now removed) with the
global across Hosts-Overview, host_agent fleet + panel, Uptime/SLA widget, and
the ping page uptime column — all now honor the configured cutoffs. Uptime keeps
its green 'good' look when not degraded.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

steward/app.py

@@ -59,6 +59,7 @@ def create_app(
         from .core.settings import (
             load_settings_sync, to_smtp_cfg, to_webhook_cfg,
             to_ansible_cfg, to_plugins_cfg, to_oidc_cfg, to_ldap_cfg,
+            to_thresholds_cfg,
         )
         _settings = load_settings_sync(app.config["DATABASE_URL"])
         app.config.update(
@@ -71,8 +72,10 @@ def create_app(
             PLUGINS=to_plugins_cfg(_settings),
             OIDC=to_oidc_cfg(_settings),
             LDAP=to_ldap_cfg(_settings),
+            THRESHOLDS=to_thresholds_cfg(_settings),
         )
     else:
+        from .core.settings import to_thresholds_cfg
         app.config.update(
             SESSION_LIFETIME_HOURS=8,
             DATA_RETENTION_DAYS=90,
@@ -83,8 +86,17 @@ def create_app(
             PLUGINS={},
             OIDC={"enabled": False},
             LDAP={"enabled": False},
+            THRESHOLDS=to_thresholds_cfg({}),
         )
 
+    # `threshold_style(value, kind)` — degraded-value coloring for templates,
+    # reading the configurable cutoffs from app.config["THRESHOLDS"]. Available
+    # in every template (core + plugin) via the jinja global.
+    from .core.settings import threshold_style_for as _ts_for
+    app.jinja_env.globals["threshold_style"] = (
+        lambda value, kind: _ts_for(value, kind, app.config.get("THRESHOLDS", {}))
+    )
+
     # ── 5. Plugin migrations ───────────────────────────────────────────────────
     # Step 3 already applied all plugin migrations via discover_all_plugin_migration_dirs,
     # so this is a no-op on normal startup. We still run it with all discovered dirs so

steward/core/settings.py

@@ -68,6 +68,16 @@ DEFAULTS: dict[str, Any] = {
     "ansible.max_concurrent_runs": 3,
     "ping.threshold.good_ms": 50,
     "ping.threshold.warn_ms": 200,
+    # Degraded/critical cutoffs for metric coloring (warn=amber, crit=red).
+    # cpu/mem/disk/load are percentages (load is load-per-core %); temp in °C.
+    # uptime is "higher is better" so its warn/crit are floors. Ping latency
+    # reuses ping.threshold.good_ms/warn_ms above.
+    "thresholds.cpu_warn": 80, "thresholds.cpu_crit": 90,
+    "thresholds.mem_warn": 80, "thresholds.mem_crit": 90,
+    "thresholds.disk_warn": 80, "thresholds.disk_crit": 90,
+    "thresholds.load_warn": 80, "thresholds.load_crit": 100,
+    "thresholds.temp_warn": 70, "thresholds.temp_crit": 85,
+    "thresholds.uptime_warn": 99.0, "thresholds.uptime_crit": 95.0,
     "plugins.index_url": "https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml",
     # Default-enabled plugins. These are the generic, non-vendor-specific
     # bundled plugins (protocols/standards, not a single product) — useful on
@@ -224,6 +234,53 @@ def to_ldap_cfg(settings: dict[str, Any]) -> dict:
     return {k[len("ldap."):]: settings.get(k, DEFAULTS[k]) for k in DEFAULTS if k.startswith("ldap.")}
 
 
+def to_thresholds_cfg(settings: dict[str, Any]) -> dict:
+    """Per-metric (warn, crit, direction) policy for degraded-value coloring.
+
+    Centralizes "what counts as degraded" so templates just name a metric kind
+    via the threshold_style() jinja global. `dir` is "high" (higher is worse,
+    e.g. CPU) or "low" (lower is worse, e.g. uptime %). Latency reuses the
+    existing ping good/warn thresholds.
+    """
+    g = settings.get
+    return {
+        "cpu":     {"warn": g("thresholds.cpu_warn", 80),  "crit": g("thresholds.cpu_crit", 90),  "dir": "high"},
+        "mem":     {"warn": g("thresholds.mem_warn", 80),  "crit": g("thresholds.mem_crit", 90),  "dir": "high"},
+        "disk":    {"warn": g("thresholds.disk_warn", 80), "crit": g("thresholds.disk_crit", 90), "dir": "high"},
+        "load":    {"warn": g("thresholds.load_warn", 80), "crit": g("thresholds.load_crit", 100), "dir": "high"},
+        "temp":    {"warn": g("thresholds.temp_warn", 70), "crit": g("thresholds.temp_crit", 85), "dir": "high"},
+        "latency": {"warn": g("ping.threshold.good_ms", 50), "crit": g("ping.threshold.warn_ms", 200), "dir": "high"},
+        "uptime":  {"warn": g("thresholds.uptime_warn", 99.0), "crit": g("thresholds.uptime_crit", 95.0), "dir": "low"},
+    }
+
+
+def threshold_style_for(value: Any, kind: str, thresholds: dict) -> str:
+    """Return an inline-style fragment (amber at warn, red at crit) for a metric.
+
+    The Python twin of the old _macros.metric_style, but reading configurable
+    cutoffs from `thresholds` (see to_thresholds_cfg) and handling both
+    directions. Empty string when normal/unknown/None — drop straight into a
+    span's style="".
+    """
+    if value is None:
+        return ""
+    th = thresholds.get(kind)
+    if not th:
+        return ""
+    warn, crit, direction = th["warn"], th["crit"], th.get("dir", "high")
+    if direction == "low":
+        if value < crit:
+            return "color:var(--red);font-weight:700;"
+        if value < warn:
+            return "color:var(--yellow);font-weight:600;"
+    else:
+        if value >= crit:
+            return "color:var(--red);font-weight:700;"
+        if value >= warn:
+            return "color:var(--yellow);font-weight:600;"
+    return ""
+
+
 def to_plugins_cfg(settings: dict[str, Any]) -> dict:
     """Assemble {plugin_name: {...config}} from all plugin.* keys."""
     result = {}

steward/settings/routes.py

@@ -10,7 +10,7 @@ from steward.models.users import UserRole
 from steward.core.settings import (
     get_all_settings, set_setting,
     to_smtp_cfg, to_webhook_cfg, to_ansible_cfg, to_plugins_cfg,
-    to_oidc_cfg, to_ldap_cfg,
+    to_oidc_cfg, to_ldap_cfg, to_thresholds_cfg,
 )
 
 settings_bp = Blueprint("settings", __name__, url_prefix="/settings")
@@ -72,6 +72,7 @@ async def _reload_app_config() -> None:
         PLUGINS=to_plugins_cfg(fresh),
         OIDC=to_oidc_cfg(fresh),
         LDAP=to_ldap_cfg(fresh),
+        THRESHOLDS=to_thresholds_cfg(fresh),
     )
 
 
@@ -113,6 +114,49 @@ async def save_general():
     return redirect(url_for("settings.general"))
 
 
+# ── Monitoring thresholds ─────────────────────────────────────────────────────
+
+# (form field, setting key, is_float) for every tunable cutoff.
+_THRESHOLD_FIELDS = [
+    ("cpu_warn", "thresholds.cpu_warn", False), ("cpu_crit", "thresholds.cpu_crit", False),
+    ("mem_warn", "thresholds.mem_warn", False), ("mem_crit", "thresholds.mem_crit", False),
+    ("disk_warn", "thresholds.disk_warn", False), ("disk_crit", "thresholds.disk_crit", False),
+    ("load_warn", "thresholds.load_warn", False), ("load_crit", "thresholds.load_crit", False),
+    ("temp_warn", "thresholds.temp_warn", False), ("temp_crit", "thresholds.temp_crit", False),
+    ("uptime_warn", "thresholds.uptime_warn", True), ("uptime_crit", "thresholds.uptime_crit", True),
+    ("latency_warn", "ping.threshold.good_ms", False), ("latency_crit", "ping.threshold.warn_ms", False),
+]
+
+
+@settings_bp.get("/thresholds/")
+@require_role(UserRole.admin)
+async def thresholds():
+    async with current_app.db_sessionmaker() as db:
+        settings = await get_all_settings(db)
+    return await render_template("settings/thresholds.html", settings=settings)
+
+
+@settings_bp.post("/thresholds/")
+@require_role(UserRole.admin)
+async def save_thresholds():
+    form = await request.form
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            for field, key, is_float in _THRESHOLD_FIELDS:
+                raw = form.get(field, "")
+                if raw == "":
+                    continue
+                try:
+                    val = float(raw) if is_float else int(raw)
+                except (TypeError, ValueError):
+                    continue
+                await set_setting(db, key, val)
+    await _reload_app_config()
+    await log_audit(current_app, session.get("user_id"), session.get("username", ""),
+                    "settings.saved", detail={"section": "thresholds"})
+    return redirect(url_for("settings.thresholds"))
+
+
 # ── Notifications (SMTP + Webhook) ────────────────────────────────────────────
 
 @settings_bp.get("/notifications/")

steward/templates/_macros.html

@@ -17,24 +17,6 @@
 </nav>
 {%- endmacro %}
 
-{# Threshold emphasis for a numeric metric (CPU/mem/disk %, etc).
-   Returns an inline-style fragment to drop into a span's style="": amber +
-   semibold at >= warn, red + bold at >= crit, empty (inherit) when the value
-   is normal or None. This makes the FAILING METRIC ITSELF stand out — not just
-   the status dot. Defaults suit percentage gauges (warn 80 / crit 90); pass
-   explicit warn/crit for other scales. Keep thresholds here so they live in one
-   place rather than hardcoded per template. #}
-{% macro metric_style(value, warn=80, crit=90) -%}
-{%- if value is not none and value >= crit -%}color:var(--red);font-weight:700;
-{%- elif value is not none and value >= warn -%}color:var(--yellow);font-weight:600;
-{%- endif -%}
-{%- endmacro %}
-
-{# Inverted emphasis for "higher is better" metrics like uptime %: amber below
-   `warn`, red below `crit`. The mirror of metric_style — same one-place-for-
-   thresholds discipline, opposite direction. Defaults suit SLA percentages. #}
-{% macro uptime_style(value, warn=99.0, crit=95.0) -%}
-{%- if value is not none and value < crit -%}color:var(--red);font-weight:700;
-{%- elif value is not none and value < warn -%}color:var(--yellow);font-weight:600;
-{%- endif -%}
-{%- endmacro %}
+{# Metric threshold coloring moved to the `threshold_style(value, kind)` jinja
+   global (steward/core/settings.py + app.py), which reads operator-configurable
+   cutoffs from Settings → Thresholds. Call it as threshold_style(v, 'cpu'). #}

steward/templates/hosts/overview_widget.html

@@ -1,7 +1,6 @@
 {# Unified Hosts widget — monitor status + agent glance per host, links to the hub.
    Name sits on its own line (never truncated); its data wraps beneath it, grouped
    under the host. Prefer vertical growth over cramming a row. #}
-{% from "_macros.html" import metric_style, uptime_style %}
 {% if hosts %}
 <div class="host-blocks">
   {% for host in hosts %}
@@ -27,15 +26,15 @@
     <div style="display:flex;flex-wrap:wrap;gap:0.25rem 0.9rem;margin-top:0.25rem;padding-left:1.1rem;
                 font-variant-numeric:tabular-nums;color:var(--text-muted);font-size:0.75rem;">
       {% if host.ping_enabled and ping and ping.response_time_ms is not none %}
-      <span title="Ping latency" style="{{ metric_style(ping.response_time_ms, warn=100, crit=250) }}"><span style="color:var(--text-dim);">ping</span> {{ "%.0f"|format(ping.response_time_ms) }}ms</span>
+      <span title="Ping latency" style="{{ threshold_style(ping.response_time_ms, 'latency') }}"><span style="color:var(--text-dim);">ping</span> {{ "%.0f"|format(ping.response_time_ms) }}ms</span>
       {% endif %}
       {% if ut.get('24h') is not none %}
-      <span title="Uptime 24h" style="{{ uptime_style(ut['24h']) }}"><span style="color:var(--text-dim);">24h</span> {{ "%.1f"|format(ut['24h']) }}%</span>
+      <span title="Uptime 24h" style="{{ threshold_style(ut['24h'], 'uptime') }}"><span style="color:var(--text-dim);">24h</span> {{ "%.1f"|format(ut['24h']) }}%</span>
       {% endif %}
       {% if a %}
-      {% if a.get('cpu_pct') is not none %}<span title="CPU — value + last-hour trend" style="display:inline-flex;align-items:center;gap:0.3rem;{{ metric_style(a.cpu_pct) }}"><span style="color:var(--text-dim);">cpu</span> {{ "%.0f"|format(a.cpu_pct) }}%{% if spark %}<span style="line-height:0;opacity:0.85;">{{ spark | safe }}</span>{% endif %}</span>{% endif %}
-      {% if a.get('mem_used_pct') is not none %}<span title="Memory" style="{{ metric_style(a.mem_used_pct) }}"><span style="color:var(--text-dim);">mem</span> {{ "%.0f"|format(a.mem_used_pct) }}%</span>{% endif %}
-      {% if a.get('disk_root') is not none %}<span title="Root filesystem (/)" style="{{ metric_style(a.disk_root) }}"><span style="color:var(--text-dim);">disk /</span> {{ "%.0f"|format(a.disk_root) }}%</span>{% endif %}
+      {% if a.get('cpu_pct') is not none %}<span title="CPU — value + last-hour trend" style="display:inline-flex;align-items:center;gap:0.3rem;{{ threshold_style(a.cpu_pct, 'cpu') }}"><span style="color:var(--text-dim);">cpu</span> {{ "%.0f"|format(a.cpu_pct) }}%{% if spark %}<span style="line-height:0;opacity:0.85;">{{ spark | safe }}</span>{% endif %}</span>{% endif %}
+      {% if a.get('mem_used_pct') is not none %}<span title="Memory" style="{{ threshold_style(a.mem_used_pct, 'mem') }}"><span style="color:var(--text-dim);">mem</span> {{ "%.0f"|format(a.mem_used_pct) }}%</span>{% endif %}
+      {% if a.get('disk_root') is not none %}<span title="Root filesystem (/)" style="{{ threshold_style(a.disk_root, 'disk') }}"><span style="color:var(--text-dim);">disk /</span> {{ "%.0f"|format(a.disk_root) }}%</span>{% endif %}
       {% if not a.fresh %}<span title="Agent data is stale" style="color:var(--yellow);">stale</span>{% endif %}
       {% else %}
       <span style="color:var(--text-dim);" title="No agent reporting">no agent</span>

steward/templates/hosts/uptime_widget.html

@@ -26,14 +26,11 @@
     <span style="text-align:center;font-family:ui-monospace,monospace;font-size:0.78rem;">
       {% if pct is none %}
         <span style="color:var(--text-dim);">—</span>
-      {% elif pct >= 99.9 %}
-        <span style="color:var(--green);">{{ "%.1f"|format(pct) }}%</span>
-      {% elif pct >= 99 %}
-        <span style="color:var(--yellow);">{{ "%.1f"|format(pct) }}%</span>
-      {% elif pct >= 95 %}
-        <span style="color:var(--orange);">{{ "%.1f"|format(pct) }}%</span>
       {% else %}
-        <span style="color:var(--red);">{{ "%.1f"|format(pct) }}%</span>
+        {# Configurable uptime thresholds (Settings → Thresholds); a normal value
+           (no degraded style) stays green to read as healthy at a glance. #}
+        {% set us = threshold_style(pct, 'uptime') %}
+        <span style="{{ us if us else 'color:var(--green);' }}">{{ "%.1f"|format(pct) }}%</span>
       {% endif %}
     </span>
     {% endfor %}

steward/templates/ping/rows.html

@@ -54,11 +54,8 @@
   </div>
   {% if uptime_pcts is defined %}
   {% set pct = uptime_pcts.get(host.id) %}
-  <div style="min-width:4.5rem;text-align:right;font-size:0.78rem;font-variant-numeric:tabular-nums;color:
-    {% if pct is none %}var(--text-dim)
-    {% elif pct >= 99 %}var(--green)
-    {% elif pct >= 95 %}var(--yellow)
-    {% else %}var(--red){% endif %};">
+  {% set us = threshold_style(pct, 'uptime') if pct is not none else 'color:var(--text-dim);' %}
+  <div style="min-width:4.5rem;text-align:right;font-size:0.78rem;font-variant-numeric:tabular-nums;{{ us if us else 'color:var(--green);' }}">
     {% if pct is not none %}{{ "%.1f"|format(pct) }}%{% else %}—{% endif %}
     <span style="color:var(--text-dim);font-size:0.7rem;margin-left:0.15rem;">{{ range_key }}</span>
   </div>

steward/templates/settings/_tabs.html

@@ -3,6 +3,7 @@
 <div style="display:flex;gap:0;border-bottom:1px solid var(--border-mid);margin-bottom:1.5rem;">
   {% set tabs = [
     ("general",       "General",        "/settings/general/"),
+    ("thresholds",    "Thresholds",     "/settings/thresholds/"),
     ("notifications", "Notifications",  "/settings/notifications/"),
     ("reports",       "Reports",        "/settings/reports/"),
     ("auth",          "Auth",           "/settings/auth/"),

steward/templates/settings/thresholds.html

@@ -0,0 +1,50 @@
+{# steward/templates/settings/thresholds.html — tunable degraded/critical cutoffs #}
+{% extends "base.html" %}
+{% block title %}Settings — Thresholds — Steward{% endblock %}
+{% block content %}
+{% set active_tab = "thresholds" %}
+{% include "settings/_tabs.html" %}
+
+{% macro pair(label, warn_field, warn_key, crit_field, crit_key, unit, hint, step="1") %}
+<div class="form-group" style="margin-bottom:1.1rem;">
+  <label>{{ label }} <span style="color:var(--text-muted);font-size:0.8rem;">({{ unit }})</span></label>
+  <div style="display:flex;gap:1.25rem;flex-wrap:wrap;margin-top:0.25rem;">
+    <span style="display:flex;align-items:center;gap:0.4rem;">
+      <span style="color:var(--yellow);font-size:0.8rem;font-weight:600;">warn</span>
+      <input type="number" name="{{ warn_field }}" step="{{ step }}"
+             value="{{ settings[warn_key] }}" style="max-width:110px;">
+    </span>
+    <span style="display:flex;align-items:center;gap:0.4rem;">
+      <span style="color:var(--red);font-size:0.8rem;font-weight:600;">crit</span>
+      <input type="number" name="{{ crit_field }}" step="{{ step }}"
+             value="{{ settings[crit_key] }}" style="max-width:110px;">
+    </span>
+  </div>
+  {% if hint %}<div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">{{ hint }}</div>{% endif %}
+</div>
+{% endmacro %}
+
+<form method="post" action="/settings/thresholds/">
+<div class="card" style="max-width:640px;">
+  <h2 class="section-title" style="margin-bottom:0.5rem;">Monitoring thresholds</h2>
+  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
+    Values at or beyond <span style="color:var(--yellow);">warn</span> show amber; at or beyond
+    <span style="color:var(--red);">crit</span> show red — across dashboard widgets and host views.
+    Higher is worse for everything except uptime, where the cutoffs are floors.
+  </p>
+
+  {{ pair("CPU usage",    "cpu_warn",  "thresholds.cpu_warn",  "cpu_crit",  "thresholds.cpu_crit",  "%", "Average CPU utilization per host.") }}
+  {{ pair("Memory usage", "mem_warn",  "thresholds.mem_warn",  "mem_crit",  "thresholds.mem_crit",  "%", "Memory in use per host.") }}
+  {{ pair("Disk usage",   "disk_warn", "thresholds.disk_warn", "disk_crit", "thresholds.disk_crit", "%", "Filesystem usage.") }}
+  {{ pair("Load",         "load_warn", "thresholds.load_warn", "load_crit", "thresholds.load_crit", "% per core", "Load average normalized by core count (100% = one full core per core).") }}
+  {{ pair("Temperature",  "temp_warn", "thresholds.temp_warn", "temp_crit", "thresholds.temp_crit", "°C", "Hottest reported sensor per host.") }}
+  {{ pair("Ping latency", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "Round-trip time. Shared with the Ping widget's good/warn colouring.") }}
+  {{ pair("Uptime / SLA", "uptime_warn", "thresholds.uptime_warn", "uptime_crit", "thresholds.uptime_crit", "%", "Lower is worse — amber below warn, red below crit.", step="0.1") }}
+</div>
+
+<div style="margin-top:1rem;display:flex;align-items:center;gap:1rem;">
+  <button type="submit" class="btn">Save</button>
+  <span style="font-size:0.82rem;color:var(--text-muted);">Takes effect immediately — no restart.</span>
+</div>
+</form>
+{% endblock %}