FabledSteward/steward/templates/settings/thresholds.html

{# steward/templates/settings/thresholds.html — tunable degraded/critical cutoffs #}
{% extends "base.html" %}
{% from "_macros.html" import crumbs %}
{% block title %}Settings — Thresholds & Retention — Steward{% endblock %}
{% from "settings/_tabs.html" import settings_tabs %}
{% block section_tabs %}{{ settings_tabs("thresholds") }}{% endblock %}
{% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("Thresholds & retention", "")]) }}{% endblock %}
{% block content %}

{% macro pair(label, warn_field, warn_key, crit_field, crit_key, unit, hint, step="1") %}
<div class="form-group" style="margin-bottom:1.1rem;">
  <label>{{ label }} <span style="color:var(--text-muted);font-size:0.8rem;">({{ unit }})</span></label>
  <div style="display:flex;gap:1.25rem;flex-wrap:wrap;margin-top:0.25rem;">
    <span style="display:flex;align-items:center;gap:0.4rem;">
      <span style="color:var(--yellow);font-size:0.8rem;font-weight:500;">warn</span>
      <input type="number" name="{{ warn_field }}" step="{{ step }}"
             value="{{ settings[warn_key] }}" style="max-width:110px;">
    </span>
    <span style="display:flex;align-items:center;gap:0.4rem;">
      <span style="color:var(--red);font-size:0.8rem;font-weight:500;">crit</span>
      <input type="number" name="{{ crit_field }}" step="{{ step }}"
             value="{{ settings[crit_key] }}" style="max-width:110px;">
    </span>
  </div>
  {% if hint %}<div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">{{ hint }}</div>{% endif %}
</div>
{% endmacro %}

<form method="post" action="/settings/thresholds/">{{ csrf_field() }}
<div class="card" style="max-width:640px;">
  <h2 class="section-title" style="margin-bottom:0.5rem;">Monitoring thresholds</h2>
  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
    Values at or beyond <span style="color:var(--yellow);">warn</span> show amber; at or beyond
    <span style="color:var(--red);">crit</span> show red — across dashboard widgets and host views.
    Higher is worse for everything except uptime, where the cutoffs are floors.
  </p>

  {{ pair("CPU usage",    "cpu_warn",  "thresholds.cpu_warn",  "cpu_crit",  "thresholds.cpu_crit",  "%", "Average CPU utilization per host. A host or container with its own normal is judged against that instead (see Normals below).") }}
  {{ pair("Memory usage", "mem_warn",  "thresholds.mem_warn",  "mem_crit",  "thresholds.mem_crit",  "%", "Memory in use per host, or a container's share of its limit. Judged against a normal once there is one.") }}
  {{ pair("Disk usage",   "disk_warn", "thresholds.disk_warn", "disk_crit", "thresholds.disk_crit", "%", "Filesystem usage.") }}
  {{ pair("Load",         "load_warn", "thresholds.load_warn", "load_crit", "thresholds.load_crit", "% per core", "Load average normalized by core count (100% = one full core per core). Judged against a host's normal once it has one.") }}
  {{ pair("Temperature",  "temp_warn", "thresholds.temp_warn", "temp_crit", "thresholds.temp_crit", "°C", "Hottest reported sensor per host.") }}
  {{ pair("Response time", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "How long a monitor's check takes to answer. Also grades Traefik routers' p95 and the UniFi WAN round trip. A monitor or router with its own normal is judged against that instead.") }}
  {{ pair("Uptime / SLA", "uptime_warn", "thresholds.uptime_warn", "uptime_crit", "thresholds.uptime_crit", "%", "Lower is worse — amber below warn, red below crit.", step="0.1") }}
  {{ pair("TLS certificate expiry", "tls_warn", "thresholds.tls_warn_days", "tls_crit", "thresholds.tls_crit_days", "days left", "HTTPS monitors whose certificate expires within these many days are flagged. Lower is worse.") }}
  {{ pair("Clock drift", "clock_warn", "thresholds.clock_warn_s", "clock_crit", "thresholds.clock_crit_s", "seconds", "How far a host's clock may be from Steward's, either way, before its page and the Hosts list call it out.") }}

  <div class="form-group" style="margin-bottom:0.25rem;">
    <label>Container crash loop <span style="color:var(--text-muted);font-size:0.8rem;">(deaths per hour)</span></label>
    <div style="margin-top:0.25rem;">
      <input type="number" name="crash_loop_deaths" min="1" step="1"
             value="{{ settings['thresholds.crash_loop_deaths'] }}" style="max-width:110px;">
    </div>
    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">
      A container that dies (or is OOM-killed) this many times within an hour is reported as crash-looping.
    </div>
  </div>
</div>

{# Traefik's router cutoffs, while the plugin runs; its page links here (#5380). #}
{% if traefik_on %}
<div class="card" id="traefik" style="max-width:640px;margin-top:1.25rem;">
  <h2 class="section-title" style="margin-bottom:0.5rem;">Traefik routers</h2>
  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
    How much of a router's traffic, over the last 15 minutes, may end in an error before the
    <a href="/plugins/traefik/">Traefik page</a> and the Routers widget list it.
  </p>
  <div class="form-group" style="margin-bottom:1.1rem;">
    <label for="error_5xx_pct">Server errors (5xx) <span style="color:var(--text-muted);font-size:0.8rem;">(% of responses)</span></label>
    <input type="number" id="error_5xx_pct" name="error_5xx_pct" min="0" max="100" step="0.01"
           value="{{ settings['traefik.threshold.error_5xx_pct'] }}" style="max-width:110px;margin-top:0.25rem;">
    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">Above this share, a router is failing (red), whatever its history.</div>
  </div>
  <div class="form-group" style="margin-bottom:0.25rem;">
    <label for="error_4xx_pct">Client errors (4xx) <span style="color:var(--text-muted);font-size:0.8rem;">(% of responses)</span></label>
    <input type="number" id="error_4xx_pct" name="error_4xx_pct" min="0" max="100" step="0.1"
           value="{{ settings['traefik.threshold.error_4xx_pct'] }}" style="max-width:110px;margin-top:0.25rem;">
    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">Above this share, a router is worth a look (amber): missing routes, failed sign-ins.
      Once a router has 5 days of history it is judged against its own normal for the hour instead, as is its p95,
      so one that always answers some 4xx stops being listed.</div>
  </div>
  <div class="form-group" style="margin:1.1rem 0 0.25rem;">
    <label for="min_requests">Fewest requests to judge 4xx and 5xx <span style="color:var(--text-muted);font-size:0.8rem;">(in 15 minutes)</span></label>
    <input type="number" id="min_requests" name="min_requests" min="0" step="1"
           value="{{ settings['traefik.threshold.min_requests'] }}" style="max-width:110px;margin-top:0.25rem;">
    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">Below this, a router's 4xx and 5xx shares aren't judged, on its page or as an anomaly: one 404 of three requests is 33%, two 500s of sixteen 12%.
      A quiet router that fails every request is left to its uptime monitor until it has this many.</div>
  </div>
  <div class="form-group" style="margin:1.1rem 0 0.25rem;">
    <label for="min_5xx">Fewest server errors to fail <span style="color:var(--text-muted);font-size:0.8rem;">(5xx in 15 minutes)</span></label>
    <input type="number" id="min_5xx" name="min_5xx" min="1" step="1"
           value="{{ settings['traefik.threshold.min_5xx'] }}" style="max-width:110px;margin-top:0.25rem;">
    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">A router past the 5xx share above is failing only once it has answered this many 5xx: one 502 among 70 requests is 1.4%, but it is one error.</div>
  </div>
  <div class="form-group" style="margin:1.1rem 0 0.25rem;">
    <label for="min_4xx">Fewest client errors worth a look <span style="color:var(--text-muted);font-size:0.8rem;">(4xx in 15 minutes)</span></label>
    <input type="number" id="min_4xx" name="min_4xx" min="1" step="1"
           value="{{ settings['traefik.threshold.min_4xx'] }}" style="max-width:110px;margin-top:0.25rem;">
    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">A router's 4xx share is flagged, on its page or as an anomaly, only once it has answered this many 4xx: a few 404s and failed sign-ins are part of a normal day; a lot of them is what needs acting on.</div>
  </div>
</div>
{% endif %}

{% macro days(label, field, key, hint) %}
<div class="form-group" style="margin-bottom:1.1rem;">
  <label>{{ label }} <span style="color:var(--text-muted);font-size:0.8rem;">(days)</span></label>
  <div style="margin-top:0.25rem;">
    <input type="number" name="{{ field }}" min="1" step="1"
           value="{{ settings[key] }}" style="max-width:110px;">
  </div>
  {% if hint %}<div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">{{ hint }}</div>{% endif %}
</div>
{% endmacro %}

{# Each normal's floor (#5742): the smallest departure from it worth calling an anomaly. #}
{% macro floor(label, key, unit, hint, step="1") %}
<div class="form-group" style="margin-bottom:1.1rem;">
  <label for="{{ key }}">{{ label }} <span style="color:var(--text-muted);font-size:0.8rem;">({{ unit }})</span></label>
  <input type="number" id="{{ key }}" name="{{ key }}" min="0" step="{{ step }}"
         value="{{ settings[key] }}" style="max-width:110px;margin-top:0.25rem;display:block;">
  {% if hint %}<div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">{{ hint }}</div>{% endif %}
</div>
{% endmacro %}

<div class="card" id="normals" style="max-width:640px;margin-top:1.5rem;">
  <h2 class="section-title" style="margin-bottom:0.5rem;">Normals</h2>
  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
    Once one has 5 days of history, these learn what is normal for each host, container, monitor or router
    at each hour, and are judged against that instead of the cutoffs above: a reading over a cutoff but usual
    for that hour shows healthy, and one well above its usual level is a warning and an anomaly to review.
    Disk space, a monitor that is down, an agent that stopped reporting and a router's 5xx share are failures
    and always keep their cutoffs. A reading counts as out of its normal only when it is a quarter above its
    usual level <em>and</em> at least this much above it:
  </p>
  {% if "host_agent" in (plugins_on or ()) %}
  {{ floor("Host CPU", "norms.floor.host_cpu_pct", "% points", "CPU is bursty: a build or a backup moves it this much without anything being wrong.") }}
  {{ floor("Host memory", "norms.floor.host_mem_pct", "% points", "Memory moves slowly, so a smaller step already means something changed.") }}
  {{ floor("Host load per core", "norms.floor.host_load_pct", "% points", "Bursty, like CPU.") }}
  {% endif %}
  {% if "docker" in (plugins_on or ()) %}
  {{ floor("Container CPU", "norms.floor.container_cpu_pct", "% of one core", "10 is a tenth of a core; with many containers, smaller steps are everyday noise.") }}
  {{ floor("Container memory", "norms.floor.container_mem_pct", "% points", "Of its limit, or of the host's memory when it has none, so a container without a limit only counts large growth.") }}
  {% endif %}
  {{ floor("Monitor response time", "norms.floor.monitor_ms", "ms", "One check per interval is noisy; a slower reply than this over its usual is worth a look.") }}
  {% if traefik_on %}
  {{ floor("Router 4xx share", "norms.floor.router_4xx_pct", "% points", "", step="0.1") }}
  {{ floor("Router p95", "norms.floor.router_p95_ms", "ms", "") }}
  {% endif %}
</div>

<div class="card" style="max-width:640px;margin-top:1.5rem;">
  <h2 class="section-title" style="margin-bottom:0.5rem;">Metric anomalies</h2>
  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
    A metric that leaves what is normal for it at that hour is listed under
    Alerts → Anomalies to review. One nobody marks normal or real within this
    window, once it has ended, lapses: it leaves the list and is kept as not
    reviewed, and its hours are never learned.
  </p>
  {{ days("Review window", "anomalies_review_days", "anomalies.review_days",
          "An anomaly still going never lapses.") }}
</div>

<div class="card" style="max-width:640px;margin-top:1.5rem;">
  <h2 class="section-title" style="margin-bottom:0.5rem;">Docker lifecycle events</h2>
  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
    How long container start, stop, die and health changes are kept. Container CPU
    and memory are kept with the other metrics, under Metric retention below.
    Applied by the hourly cleanup task — a change takes effect on its next run.
  </p>

  {{ days("Lifecycle events", "docker_events_days", "docker.retention.events_days",
          "Keep container start/stop/die/health history this long.") }}
</div>

<div class="card" style="max-width:640px;margin-top:1rem;">
  <h2 class="section-title" style="margin-bottom:0.5rem;">Container logs</h2>
  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
    The host agent tails each container's logs and pushes them here, viewable per
    container. On by default for every container; storage is bounded by a
    per-container ring (oldest lines rotate out once the age or size cap is hit,
    whichever comes first). Applied by the hourly cleanup and on ingest.
  </p>

  <div class="form-group" style="margin-bottom:1.1rem;">
    <label style="display:flex;align-items:center;gap:0.5rem;cursor:pointer;">
      <input type="checkbox" name="docker_logs_enabled" style="width:auto;"
             {% if settings["docker.logs.enabled"] %}checked{% endif %}>
      Collect container logs
    </label>
    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">
      Global kill-switch. When off, pushed log lines are dropped and no new logs are stored.
    </div>
  </div>

  <div class="form-group" style="margin-bottom:1.1rem;">
    <label>Exclude containers</label>
    <div style="margin-top:0.25rem;">
      <input type="text" name="docker_logs_exclude"
             value="{{ settings['docker.logs.exclude'] | join(', ') }}"
             placeholder="watchtower, some-chatty-service" style="width:100%;max-width:420px;">
    </div>
    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">
      Comma-separated container names whose logs are dropped on ingest (e.g. known-noisy ones).
    </div>
  </div>

  {{ days("Log retention", "docker_logs_retention_days", "docker.logs.retention_days",
          "Keep each container's log lines at most this long before rotating them out.") }}

  <div class="form-group" style="margin-bottom:0.25rem;">
    <label>Max size per container <span style="color:var(--text-muted);font-size:0.8rem;">(MB)</span></label>
    <div style="margin-top:0.25rem;">
      <input type="number" name="docker_logs_max_mb" min="1" step="1"
             value="{{ (settings['docker.logs.max_bytes_per_container'] // 1000000) or 1 }}"
             style="max-width:110px;">
    </div>
    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">
      Newest lines are kept up to this size per container; older lines rotate out first.
    </div>
  </div>
</div>

<div class="card" style="max-width:640px;margin-top:1rem;">
  <h2 class="section-title" style="margin-bottom:0.5rem;">Metric retention</h2>
  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
    Bounds how much metric history is stored: hosts (CPU, memory, disk, network,
    temps, …), containers (CPU and memory), monitors and the other sources. Raw
    per-sample points are kept for the raw window, then rolled up into hourly
    figures (average, peak, and the level held) kept for the rollup window. Charts
    read raw for the recent part of a range and hourly for older data. Applied by
    the hourly cleanup.
  </p>

  {{ days("Raw metrics", "metrics_raw_days", "metrics.retention.raw_days",
          "Keep per-sample metrics this long, then roll up to hourly.") }}
  {{ days("Rolled-up metrics", "metrics_rollup_days", "metrics.retention.rollup_days",
          "Keep the hourly-averaged series this long for multi-week history.") }}
</div>

<div style="margin-top:1rem;display:flex;align-items:center;gap:1rem;">
  <button type="submit" class="btn">Save</button>
  <span style="font-size:0.82rem;color:var(--text-muted);">Takes effect immediately — no restart.</span>
</div>
</form>
{% endblock %}