Files
bvandeusenandClaude Opus 5.5 f3433fe3ce
CI / lint (push) Successful in 2s
CI / unit (push) Failing after 1m26s
CI / integration (push) Failing after 1m27s
CI / publish (push) Skipped
feat(norms): hosts, containers and monitor response times are graded against their own normal (#5742)
Host CPU, memory and load per core, container CPU and memory, and every
monitor's response time learn a normal per hour (milestone 513, System 182).
A reading past its fixed cutoff but within its normal reads healthy and says
what is usual at this hour; one well out of it keeps its grade and is a
metric anomaly. Disk, a monitor down and an agent offline keep their cutoffs.

- Each normal's floor, the smallest departure that counts, is a setting:
  Settings -> Thresholds -> Normals (norms.floor.*). Defaults: host CPU 10
  points, host memory 5, load per core 10, container CPU 10 points of one
  core, container memory 5, monitor response 100 ms; Traefik's 2-point 4xx
  and 50 ms p95 floors move there from constants. learn_normal takes a
  floor function (floor_setting) read at each pass.
- learn_normal(parts=False): a host learns, its cores don't.
- Ingest records load_pct (load_1m / cores x 100), the figure pages grade.
- Shared grading: anomalies.against / graded, worst_reading(normals=),
  hosts' host_grades, docker usage(thresholds=, norms=), monitors'
  latency_grade. Applied to host vitals, the Hosts widget, MCP overview,
  Docker usage, and the monitor list, widget and page.
- The monitor and container pages list their anomalies.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 16:17:04 -04:00

146 lines
7.5 KiB
HTML

{% extends "base.html" %}
{% from "_macros.html" import crumbs, page_header, stat_tile %}
{# One monitor: its state and for how long, latency and uptime against
Settings → Thresholds (latency against its own normal once it has one,
with its anomalies), its response time over the last day, its last
checks, the failures behind them and how it is set up. #}
{% block title %}{{ monitor.name }} — Monitors — Steward{% endblock %}
{% block breadcrumb %}{{ crumbs([("Monitors", "/monitors/"), (monitor.name, "")]) }}{% endblock %}
{% block content %}
{% set can_edit = session.user_role in ['operator', 'admin'] %}
{% call page_header(monitor.name, (monitor.type|monitor_type) ~ " · " ~ monitor.target) %}
{% if can_edit %}
<a class="btn btn-sm btn-ghost" href="/monitors/{{ monitor.id }}/edit">Edit</a>
<form method="post" action="/monitors/{{ monitor.id }}/toggle">{{ csrf_field() }}
<button type="submit" class="btn btn-sm btn-ghost">{{ "Pause" if monitor.enabled else "Resume" }}</button>
</form>
<form method="post" action="/monitors/{{ monitor.id }}/delete" data-confirm="Delete monitor {{ monitor.name }}? Its check history goes with it.">{{ csrf_field() }}
<button type="submit" class="btn btn-sm btn-danger">Delete</button>
</form>
{% endif %}
{% endcall %}
<div class="stat-strip">
{% set since_hint = ((("for at least " if since_at_least else "for ") ~ (since|ago|striptags|replace(" ago", ""))) if since and state in ("up", "down", "degraded") else none) %}
{{ stat_tile("State", status_info(state)[1], state=state, hint=since_hint) }}
{% if latest and latest.response_ms is not none %}
{{ stat_tile("Latency", "%.0f ms"|format(latest.response_ms), state=latency[0], hint=latency[1]) }}
{% endif %}
{% for window in ("24h", "7d", "30d") %}
{% set pct = uptime.get(window) %}
{{ stat_tile("Uptime " ~ window, "%.2f%%"|format(pct) if pct is not none else "—", state=status_class(pct, "uptime")) }}
{% endfor %}
{% if tls_days is not none %}
{{ stat_tile("TLS certificate", "expired" if tls_days < 0 else (tls_days|round|int) ~ " days", state="crit" if tls_days < 0 else status_class(tls_days, "tls_days")) }}
{% endif %}
</div>
{% with back="/monitors/" ~ monitor.id, base_resource=monitor.name %}{% include "_anomaly_card.html" %}{% endwith %}
{% set timed = history | selectattr("ms", "ne", none) | list %}
{% if timed|length > 1 %}
<section class="card" aria-labelledby="m-chart">
<h2 id="m-chart" class="section-title">Response time · last 24h</h2>
<div style="position:relative;height:180px;" id="m-chart-box">
<canvas id="m-chart-canvas" aria-label="Response time over the last 24 hours, failed checks in red"></canvas>
</div>
<p class="field-hint">Average per {{ ((24 * 60) // 96) }} minutes, your local time. Red bars mark failed checks.</p>
</section>
<script>
(function () {
var box = document.getElementById("m-chart-box"), ctx = document.getElementById("m-chart-canvas");
if (!window.Chart) { box.innerHTML = '<p class="field-hint">The chart library didn\'t load.</p>'; return; }
var pts = {{ history | tojson }};
var hhmm = function (ms) { return new Date(ms).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" }); };
var maxMs = Math.max.apply(null, pts.map(function (p) { return p.ms || 0; }));
var chart = new Chart(ctx.getContext("2d"), {
data: {
labels: pts.map(function (p) { return p.at; }),
datasets: [
{ type: "line", label: "Response (ms)", data: pts.map(function (p) { return p.ms; }),
borderColor: stewardColor("--blue"), backgroundColor: stewardColor("--blue-dim"), fill: true,
tension: 0.3, pointRadius: 0, borderWidth: 1.5, spanGaps: false, yAxisID: "y" },
{ type: "bar", label: "Failed checks", data: pts.map(function (p) { return p.down ? maxMs : null; }),
backgroundColor: stewardColor("--red"), barPercentage: 1, categoryPercentage: 1, yAxisID: "y" },
],
},
options: {
responsive: true, maintainAspectRatio: false, animation: false,
interaction: { mode: "index", intersect: false },
plugins: {
legend: { display: false },
tooltip: { callbacks: {
title: function (items) { return items.length ? hhmm(pts[items[0].dataIndex].at) : ""; },
label: function (item) {
var p = pts[item.dataIndex];
return item.datasetIndex === 0 ? (p.ms == null ? "no response time" : p.ms + " ms")
: p.down + " failed check" + (p.down === 1 ? "" : "s");
},
} },
},
scales: {
x: { ticks: { font: { size: 10 }, maxRotation: 0, autoSkip: true, maxTicksLimit: 8,
callback: function (v) { return hhmm(pts[v].at); } }, grid: { display: false } },
y: { min: 0, ticks: { font: { size: 10 } }, title: { display: true, text: "ms", font: { size: 9 } } },
},
},
});
setTimeout(function () { if (ctx.isConnected) { chart.resize(); chart.update("none"); } }, 250);
})();
</script>
{% endif %}
<section class="card" aria-labelledby="m-checks">
<h2 id="m-checks" class="section-title">Last 30 checks</h2>
{% if heartbeat %}
<div class="ping-pills mon-pills" style="margin:0.5rem 0;">
{% for _ in range([30 - heartbeat|length, 0]|max) %}<span class="pill pill-empty" title="No data"></span>{% endfor %}
{% for h in heartbeat %}<span class="pill pill-{{ h.state }}" title="{{ h.title }}"></span>{% endfor %}
</div>
<p class="field-hint">Newest on the right; hover one for its result.{% if latest %} Last checked {{ latest.checked_at|ago }}.{% endif %}</p>
{% else %}
<p class="field-hint">{{ "Not checked yet. The first check runs on the next scheduler tick." if monitor.enabled else "Paused, and never checked." }}</p>
{% endif %}
</section>
<section class="card" aria-labelledby="m-failures">
<h2 id="m-failures" class="section-title">Recent failures</h2>
{% if failures %}
<table class="table">
<thead><tr><th>When</th><th>Error</th>{% if monitor.type == "http" %}<th>Status</th>{% endif %}</tr></thead>
<tbody>
{% for f in failures %}
<tr>
<td style="white-space:nowrap;">{{ f.checked_at|ago }}</td>
<td style="overflow-wrap:anywhere;">{{ f.error_msg or "Down" }}</td>
{% if monitor.type == "http" %}<td>{{ f.status_code or "—" }}</td>{% endif %}
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p class="field-hint">No failed checks on record.</p>
{% endif %}
</section>
<section class="card" aria-labelledby="m-config">
<h2 id="m-config" class="section-title">Settings</h2>
<dl class="kv">
<dt>Type</dt><dd>{{ monitor.type|monitor_type }}</dd>
<dt>Target</dt><dd style="overflow-wrap:anywhere;">{{ monitor.target }}</dd>
<dt>Checked every</dt>
<dd>{% if monitor.check_interval_seconds %}{{ monitor.check_interval_seconds|duration }}{% else %}{{ poll_interval|duration }} <span class="tone-dim">(the global interval, Settings)</span>{% endif %}</dd>
{% if host %}<dt>Host</dt><dd><a href="/hosts/{{ host.id }}">{{ host.name }}</a></dd>{% endif %}
{% for key, value in monitor.config.items() %}
{% if value not in (none, "", {}) %}
<dt>{{ field_label(key) }}</dt>
<dd style="overflow-wrap:anywhere;">{{ value|join(", ") if value is mapping else ("yes" if value is sameas true else ("no" if value is sameas false else value)) }}</dd>
{% endif %}
{% endfor %}
{% if monitor.type == "dns" and not monitor.config.get("resolver") %}
<dt>Resolver</dt><dd>the container's own (its resolv.conf)</dd>
{% endif %}
</dl>
</section>
{% endblock %}