diff --git a/steward/monitors/routes.py b/steward/monitors/routes.py index ab1544b..5617b82 100644 --- a/steward/monitors/routes.py +++ b/steward/monitors/routes.py @@ -114,6 +114,7 @@ async def detail(monitor_id: str): return redirect(url_for("monitors.index")) st = (await monitor_svc.monitor_states(db, [m]))[m.id] failures = await monitor_svc.recent_failures(db, m.id) + history = await monitor_svc.response_history(db, m.id, datetime.now(timezone.utc)) host = await db.get(Host, m.host_id) if m.host_id else None thresholds = current_app.config.get("THRESHOLDS", {}) latest = st["latest"] @@ -129,6 +130,7 @@ async def detail(monitor_id: str): since=since, since_at_least=since_at_least, uptime=st["uptime"], tls_days=tls_days, heartbeat=[{"state": "up" if r.is_up else "down", "title": _heartbeat_title(m.type, r)} for r in st["recent"]], + history=history, poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60), ) diff --git a/steward/monitors/service.py b/steward/monitors/service.py index 17626fc..f3efc53 100644 --- a/steward/monitors/service.py +++ b/steward/monitors/service.py @@ -8,7 +8,7 @@ from __future__ import annotations import json from collections.abc import Mapping -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from sqlalchemy import func, select @@ -255,6 +255,26 @@ async def monitor_states(db, monitors: list[Monitor]) -> dict[str, dict]: return out +async def response_history(db, monitor_id: str, now: datetime, hours: int = 24, + points: int = 96) -> list[dict]: + """The monitor's response time and failed checks over the last `hours`, + in about `points` buckets, oldest first: [{at (epoch ms), ms, down}]. + `ms` is the bucket's average (None when no check returned a time, as for + DNS); `down` counts the failed checks in it.""" + width = max(60, hours * 3600 // points) + bucket = func.date_bin(func.make_interval(0, 0, 0, 0, 0, 0, width), MonitorResult.checked_at, + func.to_timestamp(0)).label("bucket") + rows = (await db.execute( + select(bucket, func.avg(MonitorResult.response_ms), + func.count().filter(MonitorResult.is_up.is_(False))) + .where(MonitorResult.monitor_id == monitor_id, + MonitorResult.checked_at >= now - timedelta(hours=hours)) + .group_by(bucket).order_by(bucket) + )).all() + return [{"at": int(at.timestamp() * 1000), "ms": None if ms is None else round(float(ms), 1), + "down": int(down)} for at, ms, down in rows] + + async def recent_failures(db, monitor_id: str, limit: int = 10) -> list[MonitorResult]: """The monitor's latest failed checks, newest first.""" return list((await db.execute( diff --git a/steward/templates/monitors/_fields.html b/steward/templates/monitors/_fields.html index 5e3a1de..372c19c 100644 --- a/steward/templates/monitors/_fields.html +++ b/steward/templates/monitors/_fields.html @@ -2,6 +2,16 @@ script (mtoggle) shows only the ones relevant to the selected type. #} {% macro type_fields(config={}) %} +{# What each type checks, shown under the row for the selected type. #} +

+ Ping sends an ICMP echo to the target, a hostname or IP. Up when it answers; the round trip is the latency.

+

+ TCP opens a connection to the target on the port below (22 for SSH, 443 for HTTPS, …). Up when the port accepts; the connect time is the latency. Nothing is sent.

+

+ HTTP requests the target URL (http:// or https://). Up when the status matches, and the page contains the text below if you set one. For https, the certificate's expiry is tracked too.

+

+ DNS looks the target name up (google.com, nas.lan, …) through the resolver Steward's container uses. Up when it resolves, and to the expected IP if you set one. No response time is recorded.

+ {# tcp #}
diff --git a/steward/templates/monitors/detail.html b/steward/templates/monitors/detail.html index 2e7162e..ab0d60e 100644 --- a/steward/templates/monitors/detail.html +++ b/steward/templates/monitors/detail.html @@ -1,25 +1,26 @@ {% extends "base.html" %} -{% from "_macros.html" import crumbs, page_header, stat_tile, status_badge, icon_button, empty_state %} -{# One monitor: its state and since when, latency and uptime against Settings → - Thresholds, its last checks, and the failures behind them. #} +{% from "_macros.html" import crumbs, page_header, stat_tile %} +{# One monitor: its state and for how long, latency and uptime against + Settings → Thresholds, its response time over the last day, its last + checks, the failures behind them and how it is set up. #} {% block title %}{{ monitor.name }} — Monitors — Steward{% endblock %} {% block breadcrumb %}{{ crumbs([("Monitors", "/monitors/"), (monitor.name, "")]) }}{% endblock %} {% block content %} {% set can_edit = session.user_role in ['operator', 'admin'] %} {% call page_header(monitor.name, (monitor.type|monitor_type) ~ " · " ~ monitor.target) %} {% if can_edit %} - {{ icon_button("✎", "Edit " ~ monitor.name, href="/monitors/" ~ monitor.id ~ "/edit") }} + Edit
{{ csrf_field() }} - {{ icon_button("⏸" if monitor.enabled else "▶", ("Pause " if monitor.enabled else "Resume ") ~ monitor.name, type="submit") }} +
-
{{ csrf_field() }} - {{ icon_button("×", "Delete " ~ monitor.name, type="submit", cls="btn-danger", confirm="Delete monitor " ~ monitor.name ~ "? Its check history goes with it.") }} + {{ csrf_field() }} +
{% endif %} {% endcall %}
- {% set since_hint = ((("for at least " if since_at_least else "since ") ~ since|ago) if since and state in ("up", "down", "degraded") else none) %} + {% set since_hint = ((("for at least " if since_at_least else "for ") ~ (since|ago|striptags|replace(" ago", ""))) if since and state in ("up", "down", "degraded") else none) %} {{ stat_tile("State", status_info(state)[1], state=state, hint=since_hint) }} {% if latest and latest.response_ms is not none %} {{ stat_tile("Latency", "%.0f ms"|format(latest.response_ms), state=status_class(latest.response_ms, "latency")) }} @@ -33,16 +34,69 @@ {% endif %}
+{% set timed = history | selectattr("ms", "ne", none) | list %} +{% if timed|length > 1 %} +
+

Response time · last 24h

+
+ +
+

Average per {{ ((24 * 60) // 96) }} minutes, your local time. Red bars mark failed checks.

+
+ +{% endif %} +
-

Last checks

+

Last 30 checks

{% if heartbeat %} -
+
{% for _ in range([30 - heartbeat|length, 0]|max) %}{% endfor %} {% for h in heartbeat %}{% endfor %}
-

Newest on the right.{% if latest %} Last checked {{ latest.checked_at|ago }}.{% endif %}

+

Newest on the right; hover one for its result.{% if latest %} Last checked {{ latest.checked_at|ago }}.{% endif %}

{% else %} - {{ empty_state("Not checked yet. The first check runs on the next scheduler tick." if monitor.enabled else "Paused, and never checked.") }} +

{{ "Not checked yet. The first check runs on the next scheduler tick." if monitor.enabled else "Paused, and never checked." }}

{% endif %}
@@ -62,19 +116,21 @@ {% else %} - {{ empty_state("No failed checks on record.") }} +

No failed checks on record.

{% endif %}

Settings

+
Type
{{ monitor.type|monitor_type }}
+
Target
{{ monitor.target }}
Checked every
-
{{ monitor.check_interval_seconds|duration if monitor.check_interval_seconds else "the global interval" }}
+
{% if monitor.check_interval_seconds %}{{ monitor.check_interval_seconds|duration }}{% else %}{{ poll_interval|duration }} (the global interval, Settings){% endif %}
{% if host %}
Host
{{ host.name }}
{% endif %} {% for key, value in monitor.config.items() %} {% if value not in (none, "", {}) %} -
{{ key|replace("_", " ")|capitalize }}
+
{{ field_label(key) }}
{{ value|join(", ") if value is mapping else ("yes" if value is sameas true else ("no" if value is sameas false else value)) }}
{% endif %} {% endfor %} diff --git a/steward/templates/monitors/form.html b/steward/templates/monitors/form.html index 6f2c403..97a7dc3 100644 --- a/steward/templates/monitors/form.html +++ b/steward/templates/monitors/form.html @@ -33,14 +33,14 @@
- +
- +
{{ type_fields(config) }} diff --git a/tests/core/test_monitors_page.py b/tests/core/test_monitors_page.py index 87995cc..52b8f0c 100644 --- a/tests/core/test_monitors_page.py +++ b/tests/core/test_monitors_page.py @@ -101,13 +101,18 @@ async def test_the_monitor_page_says_since_when_and_why(app): "monitors/detail.html", monitor=m, host=None, latest=down, failures=[down], state="down", since=datetime.now(timezone.utc) - timedelta(minutes=12), since_at_least=False, tls_days=None, uptime={"24h": 93.0, "7d": None, "30d": 99.5}, - heartbeat=[{"state": "up", "title": "a"}, {"state": "down", "title": "b"}]) + heartbeat=[{"state": "up", "title": "a"}, {"state": "down", "title": "b"}], + history=[{"at": 1, "ms": 20.0, "down": 0}, {"at": 2, "ms": 30.0, "down": 0}, + {"at": 3, "ms": None, "down": 3}], poll_interval=60) assert 'Monitors' in page and "shop <api>" in page - assert "since 12m ago" in page + assert ">for 12m<" in page # how long, not "… ago" + assert "Response time · last 24h" in page and '"down": 3' in page + assert "Expected status" in page and "1m Edit" in page and ">Pause" in page assert not hex_colours(page), hex_colours(page) @@ -119,6 +124,8 @@ async def test_adding_is_its_own_page_and_keeps_what_was_typed(app): values={"name": "db", "target": "10.0.0.9", "host_id": "", "check_interval_seconds": "30"}, error="port must be between 1 and 65535, got 0.") assert 'Monitors' in page and 'action="/monitors/add"' in page + for kind in ("Ping", "TCP", "HTTP", "DNS"): # each type says what it checks + assert f"{kind}" in page assert '' in page assert 'value="db"' in page and 'value="10.0.0.9"' in page and 'value="30"' in page diff --git a/tests/integration/test_monitor_history.py b/tests/integration/test_monitor_history.py new file mode 100644 index 0000000..3ad1d5f --- /dev/null +++ b/tests/integration/test_monitor_history.py @@ -0,0 +1,46 @@ +"""Integration: a monitor's response-time history averages its checks per +bucket and counts the failed ones (#5214). Two checks share a bucket and must +merge (lesson #5262). Requires STEWARD_DATABASE_URL.""" +from __future__ import annotations + +import asyncio +import os +import uuid +from datetime import datetime, timedelta, timezone + +import pytest + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +def test_response_history_averages_a_bucket_and_counts_its_failures(app): + from steward.models.monitors import Monitor, MonitorResult + from steward.monitors.service import response_history + + now = datetime.now(timezone.utc) + width = 24 * 3600 // 96 # 15 minutes + start = datetime.fromtimestamp((int(now.timestamp()) // width - 2) * width, timezone.utc) + mid = str(uuid.uuid4()) + + def r(secs, up, ms): + return MonitorResult(monitor_id=mid, checked_at=start + timedelta(seconds=secs), is_up=up, response_ms=ms) + + async def go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add(Monitor(id=mid, name=f"m-{mid[:8]}", type="tcp", target="10.9.9.9")) + async with s.begin(): + s.add_all([r(10, True, 10.0), r(70, True, 30.0), r(130, False, None)]) + async with s.begin(): + return await response_history(s, mid, now) + + got = asyncio.run(go()) + assert got == [{"at": int(start.timestamp() * 1000), "ms": 20.0, "down": 1}]