+ {%- if not seen %}No readings yet: the first arrive within a minute of Steward starting. + {%- elif problems %}{% set said = problems|join("; ") %}{{ said[:1]|upper }}{{ said[1:] }}. + {%- else %}Steward is running normally.{% endif -%} +
+ +| Measure | Now | Last 24 h |
|---|---|---|
| {{ metric_label(r.metric) }}
+ {{ metric_about(r.metric) }} |
+ {{ metric_value(r.metric, r.now) }}
+ {%- if r.at %} {{ r.at|ago }} {% endif %} |
+ + {%- if r.low is not none %}{{ metric_value(r.metric, r.low) }} – {{ metric_value(r.metric, r.high) }} + + {%- else %}—{% endif -%} + | +
+ Recorded each minute (the cleanup each hour). Any of these can be watched by an + alert rule: pick the source "Steward itself".
+{% endblock %} diff --git a/tests/core/test_self_monitor.py b/tests/core/test_self_monitor.py index cff4316..32e1ee5 100644 --- a/tests/core/test_self_monitor.py +++ b/tests/core/test_self_monitor.py @@ -1,4 +1,4 @@ -"""Unit tests for the self-fd watchdog (no DB, no network). +"""Unit tests for the self-monitor: open files and Steward's own health (no DB, no network). Exercises the fd accounting, the percentage/warning floor, and that the recorder degrades to a no-op where fd accounting isn't available. The DB session @@ -36,9 +36,15 @@ class _Ctx: class _FakeSession(_Ctx): + last = None # the latest steward sample the unwatched gap is counted from + def begin(self): return _Ctx() + async def execute(self, stmt): + last = self.last + return type("R", (), {"scalar": staticmethod(lambda: last)})() + class _FakeApp: def db_sessionmaker(self): @@ -104,12 +110,37 @@ def test_no_warning_below_floor(monkeypatch, caplog): assert not any("soft limit" in r.message for r in caplog.records) -def test_noop_when_fd_count_unavailable(monkeypatch): - # /proc absent → must return before touching the DB. +def test_without_fd_accounting_the_rest_is_still_recorded(monkeypatch): + # /proc absent: no open-file pair, but Steward's other health still lands. + recorded = _patch_recorder(monkeypatch) monkeypatch.setattr(sm, "count_open_fds", lambda: None) - class _Boom: - def db_sessionmaker(self): - raise AssertionError("must not open a session when fds are unknown") + asyncio.run(sm.record_self_metrics(_FakeApp())) - asyncio.run(sm.record_self_metrics(_Boom())) # no raise + names = {m for (_s, _r, m, _v) in recorded} + assert "db_roundtrip_ms" in names and not names & {"open_fds", "open_fds_pct"} + + +def test_the_minute_s_ingests_are_recorded_and_the_tally_starts_again(monkeypatch): + """#5467: posts, failures and the slowest post since the last tick.""" + from steward.core import health + recorded = _patch_recorder(monkeypatch) + monkeypatch.setattr(sm, "count_open_fds", lambda: 50) + monkeypatch.setattr(health, "ingest", health.IngestTally()) + for ms, ok in ((300.0, True), (9800.0, False), (450.0, True)): + health.ingest.done(ms, ok) + + asyncio.run(sm.record_self_metrics(_FakeApp())) + + got = {m: v for (s, r, m, v) in recorded if (s, r) == ("steward", "process")} + assert (got["ingest_posts"], got["ingest_failed"], got["ingest_slowest_ms"]) == (3.0, 1.0, 9800.0) + assert got["unwatched_secs"] == 0.0 and got["db_roundtrip_ms"] >= 0 + assert health.ingest.take() == health.IngestTally() # taken: the next minute starts at zero + + +def test_unwatched_counts_only_a_real_gap(): + from datetime import datetime, timedelta, timezone + now = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc) + assert sm.unwatched_secs(None, now) == 0.0 # first ever tick + assert sm.unwatched_secs(now - timedelta(seconds=61), now) == 0.0 # an ordinary minute + assert sm.unwatched_secs(now - timedelta(minutes=7), now) == 420.0 # a deploy or outage diff --git a/tests/core/test_steward_health.py b/tests/core/test_steward_health.py new file mode 100644 index 0000000..c93d10b --- /dev/null +++ b/tests/core/test_steward_health.py @@ -0,0 +1,60 @@ +"""Settings → System (#5467): Steward's own health, a verdict then each measure +now and over the day. The verdict names what is wrong, and only that.""" +from datetime import datetime, timezone + +import pytest +from quart import render_template, session + +from steward.core import health + +AT = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc) + + +def _previews(**over): + base = {m: {"now": 0.0, "at": AT, "series": [0.0, 0.0], "low": 0.0, "high": 0.0} + for m in health.PROCESS_METRICS} + base["db_roundtrip_ms"] = {"now": 4.0, "at": AT, "series": [3.0, 5.0], "low": 2.0, "high": 9.0} + for m, v in over.items(): + base[m] = {**base[m], **v} + return base + + +@pytest.fixture +def previews(monkeypatch): + def use(**over): + got = _previews(**over) + + async def fake(db, source, resource, metric, now=None): + assert (source, resource) == ("steward", "process") + return got[metric] + monkeypatch.setattr("steward.alerts.service.metric_preview", fake) + return use + + +async def test_a_healthy_steward_has_no_problems(previews): + previews() + got = await health.process_health(None) + assert got["seen"] and got["problems"] == [] + assert [r["metric"] for r in got["rows"]] == health.PROCESS_METRICS + + +async def test_failures_and_gaps_are_named(previews): + previews(ingest_failed={"now": 2.0, "high": 2.0}, unwatched_secs={"high": 420.0}, + open_fds_pct={"now": 85.0}) + problems = (await health.process_health(None))["problems"] + assert problems[0] == "2 agent posts failed in the last minute" + assert "wasn't watching for up to 7m" in problems[1] + assert problems[2] == "open files at 85% of the limit" + + +async def test_the_page_reads_as_a_verdict_then_each_measure(app, previews): + previews(unwatched_secs={"high": 420.0}) + got = await health.process_health(None) + for r in got["rows"]: + r["spark"] = "" + async with app.test_request_context("/settings/system/"): + session["user_role"] = "admin" + page = " ".join((await render_template("settings/system.html", **got)).split()) + assert 'Steward wasn\'t watching for up to 7m' in page + assert "Database round trip" in page and "4 ms" in page and "2 ms – 9 ms" in page + assert page.index("page-verdict") < page.index("Agent posts per minute")