4 Commits
Author SHA1 Message Date
bvandeusenandClaude Opus 5.5 401bfb5251 fix(scheduler): a task marked not to run on startup waits an interval after boot (#5431)
CI / lint (push) Successful in 3s
CI / integration (push) Successful in 1m57s
CI / unit (push) Successful in 2m8s
CI / publish (push) Successful in 1m33s
_DueTracker counted a task that had never run from 0 on the loop clock,
which is host uptime, so every run_on_startup=False task (hourly data
cleanup with the Docker log prune, the weekly report check, alert
reconcile, agent age, Ansible schedules) fired on the first tick after
each deploy, while agents were flushing their backlog. Agent ingest was
slow or dropped for a few minutes after every deploy.

start_scheduler now calls tracker.begin(): such a task is first due one
interval after the loop starts.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 21:52:08 -04:00
bvandeusenandClaude Opus 5.5 5a3f8083f9 fix(ansible): a daily schedule holds its time of day instead of creeping later (#5432)
Each firing stored last_run_at = the moment the ~60 s loop noticed it,
and the next was due a whole interval after that, so every firing's
lateness carried into the next: Nightly Docker Prune moved from 05:44:45
to 05:47:05 over six days.

A firing now stores its due slot, the latest last_run_at + k * interval
not after now (due_slot), for both a launch and a skipped firing. Missed
slots after an outage are skipped, not run in a burst.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 21:51:09 -04:00
bvandeusenandClaude Opus 5.5 8edc0c30a3 fix(mcp): get_monitor says how often a monitor on the global interval runs (#5433)
check_interval_s was `m.check_interval_seconds or None`, so a monitor
following monitors.poll_interval_seconds read as null and a reader
couldn't tell how often it ran. Monitor results now carry the effective
interval and `interval_from: "monitor" | "global"`.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 21:50:20 -04:00
bvandeusenandClaude Opus 5.5 ca12652277 fix(monitors): a failed ping or TCP check says why (#5430)
tcp_check swallowed every exception and icmp_check returned on a
non-zero exit without a reason, so every down ICMP/TCP monitor stored
error_msg NULL and the Monitors page and MCP showed it down with no why.

Each failure now names its cause: "No reply within 2 s", "Connection
refused on port N", "No answer on port N within 5 s", "Can't resolve
<name>: ...", the OS error, or ping's own first stderr line. An ICMP
check that fell back to TCP says so, and a ping that outlives its
timeout is killed rather than left running.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 21:49:42 -04:00
8 changed files with 220 additions and 23 deletions
+19 -3
View File
@@ -3,7 +3,7 @@
from __future__ import annotations
import logging
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
@@ -26,6 +26,21 @@ def is_due(schedule: AnsibleSchedule, now: datetime) -> bool:
return (now - schedule.last_run_at).total_seconds() >= schedule.interval_seconds
def due_slot(schedule: AnsibleSchedule, now: datetime) -> datetime:
"""The due time a firing at `now` belongs to, kept as last_run_at.
The latest last_run_at + k * interval not after now, so the next one is
due a whole interval after this slot rather than after whenever the ~60 s
loop happened to notice it. Counting from `now` added each firing's
lateness to the next, and a daily schedule crept later every day (#5432).
Missed slots (Steward was down) are skipped, not run in a burst.
"""
if schedule.last_run_at is None or not schedule.interval_seconds:
return now
whole = (now - schedule.last_run_at).total_seconds() // schedule.interval_seconds
return schedule.last_run_at + timedelta(seconds=whole * schedule.interval_seconds)
async def run_due_schedules(app) -> int:
"""Trigger every due schedule once. Returns the number launched. A due
schedule whose previous run hasn't finished skips this firing."""
@@ -39,6 +54,7 @@ async def run_due_schedules(app) -> int:
for s in schedules:
if not is_due(s, now):
continue
slot = due_slot(s, now)
async with app.db_sessionmaker() as db:
busy = await run_in_flight(db, s.last_run_id)
if busy:
@@ -49,7 +65,7 @@ async def run_due_schedules(app) -> int:
async with db.begin():
fresh = await db.get(AnsibleSchedule, s.id)
if fresh:
fresh.last_run_at = now
fresh.last_run_at = slot
fresh.last_error = "Skipped: the previous run was still going"
logger.info("Schedule %s (%s) skipped: run %s still going",
s.id, s.name, s.last_run_id)
@@ -75,7 +91,7 @@ async def run_due_schedules(app) -> int:
async with db.begin():
fresh = await db.get(AnsibleSchedule, s.id)
if fresh:
fresh.last_run_at = now
fresh.last_run_at = slot
fresh.last_run_id = run.id if run else None
fresh.last_error = err
if run:
+13 -1
View File
@@ -42,6 +42,16 @@ class _DueTracker:
ready.append(task)
return ready
def begin(self, tasks: list[ScheduledTask], now: float) -> None:
"""Start the clock: a task that doesn't run on startup is first due
one interval from now. Without this a task that had never run counted
from 0 on the loop's clock (host uptime), so every one of them fired
on the first tick after a deploy, the hourly cleanup with its Docker
log prune among them, holding up agent ingest for minutes (#5431)."""
for task in tasks:
if not task.run_on_startup:
self.last_run.setdefault(task.name, now)
def mark_started(self, task: ScheduledTask, now: float) -> None:
self.in_flight.add(task.name)
self.last_run[task.name] = now
@@ -58,10 +68,12 @@ async def start_scheduler(tasks: list[ScheduledTask]) -> None:
tracker.mark_started(task, now)
asyncio.create_task(_run_task(task, tracker))
started = asyncio.get_event_loop().time()
tracker.begin(tasks, started)
for task in tasks:
if task.run_on_startup:
logger.info(f"Startup task: {task.name}")
_spawn(task, asyncio.get_event_loop().time())
_spawn(task, started)
while True:
now = asyncio.get_event_loop().time()
+16 -4
View File
@@ -6,6 +6,7 @@ import json
from sqlalchemy import select
from steward.core import renames
from steward.mcp._context import mcp_app
from steward.models.hosts import Host
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
from steward.models.users import UserRole
@@ -27,6 +28,15 @@ def _state(m: Monitor, latest) -> str:
return "up" if latest.is_up else "down"
def _interval(m: Monitor) -> dict:
"""How often a monitor runs: its own interval, or the global one it
follows when it has none (#5433), and which."""
if m.check_interval_seconds:
return {"check_interval_s": m.check_interval_seconds, "interval_from": "monitor"}
return {"check_interval_s": int(mcp_app().config.get("MONITORS_POLL_INTERVAL", 60)),
"interval_from": "global"}
def _tls_days(latest) -> float | None:
if latest is None or latest.tls_expires_at is None:
return None
@@ -112,8 +122,10 @@ async def get_monitor(monitor: str, failures: int = 10) -> dict:
newest first, each with its error message.
`heartbeat` is its last 30 checks oldest-first as a string, "+" up and "-"
down, to show a flapping check at a glance. Header values in an HTTP
monitor's configuration are redacted.
down, to show a flapping check at a glance. `check_interval_s` is how
often it runs; `interval_from` says whether that is its own setting
("monitor") or the global monitors interval it follows ("global").
Header values in an HTTP monitor's configuration are redacted.
"""
failures = clamp_limit(failures, 10, 50)
async with db_session() as db:
@@ -123,7 +135,7 @@ async def get_monitor(monitor: str, failures: int = 10) -> dict:
failed = await monitor_svc.recent_failures(db, m.id, failures)
return {
**_row(m, st, host_names),
"check_interval_s": m.check_interval_seconds or None,
**_interval(m),
"config": _config(m),
"heartbeat": "".join("+" if r.is_up else "-" for r in st["recent"]),
"recent_failures": [
@@ -156,7 +168,7 @@ def _saved(m: Monitor, host_names: dict[str, str]) -> dict:
"target": redact_url(m.target),
"host_name": host_names.get(m.host_id) if m.host_id else None,
"enabled": m.enabled,
"check_interval_s": m.check_interval_seconds or None,
**_interval(m),
"config": _config(m),
"next": f'get_monitor(monitor="{m.id}")',
}
+59 -13
View File
@@ -1,17 +1,38 @@
"""ICMP / TCP reachability checks — pure probes returning a result dict."""
"""ICMP / TCP reachability checks — pure probes returning a result dict.
A failed check says why in `error_msg` (#5430): a down monitor that can't
say whether nothing answered or something refused leaves the operator to
guess.
"""
from __future__ import annotations
import asyncio
import logging
import socket
import time
logger = logging.getLogger(__name__)
TCP_TIMEOUT = 5.0
DEFAULT_TCP_PORT = 80
PING_WAIT_S = 2
PING_TIMEOUT = 5.0
def tcp_failure(exc: BaseException, address: str, port: int) -> str:
"""Why a TCP connect failed, in a phrase."""
if isinstance(exc, (asyncio.TimeoutError, TimeoutError)):
return f"No answer on port {port} within {TCP_TIMEOUT:g} s"
if isinstance(exc, ConnectionRefusedError):
return f"Connection refused on port {port}"
if isinstance(exc, socket.gaierror):
return f"Can't resolve {address}: {exc.strerror or exc}"
if isinstance(exc, OSError):
return f"Can't connect to port {port}: {exc.strerror or exc}"
return str(exc) or type(exc).__name__
async def tcp_check(address: str, port: int) -> dict:
"""TCP connect probe. Returns {is_up, response_ms}."""
"""TCP connect probe. Returns {is_up, response_ms, error_msg}."""
start = time.monotonic()
try:
reader, writer = await asyncio.wait_for(
@@ -20,28 +41,53 @@ async def tcp_check(address: str, port: int) -> dict:
)
writer.close()
await writer.wait_closed()
return {"is_up": True, "response_ms": round((time.monotonic() - start) * 1000, 2)}
except Exception:
return {"is_up": False, "response_ms": None}
return {"is_up": True, "response_ms": round((time.monotonic() - start) * 1000, 2),
"error_msg": None}
except Exception as exc:
return {"is_up": False, "response_ms": None, "error_msg": tcp_failure(exc, address, port)}
def ping_failure(returncode: int, stderr: str) -> str:
"""Why `ping -c 1` failed: exit 1 is no reply; anything else is ping's
own complaint (an unknown name, no route), first line of it."""
if returncode == 1:
return f"No reply within {PING_WAIT_S} s"
said = stderr.strip().splitlines()[0] if stderr.strip() else ""
return said or f"ping failed (exit {returncode})"
async def icmp_check(address: str) -> dict:
"""ICMP echo via the system ping binary (setuid; no raw-socket privilege).
Falls back to a TCP connect on the default port if the ping binary is
unavailable or errors, so a host with ICMP filtered still gets a signal.
unavailable or errors, so a host with ICMP filtered still gets a signal;
a failure then says the TCP check stood in.
"""
start = time.monotonic()
try:
proc = await asyncio.create_subprocess_exec(
"ping", "-c", "1", "-W", "2", address,
"ping", "-c", "1", "-W", str(PING_WAIT_S), address,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.PIPE,
)
await asyncio.wait_for(proc.wait(), timeout=5.0)
if proc.returncode == 0:
return {"is_up": True, "response_ms": round((time.monotonic() - start) * 1000, 2)}
return {"is_up": False, "response_ms": None}
except Exception as exc:
logger.warning("ICMP ping failed for %s: %s, falling back to TCP", address, exc)
return await tcp_check(address, DEFAULT_TCP_PORT)
result = await tcp_check(address, DEFAULT_TCP_PORT)
if not result["is_up"]:
result["error_msg"] = f"ping unavailable, TCP check instead: {result['error_msg']}"
return result
try:
_, stderr = await asyncio.wait_for(proc.communicate(), timeout=PING_TIMEOUT)
except asyncio.TimeoutError:
proc.kill()
try:
await asyncio.wait_for(proc.wait(), timeout=PING_TIMEOUT)
except asyncio.TimeoutError:
logger.warning("ping for %s did not exit after kill", address)
return {"is_up": False, "response_ms": None,
"error_msg": f"No reply within {PING_TIMEOUT:g} s"}
if proc.returncode == 0:
return {"is_up": True, "response_ms": round((time.monotonic() - start) * 1000, 2),
"error_msg": None}
return {"is_up": False, "response_ms": None,
"error_msg": ping_failure(proc.returncode, (stderr or b"").decode(errors="replace"))}
+65
View File
@@ -0,0 +1,65 @@
"""A failed ping or TCP check says why (#5430): no reply, refused, no answer
in time, a name that doesn't resolve, or ping's own complaint; an ICMP check
that fell back to TCP says so."""
import asyncio
import socket
import pytest
from steward.monitors import ping
@pytest.mark.parametrize("exc,want", [
(asyncio.TimeoutError(), "No answer on port 22 within 5 s"),
(ConnectionRefusedError(111, "Connection refused"), "Connection refused on port 22"),
(socket.gaierror(-2, "Name or service not known"), "Can't resolve nas.lan: Name or service not known"),
(OSError(113, "No route to host"), "Can't connect to port 22: No route to host"),
(RuntimeError("odd"), "odd"),
])
def test_a_tcp_failure_says_why(monkeypatch, exc, want):
async def boom(*a, **kw):
raise exc
monkeypatch.setattr(ping.asyncio, "open_connection", boom)
got = asyncio.run(ping.tcp_check("nas.lan", 22))
assert got == {"is_up": False, "response_ms": None, "error_msg": want}
class _Proc:
def __init__(self, returncode, stderr=b""):
self.returncode, self._stderr = returncode, stderr
async def communicate(self):
return b"", self._stderr
@pytest.mark.parametrize("code,stderr,want", [
(1, b"", "No reply within 2 s"),
(2, b"ping: nas.lan: Name or service not known\n", "ping: nas.lan: Name or service not known"),
(2, b"", "ping failed (exit 2)"),
])
def test_a_ping_failure_says_why(monkeypatch, code, stderr, want):
async def spawn(*a, **kw):
return _Proc(code, stderr)
monkeypatch.setattr(ping.asyncio, "create_subprocess_exec", spawn)
got = asyncio.run(ping.icmp_check("nas.lan"))
assert got == {"is_up": False, "response_ms": None, "error_msg": want}
def test_a_reply_is_up_with_no_error(monkeypatch):
async def spawn(*a, **kw):
return _Proc(0)
monkeypatch.setattr(ping.asyncio, "create_subprocess_exec", spawn)
got = asyncio.run(ping.icmp_check("nas.lan"))
assert got["is_up"] is True and got["error_msg"] is None
def test_without_ping_the_tcp_stand_in_says_so(monkeypatch):
async def no_ping(*a, **kw):
raise FileNotFoundError("ping")
async def refused(*a, **kw):
raise ConnectionRefusedError(111, "Connection refused")
monkeypatch.setattr(ping.asyncio, "create_subprocess_exec", no_ping)
monkeypatch.setattr(ping.asyncio, "open_connection", refused)
got = asyncio.run(ping.icmp_check("nas.lan"))
assert got["error_msg"] == "ping unavailable, TCP check instead: Connection refused on port 80"
+17
View File
@@ -56,3 +56,20 @@ def test_independent_tasks_do_not_block_each_other():
tr = _DueTracker()
tr.mark_started(a, now=0) # a hangs
assert tr.due([a, b], now=10) == [b] # b still fires
def test_a_task_not_run_on_startup_waits_one_interval_after_boot():
"""#5431: the loop's clock is host uptime, far past every interval. A task
marked not to run on startup used to count from 0 and fire on the first
tick anyway; it now waits a whole interval from when the loop began."""
boot = 1_000_000.0
hourly = _task("cleanup", 3600)
eager = ScheduledTask(name="poll", coro_factory=lambda: None, interval_seconds=60,
run_on_startup=True)
tr = _DueTracker()
tr.begin([hourly, eager], now=boot)
assert tr.due([hourly], now=boot + 1) == []
assert tr.due([hourly], now=boot + 3599) == []
assert tr.due([hourly], now=boot + 3600) == [hourly]
assert "poll" not in tr.last_run # started by start_scheduler itself
@@ -76,6 +76,9 @@ def test_monitor_rule_ack_and_maintenance_writes(app):
assert made["target"] == "https://nas.lan" and made["host_name"] == host_name
assert made["config"]["headers"] == {"Authorization": "<redacted>"}
assert made["config"]["verify_ssl"] is True
# No interval of its own: it says which it follows, and how often (#5433).
assert (made["check_interval_s"], made["interval_from"]) == (
int(app.config.get("MONITORS_POLL_INTERVAL", 60)), "global")
# ── rules ──────────────────────────────────────────────────────────
opts = await a.get_alert_rule_options("http")
@@ -100,7 +103,9 @@ def test_monitor_rule_ack_and_maintenance_writes(app):
assert renamed["carried"]["from"] == mon_name and renamed["carried"]["rules"] == 1
async with app.db_sessionmaker() as s:
assert (await s.get(AlertRule, rule["rule_id"])).resource_name == f"{mon_name}-2"
assert "carried" not in await m.update_monitor(f"{mon_name}-2", check_interval_s=30)
own = await m.update_monitor(f"{mon_name}-2", check_interval_s=30)
assert "carried" not in own
assert (own["check_interval_s"], own["interval_from"]) == (30, "monitor")
# Another http monitor may not take the same name: they would share one history.
with pytest.raises(ValueError, match="already named"):
await m.create_monitor(f"{mon_name}-2", "http", "other.lan")
+25 -1
View File
@@ -2,7 +2,7 @@
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from steward.ansible.scheduler import is_due
from steward.ansible.scheduler import due_slot, is_due
NOW = datetime(2026, 6, 16, 12, 0, 0, tzinfo=timezone.utc)
@@ -26,3 +26,27 @@ def test_due_when_interval_elapsed():
def test_not_due_before_interval():
assert is_due(_sched(last=NOW - timedelta(seconds=1800), interval=3600), NOW) is False
def test_a_daily_schedule_holds_its_time_of_day():
"""#5432: noticed 40 s late every day, a daily run still belongs to its
05:44:45 slot, so the lateness never adds up."""
day = 86400
last = datetime(2026, 10, 2, 5, 44, 45, tzinfo=timezone.utc)
for _ in range(6):
noticed = last + timedelta(seconds=day + 40)
assert is_due(_sched(last=last, interval=day), noticed)
last = due_slot(_sched(last=last, interval=day), noticed)
assert last == datetime(2026, 10, 8, 5, 44, 45, tzinfo=timezone.utc)
def test_missed_slots_are_skipped_not_run_in_a_burst():
last = NOW - timedelta(hours=5, minutes=10)
slot = due_slot(_sched(last=last, interval=3600), NOW)
assert slot == NOW - timedelta(minutes=10)
assert not is_due(_sched(last=slot, interval=3600), NOW)
def test_a_first_run_is_its_own_slot():
assert due_slot(_sched(last=None), NOW) == NOW