Compare commits
4
Commits
480b465b8e
...
401bfb5251
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
401bfb5251 | ||
|
|
5a3f8083f9 | ||
|
|
8edc0c30a3 | ||
|
|
ca12652277 |
@@ -3,7 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
@@ -26,6 +26,21 @@ def is_due(schedule: AnsibleSchedule, now: datetime) -> bool:
|
||||
return (now - schedule.last_run_at).total_seconds() >= schedule.interval_seconds
|
||||
|
||||
|
||||
def due_slot(schedule: AnsibleSchedule, now: datetime) -> datetime:
|
||||
"""The due time a firing at `now` belongs to, kept as last_run_at.
|
||||
|
||||
The latest last_run_at + k * interval not after now, so the next one is
|
||||
due a whole interval after this slot rather than after whenever the ~60 s
|
||||
loop happened to notice it. Counting from `now` added each firing's
|
||||
lateness to the next, and a daily schedule crept later every day (#5432).
|
||||
Missed slots (Steward was down) are skipped, not run in a burst.
|
||||
"""
|
||||
if schedule.last_run_at is None or not schedule.interval_seconds:
|
||||
return now
|
||||
whole = (now - schedule.last_run_at).total_seconds() // schedule.interval_seconds
|
||||
return schedule.last_run_at + timedelta(seconds=whole * schedule.interval_seconds)
|
||||
|
||||
|
||||
async def run_due_schedules(app) -> int:
|
||||
"""Trigger every due schedule once. Returns the number launched. A due
|
||||
schedule whose previous run hasn't finished skips this firing."""
|
||||
@@ -39,6 +54,7 @@ async def run_due_schedules(app) -> int:
|
||||
for s in schedules:
|
||||
if not is_due(s, now):
|
||||
continue
|
||||
slot = due_slot(s, now)
|
||||
async with app.db_sessionmaker() as db:
|
||||
busy = await run_in_flight(db, s.last_run_id)
|
||||
if busy:
|
||||
@@ -49,7 +65,7 @@ async def run_due_schedules(app) -> int:
|
||||
async with db.begin():
|
||||
fresh = await db.get(AnsibleSchedule, s.id)
|
||||
if fresh:
|
||||
fresh.last_run_at = now
|
||||
fresh.last_run_at = slot
|
||||
fresh.last_error = "Skipped: the previous run was still going"
|
||||
logger.info("Schedule %s (%s) skipped: run %s still going",
|
||||
s.id, s.name, s.last_run_id)
|
||||
@@ -75,7 +91,7 @@ async def run_due_schedules(app) -> int:
|
||||
async with db.begin():
|
||||
fresh = await db.get(AnsibleSchedule, s.id)
|
||||
if fresh:
|
||||
fresh.last_run_at = now
|
||||
fresh.last_run_at = slot
|
||||
fresh.last_run_id = run.id if run else None
|
||||
fresh.last_error = err
|
||||
if run:
|
||||
|
||||
@@ -42,6 +42,16 @@ class _DueTracker:
|
||||
ready.append(task)
|
||||
return ready
|
||||
|
||||
def begin(self, tasks: list[ScheduledTask], now: float) -> None:
|
||||
"""Start the clock: a task that doesn't run on startup is first due
|
||||
one interval from now. Without this a task that had never run counted
|
||||
from 0 on the loop's clock (host uptime), so every one of them fired
|
||||
on the first tick after a deploy, the hourly cleanup with its Docker
|
||||
log prune among them, holding up agent ingest for minutes (#5431)."""
|
||||
for task in tasks:
|
||||
if not task.run_on_startup:
|
||||
self.last_run.setdefault(task.name, now)
|
||||
|
||||
def mark_started(self, task: ScheduledTask, now: float) -> None:
|
||||
self.in_flight.add(task.name)
|
||||
self.last_run[task.name] = now
|
||||
@@ -58,10 +68,12 @@ async def start_scheduler(tasks: list[ScheduledTask]) -> None:
|
||||
tracker.mark_started(task, now)
|
||||
asyncio.create_task(_run_task(task, tracker))
|
||||
|
||||
started = asyncio.get_event_loop().time()
|
||||
tracker.begin(tasks, started)
|
||||
for task in tasks:
|
||||
if task.run_on_startup:
|
||||
logger.info(f"Startup task: {task.name}")
|
||||
_spawn(task, asyncio.get_event_loop().time())
|
||||
_spawn(task, started)
|
||||
|
||||
while True:
|
||||
now = asyncio.get_event_loop().time()
|
||||
|
||||
@@ -6,6 +6,7 @@ import json
|
||||
from sqlalchemy import select
|
||||
|
||||
from steward.core import renames
|
||||
from steward.mcp._context import mcp_app
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor
|
||||
from steward.models.users import UserRole
|
||||
@@ -27,6 +28,15 @@ def _state(m: Monitor, latest) -> str:
|
||||
return "up" if latest.is_up else "down"
|
||||
|
||||
|
||||
def _interval(m: Monitor) -> dict:
|
||||
"""How often a monitor runs: its own interval, or the global one it
|
||||
follows when it has none (#5433), and which."""
|
||||
if m.check_interval_seconds:
|
||||
return {"check_interval_s": m.check_interval_seconds, "interval_from": "monitor"}
|
||||
return {"check_interval_s": int(mcp_app().config.get("MONITORS_POLL_INTERVAL", 60)),
|
||||
"interval_from": "global"}
|
||||
|
||||
|
||||
def _tls_days(latest) -> float | None:
|
||||
if latest is None or latest.tls_expires_at is None:
|
||||
return None
|
||||
@@ -112,8 +122,10 @@ async def get_monitor(monitor: str, failures: int = 10) -> dict:
|
||||
newest first, each with its error message.
|
||||
|
||||
`heartbeat` is its last 30 checks oldest-first as a string, "+" up and "-"
|
||||
down, to show a flapping check at a glance. Header values in an HTTP
|
||||
monitor's configuration are redacted.
|
||||
down, to show a flapping check at a glance. `check_interval_s` is how
|
||||
often it runs; `interval_from` says whether that is its own setting
|
||||
("monitor") or the global monitors interval it follows ("global").
|
||||
Header values in an HTTP monitor's configuration are redacted.
|
||||
"""
|
||||
failures = clamp_limit(failures, 10, 50)
|
||||
async with db_session() as db:
|
||||
@@ -123,7 +135,7 @@ async def get_monitor(monitor: str, failures: int = 10) -> dict:
|
||||
failed = await monitor_svc.recent_failures(db, m.id, failures)
|
||||
return {
|
||||
**_row(m, st, host_names),
|
||||
"check_interval_s": m.check_interval_seconds or None,
|
||||
**_interval(m),
|
||||
"config": _config(m),
|
||||
"heartbeat": "".join("+" if r.is_up else "-" for r in st["recent"]),
|
||||
"recent_failures": [
|
||||
@@ -156,7 +168,7 @@ def _saved(m: Monitor, host_names: dict[str, str]) -> dict:
|
||||
"target": redact_url(m.target),
|
||||
"host_name": host_names.get(m.host_id) if m.host_id else None,
|
||||
"enabled": m.enabled,
|
||||
"check_interval_s": m.check_interval_seconds or None,
|
||||
**_interval(m),
|
||||
"config": _config(m),
|
||||
"next": f'get_monitor(monitor="{m.id}")',
|
||||
}
|
||||
|
||||
+59
-13
@@ -1,17 +1,38 @@
|
||||
"""ICMP / TCP reachability checks — pure probes returning a result dict."""
|
||||
"""ICMP / TCP reachability checks — pure probes returning a result dict.
|
||||
|
||||
A failed check says why in `error_msg` (#5430): a down monitor that can't
|
||||
say whether nothing answered or something refused leaves the operator to
|
||||
guess.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import socket
|
||||
import time
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TCP_TIMEOUT = 5.0
|
||||
DEFAULT_TCP_PORT = 80
|
||||
PING_WAIT_S = 2
|
||||
PING_TIMEOUT = 5.0
|
||||
|
||||
|
||||
def tcp_failure(exc: BaseException, address: str, port: int) -> str:
|
||||
"""Why a TCP connect failed, in a phrase."""
|
||||
if isinstance(exc, (asyncio.TimeoutError, TimeoutError)):
|
||||
return f"No answer on port {port} within {TCP_TIMEOUT:g} s"
|
||||
if isinstance(exc, ConnectionRefusedError):
|
||||
return f"Connection refused on port {port}"
|
||||
if isinstance(exc, socket.gaierror):
|
||||
return f"Can't resolve {address}: {exc.strerror or exc}"
|
||||
if isinstance(exc, OSError):
|
||||
return f"Can't connect to port {port}: {exc.strerror or exc}"
|
||||
return str(exc) or type(exc).__name__
|
||||
|
||||
|
||||
async def tcp_check(address: str, port: int) -> dict:
|
||||
"""TCP connect probe. Returns {is_up, response_ms}."""
|
||||
"""TCP connect probe. Returns {is_up, response_ms, error_msg}."""
|
||||
start = time.monotonic()
|
||||
try:
|
||||
reader, writer = await asyncio.wait_for(
|
||||
@@ -20,28 +41,53 @@ async def tcp_check(address: str, port: int) -> dict:
|
||||
)
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
return {"is_up": True, "response_ms": round((time.monotonic() - start) * 1000, 2)}
|
||||
except Exception:
|
||||
return {"is_up": False, "response_ms": None}
|
||||
return {"is_up": True, "response_ms": round((time.monotonic() - start) * 1000, 2),
|
||||
"error_msg": None}
|
||||
except Exception as exc:
|
||||
return {"is_up": False, "response_ms": None, "error_msg": tcp_failure(exc, address, port)}
|
||||
|
||||
|
||||
def ping_failure(returncode: int, stderr: str) -> str:
|
||||
"""Why `ping -c 1` failed: exit 1 is no reply; anything else is ping's
|
||||
own complaint (an unknown name, no route), first line of it."""
|
||||
if returncode == 1:
|
||||
return f"No reply within {PING_WAIT_S} s"
|
||||
said = stderr.strip().splitlines()[0] if stderr.strip() else ""
|
||||
return said or f"ping failed (exit {returncode})"
|
||||
|
||||
|
||||
async def icmp_check(address: str) -> dict:
|
||||
"""ICMP echo via the system ping binary (setuid; no raw-socket privilege).
|
||||
|
||||
Falls back to a TCP connect on the default port if the ping binary is
|
||||
unavailable or errors, so a host with ICMP filtered still gets a signal.
|
||||
unavailable or errors, so a host with ICMP filtered still gets a signal;
|
||||
a failure then says the TCP check stood in.
|
||||
"""
|
||||
start = time.monotonic()
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"ping", "-c", "1", "-W", "2", address,
|
||||
"ping", "-c", "1", "-W", str(PING_WAIT_S), address,
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
await asyncio.wait_for(proc.wait(), timeout=5.0)
|
||||
if proc.returncode == 0:
|
||||
return {"is_up": True, "response_ms": round((time.monotonic() - start) * 1000, 2)}
|
||||
return {"is_up": False, "response_ms": None}
|
||||
except Exception as exc:
|
||||
logger.warning("ICMP ping failed for %s: %s, falling back to TCP", address, exc)
|
||||
return await tcp_check(address, DEFAULT_TCP_PORT)
|
||||
result = await tcp_check(address, DEFAULT_TCP_PORT)
|
||||
if not result["is_up"]:
|
||||
result["error_msg"] = f"ping unavailable, TCP check instead: {result['error_msg']}"
|
||||
return result
|
||||
try:
|
||||
_, stderr = await asyncio.wait_for(proc.communicate(), timeout=PING_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
try:
|
||||
await asyncio.wait_for(proc.wait(), timeout=PING_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("ping for %s did not exit after kill", address)
|
||||
return {"is_up": False, "response_ms": None,
|
||||
"error_msg": f"No reply within {PING_TIMEOUT:g} s"}
|
||||
if proc.returncode == 0:
|
||||
return {"is_up": True, "response_ms": round((time.monotonic() - start) * 1000, 2),
|
||||
"error_msg": None}
|
||||
return {"is_up": False, "response_ms": None,
|
||||
"error_msg": ping_failure(proc.returncode, (stderr or b"").decode(errors="replace"))}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""A failed ping or TCP check says why (#5430): no reply, refused, no answer
|
||||
in time, a name that doesn't resolve, or ping's own complaint; an ICMP check
|
||||
that fell back to TCP says so."""
|
||||
import asyncio
|
||||
import socket
|
||||
|
||||
import pytest
|
||||
|
||||
from steward.monitors import ping
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exc,want", [
|
||||
(asyncio.TimeoutError(), "No answer on port 22 within 5 s"),
|
||||
(ConnectionRefusedError(111, "Connection refused"), "Connection refused on port 22"),
|
||||
(socket.gaierror(-2, "Name or service not known"), "Can't resolve nas.lan: Name or service not known"),
|
||||
(OSError(113, "No route to host"), "Can't connect to port 22: No route to host"),
|
||||
(RuntimeError("odd"), "odd"),
|
||||
])
|
||||
def test_a_tcp_failure_says_why(monkeypatch, exc, want):
|
||||
async def boom(*a, **kw):
|
||||
raise exc
|
||||
monkeypatch.setattr(ping.asyncio, "open_connection", boom)
|
||||
got = asyncio.run(ping.tcp_check("nas.lan", 22))
|
||||
assert got == {"is_up": False, "response_ms": None, "error_msg": want}
|
||||
|
||||
|
||||
class _Proc:
|
||||
def __init__(self, returncode, stderr=b""):
|
||||
self.returncode, self._stderr = returncode, stderr
|
||||
|
||||
async def communicate(self):
|
||||
return b"", self._stderr
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code,stderr,want", [
|
||||
(1, b"", "No reply within 2 s"),
|
||||
(2, b"ping: nas.lan: Name or service not known\n", "ping: nas.lan: Name or service not known"),
|
||||
(2, b"", "ping failed (exit 2)"),
|
||||
])
|
||||
def test_a_ping_failure_says_why(monkeypatch, code, stderr, want):
|
||||
async def spawn(*a, **kw):
|
||||
return _Proc(code, stderr)
|
||||
monkeypatch.setattr(ping.asyncio, "create_subprocess_exec", spawn)
|
||||
got = asyncio.run(ping.icmp_check("nas.lan"))
|
||||
assert got == {"is_up": False, "response_ms": None, "error_msg": want}
|
||||
|
||||
|
||||
def test_a_reply_is_up_with_no_error(monkeypatch):
|
||||
async def spawn(*a, **kw):
|
||||
return _Proc(0)
|
||||
monkeypatch.setattr(ping.asyncio, "create_subprocess_exec", spawn)
|
||||
got = asyncio.run(ping.icmp_check("nas.lan"))
|
||||
assert got["is_up"] is True and got["error_msg"] is None
|
||||
|
||||
|
||||
def test_without_ping_the_tcp_stand_in_says_so(monkeypatch):
|
||||
async def no_ping(*a, **kw):
|
||||
raise FileNotFoundError("ping")
|
||||
|
||||
async def refused(*a, **kw):
|
||||
raise ConnectionRefusedError(111, "Connection refused")
|
||||
monkeypatch.setattr(ping.asyncio, "create_subprocess_exec", no_ping)
|
||||
monkeypatch.setattr(ping.asyncio, "open_connection", refused)
|
||||
got = asyncio.run(ping.icmp_check("nas.lan"))
|
||||
assert got["error_msg"] == "ping unavailable, TCP check instead: Connection refused on port 80"
|
||||
@@ -56,3 +56,20 @@ def test_independent_tasks_do_not_block_each_other():
|
||||
tr = _DueTracker()
|
||||
tr.mark_started(a, now=0) # a hangs
|
||||
assert tr.due([a, b], now=10) == [b] # b still fires
|
||||
|
||||
|
||||
def test_a_task_not_run_on_startup_waits_one_interval_after_boot():
|
||||
"""#5431: the loop's clock is host uptime, far past every interval. A task
|
||||
marked not to run on startup used to count from 0 and fire on the first
|
||||
tick anyway; it now waits a whole interval from when the loop began."""
|
||||
boot = 1_000_000.0
|
||||
hourly = _task("cleanup", 3600)
|
||||
eager = ScheduledTask(name="poll", coro_factory=lambda: None, interval_seconds=60,
|
||||
run_on_startup=True)
|
||||
tr = _DueTracker()
|
||||
tr.begin([hourly, eager], now=boot)
|
||||
assert tr.due([hourly], now=boot + 1) == []
|
||||
assert tr.due([hourly], now=boot + 3599) == []
|
||||
assert tr.due([hourly], now=boot + 3600) == [hourly]
|
||||
assert "poll" not in tr.last_run # started by start_scheduler itself
|
||||
|
||||
|
||||
@@ -76,6 +76,9 @@ def test_monitor_rule_ack_and_maintenance_writes(app):
|
||||
assert made["target"] == "https://nas.lan" and made["host_name"] == host_name
|
||||
assert made["config"]["headers"] == {"Authorization": "<redacted>"}
|
||||
assert made["config"]["verify_ssl"] is True
|
||||
# No interval of its own: it says which it follows, and how often (#5433).
|
||||
assert (made["check_interval_s"], made["interval_from"]) == (
|
||||
int(app.config.get("MONITORS_POLL_INTERVAL", 60)), "global")
|
||||
|
||||
# ── rules ──────────────────────────────────────────────────────────
|
||||
opts = await a.get_alert_rule_options("http")
|
||||
@@ -100,7 +103,9 @@ def test_monitor_rule_ack_and_maintenance_writes(app):
|
||||
assert renamed["carried"]["from"] == mon_name and renamed["carried"]["rules"] == 1
|
||||
async with app.db_sessionmaker() as s:
|
||||
assert (await s.get(AlertRule, rule["rule_id"])).resource_name == f"{mon_name}-2"
|
||||
assert "carried" not in await m.update_monitor(f"{mon_name}-2", check_interval_s=30)
|
||||
own = await m.update_monitor(f"{mon_name}-2", check_interval_s=30)
|
||||
assert "carried" not in own
|
||||
assert (own["check_interval_s"], own["interval_from"]) == (30, "monitor")
|
||||
# Another http monitor may not take the same name: they would share one history.
|
||||
with pytest.raises(ValueError, match="already named"):
|
||||
await m.create_monitor(f"{mon_name}-2", "http", "other.lan")
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
|
||||
from steward.ansible.scheduler import is_due
|
||||
from steward.ansible.scheduler import due_slot, is_due
|
||||
|
||||
NOW = datetime(2026, 6, 16, 12, 0, 0, tzinfo=timezone.utc)
|
||||
|
||||
@@ -26,3 +26,27 @@ def test_due_when_interval_elapsed():
|
||||
|
||||
def test_not_due_before_interval():
|
||||
assert is_due(_sched(last=NOW - timedelta(seconds=1800), interval=3600), NOW) is False
|
||||
|
||||
|
||||
def test_a_daily_schedule_holds_its_time_of_day():
|
||||
"""#5432: noticed 40 s late every day, a daily run still belongs to its
|
||||
05:44:45 slot, so the lateness never adds up."""
|
||||
day = 86400
|
||||
last = datetime(2026, 10, 2, 5, 44, 45, tzinfo=timezone.utc)
|
||||
for _ in range(6):
|
||||
noticed = last + timedelta(seconds=day + 40)
|
||||
assert is_due(_sched(last=last, interval=day), noticed)
|
||||
last = due_slot(_sched(last=last, interval=day), noticed)
|
||||
assert last == datetime(2026, 10, 8, 5, 44, 45, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def test_missed_slots_are_skipped_not_run_in_a_burst():
|
||||
last = NOW - timedelta(hours=5, minutes=10)
|
||||
slot = due_slot(_sched(last=last, interval=3600), NOW)
|
||||
assert slot == NOW - timedelta(minutes=10)
|
||||
assert not is_due(_sched(last=slot, interval=3600), NOW)
|
||||
|
||||
|
||||
def test_a_first_run_is_its_own_slot():
|
||||
assert due_slot(_sched(last=None), NOW) == NOW
|
||||
|
||||
|
||||
Reference in New Issue
Block a user