fix(monitors): run each monitor on its own interval, from its scheduled start (#5151)

The dispatcher ticked once per MONITORS_POLL_INTERVAL and stamped
last_checked_at when the whole batch finished. With a 60 s tick and a 60 s
monitor the next tick saw 59.x s elapsed and skipped, so the monitor ran
every 120 s; a 30 s monitor could never run faster than the tick.

- The dispatcher ticks every 5 s (DISPATCH_TICK_SECONDS) and reads the
  monitors and intervals fresh each tick, so a changed interval applies at
  the next one.
- Due is measured from the scheduled start, stamped before the check runs,
  with half a tick of slack so tick jitter never costs a whole tick. A
  long-overdue monitor runs once.
- Each check runs in its own task and records its own result, so a slow
  probe holds up nobody; a monitor still being checked is not started
  again, and it is freed in a finally.
- A check past CHECK_DEADLINE_SECONDS (120) is recorded down with "Check did
  not finish within 120 s" rather than holding its monitor forever.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/app.py

@@ -367,7 +367,7 @@ def create_app(
 def _register_core_tasks(app: Quart) -> None:
     from .core.scheduler import ScheduledTask
 
-    poll_interval = app.config.get("MONITORS_POLL_INTERVAL", 60)
+    from .monitors.scheduler import DISPATCH_TICK_SECONDS as MONITOR_DISPATCH_TICK
     cleanup_interval = 3600  # hourly
 
     async def run_monitor_checks():
@@ -418,10 +418,12 @@ def _register_core_tasks(app: Quart) -> None:
             interval_seconds=60,
             run_on_startup=False,
         ),
+        # Ticks often and only starts checks: each monitor decides whether it
+        # is due from its own interval (steward/monitors/scheduler.py).
         ScheduledTask(
             name="monitor_check",
             coro_factory=run_monitor_checks,
-            interval_seconds=poll_interval,
+            interval_seconds=MONITOR_DISPATCH_TICK,
             run_on_startup=True,
         ),
         ScheduledTask(

steward/monitors/scheduler.py

@@ -1,52 +1,92 @@
-"""Single scheduled task that runs every due Monitor of any type."""
+"""Dispatch every due Monitor of any type, each check in its own task.
+
+The dispatcher ticks every DISPATCH_TICK_SECONDS. Each tick it reads the
+enabled monitors and their intervals fresh (a changed interval applies at the
+next tick), stamps the due ones with the time they were due to start, and
+starts each check on its own. A slow check (a 10 s timeout) holds up nobody
+else, and a monitor still being checked is not started twice.
+"""
 from __future__ import annotations
 import asyncio
 import logging
 from datetime import datetime, timezone
 
-from sqlalchemy import select
+from sqlalchemy import select, update
 
 from steward.core.alerts import MetricBatch
 from steward.models.monitors import Monitor, MonitorResult
-from .runner import run_monitor
+from .runner import _BLANK, run_monitor
 
 logger = logging.getLogger(__name__)
 
+# How often the dispatcher looks for due monitors. A monitor runs within one
+# tick of its interval, so this is the cadence's worst-case lateness.
+DISPATCH_TICK_SECONDS = 5
+# A check past this is recorded down rather than left holding its monitor:
+# far beyond any probe's own timeout, so it only ever catches a hang.
+CHECK_DEADLINE_SECONDS = 120
+
+
+def is_due(monitor, now: datetime, global_interval: int) -> bool:
+    """Whether a monitor's interval has elapsed since its last scheduled start.
+
+    Half a tick of slack, so a check stamped at t isn't missed at t+interval
+    by a few milliseconds of tick jitter and pushed back a whole tick."""
+    if monitor.last_checked_at is None:
+        return True
+    interval = monitor.check_interval_seconds or global_interval
+    elapsed = (now - monitor.last_checked_at).total_seconds()
+    return elapsed >= interval - DISPATCH_TICK_SECONDS / 2
+
+
+def _state(app) -> tuple[set, set]:
+    """(monitor ids being checked, their tasks) for this app."""
+    if not hasattr(app, "_monitor_checks"):
+        app._monitor_checks = (set(), set())
+    return app._monitor_checks
+
 
 async def run_due_monitors(app) -> None:
-    """Check every enabled Monitor whose interval has elapsed, persist results.
+    """Start a check for every enabled Monitor that is due and not already
+    being checked. Returns once they are started, not finished.
 
     Per-monitor `check_interval_seconds` overrides the global
-    MONITORS_POLL_INTERVAL (0 = use global), mirroring how the scheduler fires
-    on the global cadence but each monitor decides if it is actually due.
-    """
+    MONITORS_POLL_INTERVAL (0 = use global)."""
     global_interval = int(app.config.get("MONITORS_POLL_INTERVAL", 60))
     now = datetime.now(timezone.utc)
+    in_flight, tasks = _state(app)
 
     async with app.db_sessionmaker() as session:
         monitors = list((await session.execute(
             select(Monitor).where(Monitor.enabled.is_(True))
         )).scalars())
+        due = [m for m in monitors if m.id not in in_flight and is_due(m, now, global_interval)]
+        if not due:
+            return
+        # Stamp the scheduled start before checking: the next run is due one
+        # interval from now, however long this check takes.
+        await session.execute(
+            update(Monitor).where(Monitor.id.in_([m.id for m in due])).values(last_checked_at=now))
+        await session.commit()
 
-    due: list[Monitor] = []
-    for m in monitors:
-        interval = m.check_interval_seconds or global_interval
-        if m.last_checked_at is None or (now - m.last_checked_at).total_seconds() >= interval:
-            due.append(m)
-    if not due:
-        return
+    for m in due:
+        in_flight.add(m.id)
+        task = asyncio.create_task(_check_and_record(app, m))
+        tasks.add(task)
+        task.add_done_callback(tasks.discard)
 
-    pairs = await asyncio.gather(
-        *[run_monitor(m) for m in due], return_exceptions=True
-    )
-    check_time = datetime.now(timezone.utc)
 
-    async with app.db_sessionmaker() as session:
-        async with session.begin():
-            for monitor, res in zip(due, pairs):
-                if isinstance(res, Exception):
-                    logger.error("Monitor %r check errored: %s", monitor.name, res)
-                    continue
+async def _check_and_record(app, monitor: Monitor) -> None:
+    """Run one monitor's check and store its result and metrics."""
+    in_flight, _ = _state(app)
+    try:
+        try:
+            res = await asyncio.wait_for(run_monitor(monitor), CHECK_DEADLINE_SECONDS)
+        except asyncio.TimeoutError:
+            res = dict(_BLANK, error_msg=f"Check did not finish within {CHECK_DEADLINE_SECONDS} s")
+        check_time = datetime.now(timezone.utc)
+        async with app.db_sessionmaker() as session:
+            async with session.begin():
                 session.add(MonitorResult(
                     monitor_id=monitor.id,
                     checked_at=check_time,
@@ -58,10 +98,6 @@ async def run_due_monitors(app) -> None:
                     tls_expires_at=res["tls_expires_at"],
                     error_msg=res["error_msg"],
                 ))
-                db_monitor = await session.get(Monitor, monitor.id)
-                if db_monitor:
-                    db_monitor.last_checked_at = check_time
-
                 # Alert pipeline — keyed by monitor type (source) + name.
                 batch = MetricBatch()
                 batch.add(
@@ -76,3 +112,7 @@ async def run_due_monitors(app) -> None:
                         value=res["response_ms"],
                     )
                 await batch.record(session)
+    except Exception:
+        logger.exception("Monitor %r check errored", monitor.name)
+    finally:
+        in_flight.discard(monitor.id)