fix(ansible): overlapping runs: one per host, one per schedule, one action run per alert rule (#5158)

- One run per host: start_run claims every host it touches (addresses read
  from the inventory Steward wrote; a source's own inventory file is claimed
  as that file) before it takes a run slot. The claim is all-or-nothing. A run
  waiting on a host shows queued and names the run it waits for, honours a
  cancel, and fails after HOST_WAIT_SECONDS (1 h) with which host and run.
- A due schedule whose previous run is still queued or running skips the
  firing: last_run_at moves on, last_run_id still names the run going, and
  last_error says "Skipped: the previous run was still going". The Schedules
  page shows that beside the run (muted), and a launch error in red.
- An alert rule that fires while the action run from an earlier firing is
  still going starts no new run; the event says so and names the run. An
  in-process set closes the gap between the check and the run being linked.
- docs/core/ansible.md: queued/cancelled statuses and the overlap rules.

Tests: tests/core/test_ansible_overlap.py (claim keys, waiting, all-or-nothing,
cancel, deadline) and tests/integration/test_ansible_overlap.py (two real
runs on one host take turns; schedule skip then fire; alert action skip then
start).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/alerts.py

@@ -497,8 +497,47 @@ async def _run_ansible_action(
     inventory must be ones the source lists (ansible/guards.py). The firing
     event links to the run, or says why the action did not start.
     """
-    from steward.ansible.runner import trigger_run
+    from steward.ansible.runner import IN_FLIGHT, trigger_run
+    from steward.models.ansible import AnsibleRun
 
+    if rule.id in _actions_starting:
+        await _note_event(app, event_id, "Ansible action skipped: an earlier "
+                          "firing's action was still starting")
+        return
+    async with app.db_sessionmaker() as db:
+        busy = (await db.execute(
+            select(AnsibleRun.id)
+            .join(AlertEvent, AlertEvent.ansible_run_id == AnsibleRun.id)
+            .where(AlertEvent.rule_id == rule.id, AnsibleRun.status.in_(IN_FLIGHT))
+            .limit(1))).scalar_one_or_none()
+    if busy:
+        # One action run at a time per rule (#5158): a re-firing while the
+        # last one is still going would run the same fix over it.
+        await _note_event(app, event_id, f"Ansible action skipped: run {busy} "
+                          "from an earlier firing was still going")
+        return
+    _actions_starting.add(rule.id)
+    try:
+        await _start_ansible_action(app, rule, value, event_id, trigger_run)
+    finally:
+        _actions_starting.discard(rule.id)
+
+
+# Rules whose action is between the in-flight check and its run being linked
+# to the event, so two firings that close together can't both start one.
+_actions_starting: set[str] = set()
+
+
+async def _note_event(app: "Quart", event_id: str, why: str) -> None:
+    async with app.db_sessionmaker() as db:
+        async with db.begin():
+            ev = (await db.execute(
+                select(AlertEvent).where(AlertEvent.id == event_id))).scalar_one_or_none()
+            if ev is not None:
+                ev.note = f"{ev.note}; {why}" if ev.note else why
+
+
+async def _start_ansible_action(app, rule, value, event_id, trigger_run) -> None:
     action = rule.ansible_action or {}
     playbook = action.get("playbook") or ""
     inventory = action.get("inventory") or ""
@@ -529,17 +568,15 @@ async def _run_ansible_action(
     if err:
         logger.error("Ansible action for rule %r not started: %s", rule.name, err)
 
+    if run is None:
+        await _note_event(app, event_id, f"Ansible action not started: {err}")
+        return
     async with app.db_sessionmaker() as db:
         async with db.begin():
             ev = (await db.execute(
                 select(AlertEvent).where(AlertEvent.id == event_id))).scalar_one_or_none()
-            if ev is None:
-                return
-            if run is not None:
+            if ev is not None:
                 ev.ansible_run_id = run.id
-            else:
-                why = f"Ansible action not started: {err}"
-                ev.note = f"{ev.note}; {why}" if ev.note else why
 
 
 async def check_silence(app: "Quart") -> None: