diff --git a/docs/core/ansible.md b/docs/core/ansible.md
index f1a4fd8..bdde28e 100644
--- a/docs/core/ansible.md
+++ b/docs/core/ansible.md
@@ -101,12 +101,20 @@ If a client connects after the run has already completed, the endpoint immediate
| Status | Description |
|---|---|
+| `queued` | Waiting for a host another run holds, or for a run slot |
| `running` | Execution in progress |
| `success` | Playbook exited 0 |
| `failed` | Playbook exited non-zero |
| `interrupted` | App restarted while run was in progress |
+| `cancelled` | Stopped by an operator, or by Steward shutting down mid-run |
-On startup, the app marks any runs still in `running` state as `interrupted` (see `_mark_interrupted_runs()` in `app.py`).
+On startup, the app marks any runs still `running` or `queued` as `interrupted` (see `_mark_interrupted_runs()` in `app.py`).
+
+### Overlapping runs
+
+- **One run per host.** Before it takes a run slot, a run claims every host it touches. Hosts come from the inventory Steward writes, keyed by address. A run on a source's own inventory file claims that file, because Steward doesn't read its hosts. While another run holds one of its hosts, the run shows `queued` and says which run it waits for. After an hour of waiting it fails and says so.
+- **One run at a time per schedule.** A schedule that falls due while its previous run is still `queued` or `running` skips that firing. The Schedules page shows "Skipped: the previous run was still going" beside that run. The next firing comes an interval later.
+- **One action run at a time per alert rule.** When a rule fires again while the action run from an earlier firing is still going, no new run starts. The firing's event says it was skipped and names the run.
Output stored in the DB is capped at 1 MB. If truncated, `[output truncated]` is appended to the DB column, but the live SSE stream continues unaffected.
diff --git a/steward/ansible/executor.py b/steward/ansible/executor.py
index 2786b0c..3c6415e 100644
--- a/steward/ansible/executor.py
+++ b/steward/ansible/executor.py
@@ -38,6 +38,111 @@ def artifact_path(run_id: str) -> str:
return os.path.join(ARTIFACT_DIR, f"{run_id}.log")
+# ── One run per host (#5158) ──────────────────────────────────────────────────
+# Two runs against the same host would race each other's changes, so a run
+# claims every host it touches before it takes a run slot, and waits (queued)
+# while another run holds one. The claim is all-or-nothing, so two runs over
+# overlapping hosts can't each hold half and wait on the other.
+
+HOST_WAIT_SECONDS = 3600.0 # a run waiting longer than this for a host fails
+
+
+class HostBusy(Exception):
+ """A run gave up waiting for a host another run held."""
+
+
+def run_hosts(inventory_content: str | None, inventory_path: str | None) -> set[str]:
+ """What a run touches, as claim keys: the address of each host in the YAML
+ inventory Steward wrote for it, or, for a source's own inventory file
+ (whose hosts Steward doesn't read), that file."""
+ if not inventory_content:
+ return {f"inventory:{inventory_path}"} if inventory_path else set()
+ import yaml
+ try:
+ data = yaml.safe_load(inventory_content)
+ except yaml.YAMLError:
+ return set() # ansible-playbook will refuse it and say why
+ # A host listed again under a group carries no vars there, so its address
+ # is whichever listing gives one.
+ addresses: dict[str, str | None] = {}
+
+ def walk(group) -> None:
+ if not isinstance(group, dict):
+ return
+ members = group.get("hosts")
+ if isinstance(members, dict):
+ for name, hostvars in members.items():
+ addr = hostvars.get("ansible_host") if isinstance(hostvars, dict) else None
+ addresses[str(name)] = addr or addresses.get(str(name))
+ children = group.get("children")
+ if isinstance(children, dict):
+ for child in children.values():
+ walk(child)
+
+ if isinstance(data, dict):
+ for group in data.values():
+ walk(group)
+ return {str(addr or name).strip().lower() for name, addr in addresses.items()}
+
+
+def _host_gate(app: "Quart") -> dict:
+ """The claims and the wake-up event, bound to the running loop like the
+ run-slot semaphore (one loop in prod, one per test)."""
+ loop = asyncio.get_running_loop()
+ gate = getattr(app, "_ansible_host_gate", None)
+ if gate is None or gate["loop"] is not loop:
+ gate = {"loop": loop, "held": {}, "changed": asyncio.Event()}
+ app._ansible_host_gate = gate
+ return gate
+
+
+async def _claim_hosts(app: "Quart", run_id: str, hosts: set[str]) -> bool:
+ """Hold every host in hosts for run_id, waiting while another run holds
+ any of them. Returns whether it had to wait. Raises _Aborted when the run
+ is cancelled while waiting, HostBusy after HOST_WAIT_SECONDS."""
+ from steward.models.ansible import AnsibleRunStatus
+
+ gate = _host_gate(app)
+ held: dict[str, str] = gate["held"]
+ give_up = time.monotonic() + HOST_WAIT_SECONDS
+ waited = False
+ while True:
+ busy = sorted({held[h] for h in hosts if h in held and held[h] != run_id})
+ if not busy:
+ for h in hosts:
+ held[h] = run_id
+ return waited
+ if run_id in _cancelled:
+ raise _Aborted
+ remaining = give_up - time.monotonic()
+ if remaining <= 0:
+ names = ", ".join(sorted(h for h in hosts if h in held))
+ raise HostBusy(f"gave up after {HOST_WAIT_SECONDS:.0f} s waiting for "
+ f"{names}, held by run {', '.join(busy)}")
+ if not waited:
+ waited = True
+ await _set_status(app, run_id, AnsibleRunStatus.queued)
+ _broadcast(run_id, f"[queued — waiting for run {', '.join(busy)} "
+ "to finish on the same host]")
+ changed = gate["changed"]
+ try:
+ # Woken by a release; the short timeout also notices a cancel.
+ await asyncio.wait_for(changed.wait(), timeout=min(remaining, 2.0))
+ except asyncio.TimeoutError:
+ pass
+
+
+def _release_hosts(app: "Quart", run_id: str, hosts: set[str]) -> None:
+ gate = getattr(app, "_ansible_host_gate", None)
+ if gate is None:
+ return
+ for h in hosts:
+ if gate["held"].get(h) == run_id:
+ del gate["held"][h]
+ changed, gate["changed"] = gate["changed"], asyncio.Event()
+ changed.set()
+
+
@dataclass
class _Done:
status: str
@@ -392,8 +497,9 @@ async def start_run(
) -> None:
"""Execute ansible-playbook as a subprocess and update the DB run row.
- Respects a global concurrency cap (app._ansible_semaphore): if no slot is
- free the run shows as 'queued' until one frees. Supports cancellation, a
+ Waits (as 'queued') while another run holds one of its hosts (#5158), then
+ for a slot under the global concurrency cap (app._ansible_semaphore).
+ Supports cancellation, a
persistent full-log artifact, and a parsed per-host result summary.
connection is an optional per-run SSH override (user/password) for
@@ -442,17 +548,23 @@ async def start_run(
sem = asyncio.Semaphore(max(1, cap))
app._ansible_semaphore = sem
app._ansible_sem_loop = loop
- queued = sem.locked()
- if queued:
- await _set_status(app, run_id, AnsibleRunStatus.queued)
- _broadcast(run_id, "[queued — waiting for an Ansible run slot]")
- await sem.acquire()
+ hosts = run_hosts(inventory_content, inventory_path)
final_status = "failed"
logf = None
proc = None
interrupted = False
+ slot_held = False
try:
+ # The hosts first, so a run waiting on a host doesn't sit on a run slot.
+ queued = await _claim_hosts(app, run_id, hosts)
+ if sem.locked():
+ if not queued:
+ await _set_status(app, run_id, AnsibleRunStatus.queued)
+ queued = True
+ _broadcast(run_id, "[queued — waiting for an Ansible run slot]")
+ await sem.acquire()
+ slot_held = True
if run_id in _cancelled: # cancelled while queued — abort before launching
raise _Aborted
if queued:
@@ -592,7 +704,9 @@ async def start_run(
logf.close()
_run_procs.pop(run_id, None)
_cancelled.discard(run_id)
- sem.release()
+ if slot_held:
+ sem.release()
+ _release_hosts(app, run_id, hosts)
results = {"hosts": parse_recap(_run_lines.get(run_id, [])), "failures": failures}
try:
diff --git a/steward/ansible/runner.py b/steward/ansible/runner.py
index 0d1eacc..8f7ca6a 100644
--- a/steward/ansible/runner.py
+++ b/steward/ansible/runner.py
@@ -16,6 +16,19 @@ from steward.ansible.inventory_gen import (
)
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
+# A run in one of these hasn't finished: a schedule or an alert action whose
+# previous run is still in one skips its next firing (#5158).
+IN_FLIGHT = (AnsibleRunStatus.queued, AnsibleRunStatus.running)
+
+
+async def run_in_flight(db, run_id: str | None) -> bool:
+ """Whether run_id names a run that hasn't finished. Runs left running by a
+ restart are marked interrupted at startup, so this can't stick."""
+ if not run_id:
+ return False
+ run = await db.get(AnsibleRun, run_id)
+ return run is not None and run.status in IN_FLIGHT
+
async def trigger_run(
app,
diff --git a/steward/ansible/scheduler.py b/steward/ansible/scheduler.py
index f4efaff..17a67c1 100644
--- a/steward/ansible/scheduler.py
+++ b/steward/ansible/scheduler.py
@@ -7,7 +7,7 @@ from datetime import datetime, timezone
from sqlalchemy import select
-from steward.ansible.runner import trigger_run
+from steward.ansible.runner import run_in_flight, trigger_run
from steward.models.ansible_schedule import AnsibleSchedule
logger = logging.getLogger(__name__)
@@ -27,7 +27,8 @@ def is_due(schedule: AnsibleSchedule, now: datetime) -> bool:
async def run_due_schedules(app) -> int:
- """Trigger every due schedule once. Returns the number launched."""
+ """Trigger every due schedule once. Returns the number launched. A due
+ schedule whose previous run hasn't finished skips this firing."""
now = datetime.now(timezone.utc)
async with app.db_sessionmaker() as db:
schedules = (await db.execute(
@@ -38,6 +39,21 @@ async def run_due_schedules(app) -> int:
for s in schedules:
if not is_due(s, now):
continue
+ async with app.db_sessionmaker() as db:
+ busy = await run_in_flight(db, s.last_run_id)
+ if busy:
+ # One run at a time per schedule (#5158). This firing is skipped,
+ # not deferred, so the cadence holds; last_run_id still names the
+ # run that's going.
+ async with app.db_sessionmaker() as db:
+ async with db.begin():
+ fresh = await db.get(AnsibleSchedule, s.id)
+ if fresh:
+ fresh.last_run_at = now
+ fresh.last_error = "Skipped: the previous run was still going"
+ logger.info("Schedule %s (%s) skipped: run %s still going",
+ s.id, s.name, s.last_run_id)
+ continue
run = None
err: str | None = None
try:
diff --git a/steward/core/alerts.py b/steward/core/alerts.py
index 68d422b..c40f753 100644
--- a/steward/core/alerts.py
+++ b/steward/core/alerts.py
@@ -497,8 +497,47 @@ async def _run_ansible_action(
inventory must be ones the source lists (ansible/guards.py). The firing
event links to the run, or says why the action did not start.
"""
- from steward.ansible.runner import trigger_run
+ from steward.ansible.runner import IN_FLIGHT, trigger_run
+ from steward.models.ansible import AnsibleRun
+ if rule.id in _actions_starting:
+ await _note_event(app, event_id, "Ansible action skipped: an earlier "
+ "firing's action was still starting")
+ return
+ async with app.db_sessionmaker() as db:
+ busy = (await db.execute(
+ select(AnsibleRun.id)
+ .join(AlertEvent, AlertEvent.ansible_run_id == AnsibleRun.id)
+ .where(AlertEvent.rule_id == rule.id, AnsibleRun.status.in_(IN_FLIGHT))
+ .limit(1))).scalar_one_or_none()
+ if busy:
+ # One action run at a time per rule (#5158): a re-firing while the
+ # last one is still going would run the same fix over it.
+ await _note_event(app, event_id, f"Ansible action skipped: run {busy} "
+ "from an earlier firing was still going")
+ return
+ _actions_starting.add(rule.id)
+ try:
+ await _start_ansible_action(app, rule, value, event_id, trigger_run)
+ finally:
+ _actions_starting.discard(rule.id)
+
+
+# Rules whose action is between the in-flight check and its run being linked
+# to the event, so two firings that close together can't both start one.
+_actions_starting: set[str] = set()
+
+
+async def _note_event(app: "Quart", event_id: str, why: str) -> None:
+ async with app.db_sessionmaker() as db:
+ async with db.begin():
+ ev = (await db.execute(
+ select(AlertEvent).where(AlertEvent.id == event_id))).scalar_one_or_none()
+ if ev is not None:
+ ev.note = f"{ev.note}; {why}" if ev.note else why
+
+
+async def _start_ansible_action(app, rule, value, event_id, trigger_run) -> None:
action = rule.ansible_action or {}
playbook = action.get("playbook") or ""
inventory = action.get("inventory") or ""
@@ -529,17 +568,15 @@ async def _run_ansible_action(
if err:
logger.error("Ansible action for rule %r not started: %s", rule.name, err)
+ if run is None:
+ await _note_event(app, event_id, f"Ansible action not started: {err}")
+ return
async with app.db_sessionmaker() as db:
async with db.begin():
ev = (await db.execute(
select(AlertEvent).where(AlertEvent.id == event_id))).scalar_one_or_none()
- if ev is None:
- return
- if run is not None:
+ if ev is not None:
ev.ansible_run_id = run.id
- else:
- why = f"Ansible action not started: {err}"
- ev.note = f"{ev.note}; {why}" if ev.note else why
async def check_silence(app: "Quart") -> None:
diff --git a/steward/mcp/tools/automation.py b/steward/mcp/tools/automation.py
index 1e84b94..6883af2 100644
--- a/steward/mcp/tools/automation.py
+++ b/steward/mcp/tools/automation.py
@@ -527,8 +527,10 @@ async def list_schedules() -> dict:
`check: true` schedules are dry runs that never change anything.
`last_error` is set when a firing could not start at all (a source or
- playbook that went away); a run that started and failed shows in
- `last_run_status`, and get_ansible_run(last_run_id) has its log.
+ playbook that went away), or was skipped because the schedule's previous
+ run was still going (then `last_run_id` still names that run). A run that
+ started and failed shows in `last_run_status`, and
+ get_ansible_run(last_run_id) has its log.
"""
async with db_session() as db:
scheds = await _schedules(db)
diff --git a/steward/templates/ansible/schedules.html b/steward/templates/ansible/schedules.html
index e4c8ea8..32c756a 100644
--- a/steward/templates/ansible/schedules.html
+++ b/steward/templates/ansible/schedules.html
@@ -38,13 +38,18 @@
| {{ row.scope_label }} |
{{ fmt_interval(s.interval_seconds) }} |
- {% if s.last_error %}
- launch error
- {% elif row.last_run %}
+ {% if row.last_run %}
{% set sc = {"queued":"var(--text-dim)","running":"var(--yellow)","success":"var(--green)","failed":"var(--red)","interrupted":"var(--orange)","cancelled":"var(--orange)"}.get(row.last_run.status.value, "var(--text-muted)") %}
{{ row.last_run.status.value }}
- · {{ s.last_run_at.strftime("%m-%d %H:%M") }}
- {% else %}
+ · {{ row.last_run.started_at.strftime("%m-%d %H:%M") }}
+ {% endif %}
+ {% if s.last_error %}
+ {# With a run beside it, the last firing was skipped because that run was
+ still going (#5158); without one, the firing couldn't start a run. #}
+
+ {{ s.last_error }} · {{ s.last_run_at.strftime("%m-%d %H:%M") }}
+
+ {% elif not row.last_run %}
never
{% endif %}
|
diff --git a/tests/core/test_ansible_overlap.py b/tests/core/test_ansible_overlap.py
new file mode 100644
index 0000000..555726d
--- /dev/null
+++ b/tests/core/test_ansible_overlap.py
@@ -0,0 +1,93 @@
+"""One run per host (#5158): what a run claims, and how a second run waits."""
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+
+from steward.ansible import executor
+from steward.ansible.inventory_gen import generate_inventory, inventory_to_yaml
+
+
+def _target(name, address, groups=()):
+ return SimpleNamespace(name=name, address=address, ansible_vars={},
+ groups=[SimpleNamespace(name=g, ansible_vars={}) for g in groups])
+
+
+def test_a_run_claims_the_addresses_in_the_inventory_steward_wrote():
+ inv = inventory_to_yaml(generate_inventory(
+ [_target("web", "10.0.0.5", ["linux"]), _target("db", "DB.lan", ["linux"])]))
+ # A host listed again under its group (no vars there) is still its address.
+ assert executor.run_hosts(inv, None) == {"10.0.0.5", "db.lan"}
+
+
+def test_a_host_without_an_address_is_claimed_by_name():
+ assert executor.run_hosts("all:\n hosts:\n box1:\n token: x\n", None) == {"box1"}
+
+
+def test_a_run_on_a_source_inventory_claims_the_file():
+ assert executor.run_hosts(None, "inventories/prod.ini") == {"inventory:inventories/prod.ini"}
+ assert executor.run_hosts(None, "") == set()
+
+
+@pytest.fixture
+def gate(monkeypatch):
+ statuses: list[tuple[str, str]] = []
+
+ async def fake_set_status(_app, run_id, status):
+ statuses.append((run_id, status.value))
+
+ monkeypatch.setattr(executor, "_set_status", fake_set_status)
+ yield SimpleNamespace(app=SimpleNamespace(), statuses=statuses)
+ for rid in ("a", "b", "c"):
+ executor._run_lines.pop(rid, None)
+ executor._cancelled.discard(rid)
+
+
+def test_a_second_run_on_the_same_host_waits_for_the_first(gate):
+ app = gate.app
+
+ async def go():
+ assert await executor._claim_hosts(app, "a", {"web"}) is False
+ b = asyncio.create_task(executor._claim_hosts(app, "b", {"web", "db"}))
+ await asyncio.sleep(0.05)
+ assert not b.done()
+ held = app._ansible_host_gate["held"]
+ assert "db" not in held # all or nothing: b holds no half
+ assert await executor._claim_hosts(app, "c", {"db"}) is False
+ executor._release_hosts(app, "c", {"db"})
+ await asyncio.sleep(0.05)
+ assert not b.done() # web is still a's
+ executor._release_hosts(app, "a", {"web"})
+ assert await asyncio.wait_for(b, 5) is True
+ assert held == {"web": "b", "db": "b"}
+
+ asyncio.run(go())
+ assert ("b", "queued") in gate.statuses
+ assert any("waiting for run a" in line for line in executor._run_lines["b"])
+
+
+def test_a_run_cancelled_while_waiting_for_a_host_never_claims_it(gate):
+ app = gate.app
+
+ async def go():
+ await executor._claim_hosts(app, "a", {"web"})
+ b = asyncio.create_task(executor._claim_hosts(app, "b", {"web"}))
+ await asyncio.sleep(0.05)
+ executor._cancelled.add("b")
+ with pytest.raises(executor._Aborted):
+ await asyncio.wait_for(b, 5)
+ assert app._ansible_host_gate["held"] == {"web": "a"}
+
+ asyncio.run(go())
+
+
+def test_a_run_gives_up_waiting_for_a_host_and_says_which(gate, monkeypatch):
+ monkeypatch.setattr(executor, "HOST_WAIT_SECONDS", 0.2)
+ app = gate.app
+
+ async def go():
+ await executor._claim_hosts(app, "a", {"web"})
+ with pytest.raises(executor.HostBusy, match="waiting for web, held by run a"):
+ await asyncio.wait_for(executor._claim_hosts(app, "b", {"web"}), 5)
+
+ asyncio.run(go())
diff --git a/tests/integration/test_ansible_overlap.py b/tests/integration/test_ansible_overlap.py
new file mode 100644
index 0000000..d8c2f1b
--- /dev/null
+++ b/tests/integration/test_ansible_overlap.py
@@ -0,0 +1,201 @@
+"""Integration: overlapping Ansible runs (#5158).
+
+Two runs on one host take turns, a schedule whose run is still going skips its
+firing, and an alert rule whose action run is still going doesn't start
+another. Needs Postgres for the run, schedule and alert rows.
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+import uuid
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+RECAP = ("PLAY RECAP ****\\n"
+ "localhost : ok=1 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0")
+
+# Notes when it starts and ends in a shared file, so the order shows overlap.
+TURN = (
+ "import sys, time\n"
+ "open(sys.argv[1], 'a').write('start\\n')\n"
+ "time.sleep(1)\n"
+ "open(sys.argv[1], 'a').write('end\\n')\n"
+ f"print('{RECAP}')\n"
+)
+
+INVENTORY = "all:\n hosts:\n box:\n ansible_host: 10.99.0.7\n"
+
+
+@pytest.fixture
+def app():
+ if not os.environ.get("STEWARD_DATABASE_URL"):
+ pytest.skip("needs Postgres")
+ from steward.app import create_app
+ return create_app(testing=False)
+
+
+async def _new_run(app, source: str, status=None) -> str:
+ from steward.models.ansible import AnsibleRun, AnsibleRunStatus
+ run_id = str(uuid.uuid4())
+ async with app.db_sessionmaker() as s:
+ async with s.begin():
+ s.add(AnsibleRun(id=run_id, playbook_path="t.yml", inventory_path=None,
+ source_name=source, triggered_by=None,
+ status=status or AnsibleRunStatus.running))
+ return run_id
+
+
+async def _set_run(app, run_id, status) -> None:
+ from sqlalchemy import update
+ from steward.models.ansible import AnsibleRun
+ async with app.db_sessionmaker() as s:
+ async with s.begin():
+ await s.execute(update(AnsibleRun).where(AnsibleRun.id == run_id)
+ .values(status=status))
+
+
+def test_two_runs_on_one_host_take_turns(app, monkeypatch, tmp_path):
+ from sqlalchemy import select
+ from steward.ansible import executor
+ from steward.models.ansible import AnsibleRun, AnsibleRunStatus
+ turns = tmp_path / "turns"
+ monkeypatch.setattr(executor, "build_ansible_command",
+ lambda *_a, **_k: [sys.executable, "-c", TURN, str(turns)])
+ source = f"overlap-{uuid.uuid4().hex[:8]}"
+
+ async def _go():
+ first, second = await _new_run(app, source), await _new_run(app, source)
+ await asyncio.wait_for(asyncio.gather(
+ executor.start_run(app, first, "t.yml", "", str(tmp_path),
+ inventory_content=INVENTORY),
+ executor.start_run(app, second, "t.yml", "", str(tmp_path),
+ inventory_content=INVENTORY),
+ ), 60)
+ async with app.db_sessionmaker() as s:
+ return (await s.execute(select(AnsibleRun).where(
+ AnsibleRun.source_name == source))).scalars().all()
+
+ runs = asyncio.run(_go())
+ assert turns.read_text().split() == ["start", "end", "start", "end"]
+ assert {r.status for r in runs} == {AnsibleRunStatus.success}
+ assert any("waiting for run" in (r.output or "") for r in runs)
+
+
+def test_a_schedule_skips_while_its_last_run_is_going(app, monkeypatch):
+ from steward.ansible import scheduler
+ from steward.models.ansible import AnsibleRunStatus
+ from steward.models.ansible_schedule import AnsibleSchedule
+ source = f"sched-{uuid.uuid4().hex[:8]}"
+ fired: list[str] = []
+
+ async def fake_trigger(_app, **kw):
+ fired.append(kw["playbook_path"])
+ return SimpleNamespace(id=str(uuid.uuid4())), None, None
+
+ monkeypatch.setattr(scheduler, "trigger_run", fake_trigger)
+ is_due = scheduler.is_due # leave other tests' schedules alone
+ monkeypatch.setattr(scheduler, "is_due",
+ lambda s, now: s.source_name == source and is_due(s, now))
+ long_ago = datetime.now(timezone.utc) - timedelta(hours=2)
+
+ async def _go():
+ running = await _new_run(app, source)
+ sid = str(uuid.uuid4())
+ async with app.db_sessionmaker() as s:
+ async with s.begin():
+ s.add(AnsibleSchedule(id=sid, name=source, source_name=source,
+ playbook_path="t.yml", interval_seconds=60,
+ enabled=True, last_run_at=long_ago,
+ last_run_id=running))
+ try:
+ await scheduler.run_due_schedules(app)
+ async with app.db_sessionmaker() as s:
+ skipped = await s.get(AnsibleSchedule, sid)
+ assert fired == []
+ assert skipped.last_error.startswith("Skipped")
+ assert skipped.last_run_id == running # still names the run going
+ assert skipped.last_run_at > long_ago # the cadence holds
+
+ await _set_run(app, running, AnsibleRunStatus.success)
+ async with app.db_sessionmaker() as s:
+ async with s.begin():
+ (await s.get(AnsibleSchedule, sid)).last_run_at = long_ago
+ await scheduler.run_due_schedules(app)
+ async with app.db_sessionmaker() as s:
+ fresh = await s.get(AnsibleSchedule, sid)
+ assert fired == ["t.yml"]
+ assert fresh.last_error is None and fresh.last_run_id != running
+ finally:
+ async with app.db_sessionmaker() as s:
+ async with s.begin():
+ await s.delete(await s.get(AnsibleSchedule, sid))
+
+ asyncio.run(_go())
+
+
+def test_an_alert_action_skips_while_its_last_run_is_going(app, monkeypatch):
+ from steward.ansible import runner
+ from steward.core import alerts
+ from steward.models.alerts import AlertEvent, AlertOperator, AlertRule
+ from steward.models.ansible import AnsibleRunStatus
+ from steward.models.users import User, UserRole
+ tag = uuid.uuid4().hex[:8]
+ uid, rule_id = str(uuid.uuid4()), str(uuid.uuid4())
+ started: list[str] = []
+ action = {"source": "t", "playbook": "fix.yml", "inventory": "steward:all"}
+
+ def rule():
+ return AlertRule(id=rule_id, name=f"hot {tag}", source_module="host_agent",
+ resource_name=f"box-{tag}", metric_name="cpu_pct",
+ operator=AlertOperator.gt, threshold=90.0, enabled=True,
+ ansible_action=action, created_by=uid)
+
+ async def _event(run_id=None) -> str:
+ ev_id = str(uuid.uuid4())
+ async with app.db_sessionmaker() as s:
+ async with s.begin():
+ s.add(AlertEvent(id=ev_id, rule_id=rule_id, from_state="inactive",
+ to_state="firing", metric_value=95.0,
+ ansible_run_id=run_id))
+ return ev_id
+
+ async def _go():
+ async with app.db_sessionmaker() as s:
+ async with s.begin():
+ s.add(User(id=uid, username=f"u{tag}", email=f"{tag}@e.test",
+ password_hash="x", role=UserRole.admin, is_active=True))
+ async with s.begin():
+ s.add(rule())
+ going = await _new_run(app, f"alert-{tag}")
+ await _event(going) # the earlier firing's run
+ nxt = await _new_run(app, f"alert-{tag}", AnsibleRunStatus.success)
+
+ async def fake_trigger(_app, **kw):
+ started.append(kw["playbook_path"])
+ return SimpleNamespace(id=nxt), None, None
+
+ monkeypatch.setattr(runner, "trigger_run", fake_trigger)
+
+ refire = await _event()
+ await alerts._run_ansible_action(app, rule(), 95.0, refire)
+ async with app.db_sessionmaker() as s:
+ skipped = await s.get(AlertEvent, refire)
+ assert started == []
+ assert f"skipped: run {going}" in skipped.note
+ assert skipped.ansible_run_id is None
+
+ await _set_run(app, going, AnsibleRunStatus.success)
+ later = await _event()
+ await alerts._run_ansible_action(app, rule(), 95.0, later)
+ async with app.db_sessionmaker() as s:
+ linked = await s.get(AlertEvent, later)
+ assert started == ["fix.yml"]
+ assert linked.ansible_run_id == nxt
+
+ asyncio.run(_go())