refactor(maintenance): DRY the recover_stalled head-run twins (#161)

recover_stalled_head_training_runs and recover_stalled_head_auto_apply_runs were near-exact copies (coalesce-flip + keep-last-N prune, differing only in model + two constants). Extract _recover_stalled_runs(model, stall_minutes, keep_runs, label); the two tasks become thin wrappers. The other two recover tasks are deliberately NOT folded in (library-audit has no prune tail; backup uses a single started_at cutoff). test_recover_stalled_head_runs.py covers both wrappers (stalled→error, fresh survives) — previously untested. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NsmJSQxnNxGgtM5Yz4GAqi
2026-07-13 21:41:34 -04:00
parent 666b3a2ec8
commit c87f8a1bb3
2 changed files with 108 additions and 72 deletions
@@ -0,0 +1,63 @@
+"""recover_stalled_head_training_runs + recover_stalled_head_auto_apply_runs share
+one helper (_recover_stalled_runs, DRY pass #161). These pin BOTH wrappers so the
+shared source stays correct: a 'running' row with no progress past the stall
+threshold flips to 'error'; a fresh 'running' row is left alone."""
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from sqlalchemy import select
+
+from backend.app.models import HeadAutoApplyRun, HeadTrainingRun
+
+pytestmark = pytest.mark.integration
+
+
+def test_recover_stalled_head_training_runs_flips_stalled_keeps_fresh(db_sync):
+    from backend.app.tasks.maintenance import recover_stalled_head_training_runs
+
+    stale = HeadTrainingRun(
+        params={}, status="running",
+        last_progress_at=datetime.now(UTC) - timedelta(days=1),
+    )
+    fresh = HeadTrainingRun(
+        params={}, status="running", last_progress_at=datetime.now(UTC),
+    )
+    db_sync.add_all([stale, fresh])
+    db_sync.commit()
+    stale_id, fresh_id = stale.id, fresh.id
+
+    assert recover_stalled_head_training_runs.apply().get() == 1
+
+    db_sync.expire_all()
+    assert db_sync.execute(
+        select(HeadTrainingRun.status).where(HeadTrainingRun.id == stale_id)
+    ).scalar_one() == "error"
+    assert db_sync.execute(
+        select(HeadTrainingRun.status).where(HeadTrainingRun.id == fresh_id)
+    ).scalar_one() == "running"
+
+
+def test_recover_stalled_head_auto_apply_runs_flips_stalled_keeps_fresh(db_sync):
+    from backend.app.tasks.maintenance import recover_stalled_head_auto_apply_runs
+
+    stale = HeadAutoApplyRun(
+        dry_run=False, params={}, status="running",
+        last_progress_at=datetime.now(UTC) - timedelta(days=1),
+    )
+    fresh = HeadAutoApplyRun(
+        dry_run=False, params={}, status="running",
+        last_progress_at=datetime.now(UTC),
+    )
+    db_sync.add_all([stale, fresh])
+    db_sync.commit()
+    stale_id, fresh_id = stale.id, fresh.id
+
+    assert recover_stalled_head_auto_apply_runs.apply().get() == 1
+
+    db_sync.expire_all()
+    assert db_sync.execute(
+        select(HeadAutoApplyRun.status).where(HeadAutoApplyRun.id == stale_id)
+    ).scalar_one() == "error"
+    assert db_sync.execute(
+        select(HeadAutoApplyRun.status).where(HeadAutoApplyRun.id == fresh_id)
+    ).scalar_one() == "running"