bvandeusen
c87f8a1bb3
recover_stalled_head_training_runs and recover_stalled_head_auto_apply_runs were near-exact copies (coalesce-flip + keep-last-N prune, differing only in model + two constants). Extract _recover_stalled_runs(model, stall_minutes, keep_runs, label); the two tasks become thin wrappers. The other two recover tasks are deliberately NOT folded in (library-audit has no prune tail; backup uses a single started_at cutoff). test_recover_stalled_head_runs.py covers both wrappers (stalled→error, fresh survives) — previously untested. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NsmJSQxnNxGgtM5Yz4GAqi
64 lines
2.2 KiB
Python
64 lines
2.2 KiB
Python
"""recover_stalled_head_training_runs + recover_stalled_head_auto_apply_runs share
|
|
one helper (_recover_stalled_runs, DRY pass #161). These pin BOTH wrappers so the
|
|
shared source stays correct: a 'running' row with no progress past the stall
|
|
threshold flips to 'error'; a fresh 'running' row is left alone."""
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
import pytest
|
|
from sqlalchemy import select
|
|
|
|
from backend.app.models import HeadAutoApplyRun, HeadTrainingRun
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
def test_recover_stalled_head_training_runs_flips_stalled_keeps_fresh(db_sync):
|
|
from backend.app.tasks.maintenance import recover_stalled_head_training_runs
|
|
|
|
stale = HeadTrainingRun(
|
|
params={}, status="running",
|
|
last_progress_at=datetime.now(UTC) - timedelta(days=1),
|
|
)
|
|
fresh = HeadTrainingRun(
|
|
params={}, status="running", last_progress_at=datetime.now(UTC),
|
|
)
|
|
db_sync.add_all([stale, fresh])
|
|
db_sync.commit()
|
|
stale_id, fresh_id = stale.id, fresh.id
|
|
|
|
assert recover_stalled_head_training_runs.apply().get() == 1
|
|
|
|
db_sync.expire_all()
|
|
assert db_sync.execute(
|
|
select(HeadTrainingRun.status).where(HeadTrainingRun.id == stale_id)
|
|
).scalar_one() == "error"
|
|
assert db_sync.execute(
|
|
select(HeadTrainingRun.status).where(HeadTrainingRun.id == fresh_id)
|
|
).scalar_one() == "running"
|
|
|
|
|
|
def test_recover_stalled_head_auto_apply_runs_flips_stalled_keeps_fresh(db_sync):
|
|
from backend.app.tasks.maintenance import recover_stalled_head_auto_apply_runs
|
|
|
|
stale = HeadAutoApplyRun(
|
|
dry_run=False, params={}, status="running",
|
|
last_progress_at=datetime.now(UTC) - timedelta(days=1),
|
|
)
|
|
fresh = HeadAutoApplyRun(
|
|
dry_run=False, params={}, status="running",
|
|
last_progress_at=datetime.now(UTC),
|
|
)
|
|
db_sync.add_all([stale, fresh])
|
|
db_sync.commit()
|
|
stale_id, fresh_id = stale.id, fresh.id
|
|
|
|
assert recover_stalled_head_auto_apply_runs.apply().get() == 1
|
|
|
|
db_sync.expire_all()
|
|
assert db_sync.execute(
|
|
select(HeadAutoApplyRun.status).where(HeadAutoApplyRun.id == stale_id)
|
|
).scalar_one() == "error"
|
|
assert db_sync.execute(
|
|
select(HeadAutoApplyRun.status).where(HeadAutoApplyRun.id == fresh_id)
|
|
).scalar_one() == "running"
|