fix(external): recovery-sweep threshold + queue recording + split fetch timeouts (#883) #116

Merged
bvandeusen merged 4 commits from dev into main 2026-06-16 21:24:31 -04:00
2 changed files with 61 additions and 0 deletions
Showing only changes of commit 258c77dfcd - Show all commits
+8
View File
@@ -147,6 +147,14 @@ TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
"backend.app.tasks.backup.restore_images_task": 420,
# Library audit scans the full library — 2h hard limit.
"backend.app.tasks.library_audit.scan_library_for_rule": 130,
# External file-host fetches (mega/gdrive/film packs) legitimately run to
# the task's 60-min hard limit (time_limit=3600; per-fetch _FETCH_TIMEOUT
# is 50min). Its TaskRun records queue='default' (no queue override), so
# without this it fell to the 5-min default and healthy in-flight fetches
# were phantom-flagged 'RecoverySweep' before their own timeout/error could
# surface (operator-flagged 2026-06-17, target 414 swept at 6.6min). A
# task-name override is robust whatever queue the row records. 65 = 60 + 5.
"backend.app.tasks.external.fetch_external_link": 65,
}
+53
View File
@@ -453,6 +453,59 @@ def test_recover_stalled_task_runs_archive_task_uses_longer_threshold(db_sync):
assert _status(archive_stale_id) == "error"
def test_recover_stalled_task_runs_external_fetch_uses_longer_threshold(db_sync):
"""fetch_external_link legitimately runs to its 60-min hard limit, but its
TaskRun records queue='default' (no queue override), so before the
task-name override (65 min) it fell to the 5-min default and healthy
in-flight fetches were phantom-flagged 'RecoverySweep' before their own
timeout/error could surface (operator-flagged 2026-06-17, target 414 swept
at 6.6min). A 10-min-old row must survive; a 70-min-old one is flagged."""
from sqlalchemy import select
from backend.app.models import TaskRun
from backend.app.tasks.maintenance import recover_stalled_task_runs
name = "backend.app.tasks.external.fetch_external_link"
now = datetime.now(UTC)
# 10-min-old: stale by the default 5-min rule but fresh by the 65-min
# task-name override. Must survive despite recording queue='default'.
fresh_id = _make_task_run(
db_sync, status="running", queue="default", task_name=name,
started_at=now - timedelta(minutes=10),
)
# 70-min-old: past even the 65-min override (a genuine hard kill). Flagged.
stale_id = _make_task_run(
db_sync, status="running", queue="default", task_name=name,
started_at=now - timedelta(minutes=70),
)
db_sync.commit()
recovered = recover_stalled_task_runs.apply().get()
assert recovered == 1
db_sync.expire_all()
def _status(_id):
return db_sync.execute(
select(TaskRun.status).where(TaskRun.id == _id)
).scalar_one()
assert _status(fresh_id) == "running"
assert _status(stale_id) == "error"
def test_external_fetch_stuck_threshold_exceeds_hard_time_limit():
"""Invariant guard (maintenance.py:112): the fetch_external_link task-name
override MUST be ≥ its hard time_limit, else the sweep flags healthy long
fetches. Pins it so a future time_limit bump can't silently re-break it."""
from backend.app.tasks.external import fetch_external_link
from backend.app.tasks.maintenance import TASK_STUCK_THRESHOLD_MINUTES
hard_minutes = fetch_external_link.time_limit / 60
override = TASK_STUCK_THRESHOLD_MINUTES[
"backend.app.tasks.external.fetch_external_link"
]
assert override >= hard_minutes
def test_prune_task_runs_deletes_ok_older_than_24h(db_sync):
from sqlalchemy import select