feat(fc2a): add maintenance tasks (recovery + cleanup) with beat schedule

recover_interrupted_tasks runs every 5 minutes, finds ImportTask rows
stuck in 'processing' for >30 minutes (well above any legitimate import
duration), and re-queues them. cleanup_old_tasks runs daily and deletes
finished tasks older than 7 days so the task table stays an operational
view rather than an archive.

Both thresholds match ImageRepo's precedent. The 30-min stuck threshold
is documented inline so a future reader can adjust it intentionally
rather than mistaking it for a 'magic number'.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-14 12:09:09 -04:00
parent d3bb8c509a
commit 509c19ce86
3 changed files with 162 additions and 0 deletions
+11
View File
@@ -44,6 +44,17 @@ def make_celery() -> Celery:
task_acks_late=True,
worker_prefetch_multiplier=1,
broker_connection_retry_on_startup=True,
beat_schedule={
"recover-interrupted-tasks": {
"task": "backend.app.tasks.maintenance.recover_interrupted_tasks",
"schedule": 300.0, # every 5 minutes
},
"cleanup-old-tasks": {
"task": "backend.app.tasks.maintenance.cleanup_old_tasks",
"schedule": 86400.0, # daily
},
},
timezone="UTC",
)
return app
+74
View File
@@ -0,0 +1,74 @@
"""Periodic maintenance: recover stuck import tasks, garbage-collect old finished tasks."""
from datetime import datetime, timedelta, timezone
from sqlalchemy import create_engine, delete, select, update
from sqlalchemy.orm import sessionmaker
from ..celery_app import celery
from ..config import get_config
from ..models import ImportTask
STUCK_THRESHOLD_MINUTES = 30
OLD_TASK_DAYS = 7
def _sync_session_factory():
cfg = get_config()
engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True)
return sessionmaker(engine, expire_on_commit=False)
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
def recover_interrupted_tasks() -> int:
"""Find ImportTask rows stuck in 'processing' for >30 min and re-queue them.
Why 30 min: large videos can legitimately take many minutes to import;
30 is a safe gate that catches actual crashes (which leave the row stuck
forever) without resetting slow-but-still-running jobs.
"""
SessionLocal = _sync_session_factory()
cutoff = datetime.now(timezone.utc) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
with SessionLocal() as session:
stuck_ids = session.execute(
select(ImportTask.id)
.where(ImportTask.status == "processing")
.where(ImportTask.started_at < cutoff)
).scalars().all()
if not stuck_ids:
return 0
session.execute(
update(ImportTask)
.where(ImportTask.id.in_(stuck_ids))
.values(status="queued", started_at=None, error="recovered from stuck state")
)
session.commit()
from .import_file import import_media_file
for tid in stuck_ids:
import_media_file.delay(tid)
return len(stuck_ids)
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks")
def cleanup_old_tasks() -> int:
"""Delete completed/skipped/failed ImportTask rows older than 7 days.
Why 7 days: long enough to debug an issue an operator only notices days
later; short enough that the task table stays a useful operational view
rather than an archive. Matches IR's default.
"""
SessionLocal = _sync_session_factory()
cutoff = datetime.now(timezone.utc) - timedelta(days=OLD_TASK_DAYS)
with SessionLocal() as session:
result = session.execute(
delete(ImportTask)
.where(ImportTask.status.in_(["complete", "skipped", "failed"]))
.where(ImportTask.finished_at < cutoff)
)
session.commit()
return result.rowcount or 0
+77
View File
@@ -0,0 +1,77 @@
"""Unit tests for the maintenance tasks. Eager-mode Celery so the task
function runs synchronously inside the same DB transaction as the test.
"""
from datetime import datetime, timedelta, timezone
import pytest
from backend.app.celery_app import celery
from backend.app.models import ImportBatch, ImportTask
@pytest.fixture(autouse=True)
def eager():
celery.conf.task_always_eager = True
yield
celery.conf.task_always_eager = False
def _make_batch(session) -> int:
batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick")
session.add(batch)
session.flush()
return batch.id
def test_recover_interrupted_only_old(db_sync):
batch_id = _make_batch(db_sync)
now = datetime.now(timezone.utc)
fresh = ImportTask(
batch_id=batch_id, source_path="/import/a.jpg", task_type="media",
status="processing", started_at=now - timedelta(minutes=5),
)
stale = ImportTask(
batch_id=batch_id, source_path="/import/b.jpg", task_type="media",
status="processing", started_at=now - timedelta(hours=2),
)
db_sync.add_all([fresh, stale])
db_sync.commit()
from backend.app.tasks.maintenance import recover_interrupted_tasks
recovered = recover_interrupted_tasks.apply().get()
assert recovered == 1
db_sync.refresh(fresh)
db_sync.refresh(stale)
assert fresh.status == "processing"
assert stale.status == "queued"
assert stale.started_at is None
def test_cleanup_old_deletes_finished_old(db_sync):
batch_id = _make_batch(db_sync)
now = datetime.now(timezone.utc)
old_complete = ImportTask(
batch_id=batch_id, source_path="/import/a.jpg", task_type="media",
status="complete", finished_at=now - timedelta(days=10),
)
recent_complete = ImportTask(
batch_id=batch_id, source_path="/import/b.jpg", task_type="media",
status="complete", finished_at=now - timedelta(days=2),
)
old_pending = ImportTask(
batch_id=batch_id, source_path="/import/c.jpg", task_type="media",
status="pending",
)
db_sync.add_all([old_complete, recent_complete, old_pending])
db_sync.commit()
from backend.app.tasks.maintenance import cleanup_old_tasks
deleted = cleanup_old_tasks.apply().get()
assert deleted == 1
remaining = {t.source_path for t in db_sync.query(ImportTask).all()}
assert remaining == {"/import/b.jpg", "/import/c.jpg"}