feat(fc2a): add maintenance tasks (recovery + cleanup) with beat schedule

recover_interrupted_tasks runs every 5 minutes, finds ImportTask rows stuck in 'processing' for >30 minutes (well above any legitimate import duration), and re-queues them. cleanup_old_tasks runs daily and deletes finished tasks older than 7 days so the task table stays an operational view rather than an archive. Both thresholds match ImageRepo's precedent. The 30-min stuck threshold is documented inline so a future reader can adjust it intentionally rather than mistaking it for a 'magic number'. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-14 12:09:09 -04:00
parent d3bb8c509a
commit 509c19ce86
3 changed files with 162 additions and 0 deletions
@@ -0,0 +1,74 @@
+"""Periodic maintenance: recover stuck import tasks, garbage-collect old finished tasks."""
+
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import create_engine, delete, select, update
+from sqlalchemy.orm import sessionmaker
+
+from ..celery_app import celery
+from ..config import get_config
+from ..models import ImportTask
+
+
+STUCK_THRESHOLD_MINUTES = 30
+OLD_TASK_DAYS = 7
+
+
+def _sync_session_factory():
+    cfg = get_config()
+    engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True)
+    return sessionmaker(engine, expire_on_commit=False)
+
+
+@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
+def recover_interrupted_tasks() -> int:
+    """Find ImportTask rows stuck in 'processing' for >30 min and re-queue them.
+
+    Why 30 min: large videos can legitimately take many minutes to import;
+    30 is a safe gate that catches actual crashes (which leave the row stuck
+    forever) without resetting slow-but-still-running jobs.
+    """
+    SessionLocal = _sync_session_factory()
+    cutoff = datetime.now(timezone.utc) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
+    with SessionLocal() as session:
+        stuck_ids = session.execute(
+            select(ImportTask.id)
+            .where(ImportTask.status == "processing")
+            .where(ImportTask.started_at < cutoff)
+        ).scalars().all()
+
+        if not stuck_ids:
+            return 0
+
+        session.execute(
+            update(ImportTask)
+            .where(ImportTask.id.in_(stuck_ids))
+            .values(status="queued", started_at=None, error="recovered from stuck state")
+        )
+        session.commit()
+
+        from .import_file import import_media_file
+        for tid in stuck_ids:
+            import_media_file.delay(tid)
+
+        return len(stuck_ids)
+
+
+@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks")
+def cleanup_old_tasks() -> int:
+    """Delete completed/skipped/failed ImportTask rows older than 7 days.
+
+    Why 7 days: long enough to debug an issue an operator only notices days
+    later; short enough that the task table stays a useful operational view
+    rather than an archive. Matches IR's default.
+    """
+    SessionLocal = _sync_session_factory()
+    cutoff = datetime.now(timezone.utc) - timedelta(days=OLD_TASK_DAYS)
+    with SessionLocal() as session:
+        result = session.execute(
+            delete(ImportTask)
+            .where(ImportTask.status.in_(["complete", "skipped", "failed"]))
+            .where(ImportTask.finished_at < cutoff)
+        )
+        session.commit()
+        return result.rowcount or 0