"""Periodic maintenance: recover stuck import tasks, garbage-collect old finished tasks.""" import logging import subprocess from datetime import UTC, datetime, timedelta from pathlib import Path from PIL import Image from sqlalchemy import create_engine, delete, select, update from sqlalchemy.orm import sessionmaker from ..celery_app import celery from ..config import get_config from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask from ..utils.phash import compute_phash log = logging.getLogger(__name__) STUCK_THRESHOLD_MINUTES = 30 OLD_TASK_DAYS = 7 PHASH_PAGE = 500 VERIFY_PAGE = 200 FFPROBE_TIMEOUT_SECONDS = 10 def _sync_session_factory(): cfg = get_config() engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) return sessionmaker(engine, expire_on_commit=False) @celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks") def recover_interrupted_tasks() -> int: """Find ImportTask rows stuck in 'processing' for >30 min and re-queue them. Why 30 min: large videos can legitimately take many minutes to import; 30 is a safe gate that catches actual crashes (which leave the row stuck forever) without resetting slow-but-still-running jobs. """ SessionLocal = _sync_session_factory() cutoff = datetime.now(UTC) - timedelta(minutes=STUCK_THRESHOLD_MINUTES) with SessionLocal() as session: stuck_ids = session.execute( select(ImportTask.id) .where(ImportTask.status == "processing") .where(ImportTask.started_at < cutoff) ).scalars().all() if not stuck_ids: return 0 session.execute( update(ImportTask) .where(ImportTask.id.in_(stuck_ids)) .values(status="queued", started_at=None, error="recovered from stuck state") ) session.commit() from .import_file import import_media_file for tid in stuck_ids: import_media_file.delay(tid) return len(stuck_ids) @celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks") def cleanup_old_tasks() -> int: """Delete completed/skipped/failed ImportTask rows older than 7 days. Why 7 days: long enough to debug an issue an operator only notices days later; short enough that the task table stays a useful operational view rather than an archive. Matches IR's default. """ SessionLocal = _sync_session_factory() cutoff = datetime.now(UTC) - timedelta(days=OLD_TASK_DAYS) with SessionLocal() as session: result = session.execute( delete(ImportTask) .where(ImportTask.status.in_(["complete", "skipped", "failed"])) .where(ImportTask.finished_at < cutoff) ) session.commit() return result.rowcount or 0 @celery.task(name="backend.app.tasks.maintenance.backfill_phash") def backfill_phash() -> int: """Recompute phash for stored images that have none (imported before FC-2d-i+ii). Keyset-paginated by id (restart-safe), NULL-only fill, idempotent. Videos legitimately keep phash NULL. A missing/unreadable file is logged and left NULL — never fails the task.""" SessionLocal = _sync_session_factory() updated = 0 last_id = 0 with SessionLocal() as session: while True: rows = session.execute( select(ImageRecord) .where(ImageRecord.id > last_id) .where(ImageRecord.phash.is_(None)) .where(ImageRecord.mime.like("image/%")) .order_by(ImageRecord.id.asc()) .limit(PHASH_PAGE) ).scalars().all() if not rows: break for rec in rows: try: with Image.open(rec.path) as im: ph = compute_phash(im) except Exception as exc: log.warning( "backfill_phash: unreadable %s: %s", rec.path, exc ) ph = None if ph is not None and rec.phash is None: rec.phash = ph updated += 1 session.commit() last_id = rows[-1].id return updated def _verify_one(path: Path, expected_sha: str, mime: str, sha_fn) -> str: """Compute the integrity verdict for one file. Status precedence: failed_verification (can't run) > corrupt (sha mismatch / decode fails) > ok (passes both). Never raises.""" try: if not path.is_file(): return "failed_verification" try: actual_sha = sha_fn(path) except (OSError, PermissionError): return "failed_verification" if actual_sha != expected_sha: return "corrupt" if mime and mime.startswith("image/"): try: with Image.open(path) as im: im.verify() except Exception: return "corrupt" return "ok" if mime and mime.startswith("video/"): try: proc = subprocess.run( ["ffprobe", "-v", "error", "-i", str(path)], capture_output=True, timeout=FFPROBE_TIMEOUT_SECONDS, ) except FileNotFoundError: # ffprobe binary missing — environment problem, not file. return "failed_verification" except subprocess.TimeoutExpired: return "corrupt" return "ok" if proc.returncode == 0 else "corrupt" # Unknown mime — sha matched already; trust that. return "ok" except Exception as exc: log.warning("verify_integrity unexpected error for %s: %s", path, exc) return "failed_verification" @celery.task(name="backend.app.tasks.maintenance.verify_integrity") def verify_integrity() -> int: """Verify every ImageRecord file: sha256 recompute + decode/probe (PIL for images; ffprobe for videos). Writes integrity_status (always — the column reflects the most recent verdict). Keyset-paginated, fail-soft per row, idempotent. Returns the total count verified.""" from ..services.importer import _sha256_of # reuse the importer's helper SessionLocal = _sync_session_factory() total = 0 counts = {"ok": 0, "corrupt": 0, "failed_verification": 0} last_id = 0 with SessionLocal() as session: while True: rows = session.execute( select(ImageRecord) .where(ImageRecord.id > last_id) .order_by(ImageRecord.id.asc()) .limit(VERIFY_PAGE) ).scalars().all() if not rows: break for rec in rows: rec.integrity_status = _verify_one( Path(rec.path), rec.sha256, rec.mime, _sha256_of ) counts[rec.integrity_status] = ( counts.get(rec.integrity_status, 0) + 1 ) total += 1 session.commit() last_id = rows[-1].id log.info("verify_integrity verdicts: %s (total %d)", counts, total) return total @celery.task(name="backend.app.tasks.maintenance.cleanup_old_download_events") def cleanup_old_download_events() -> int: """FC-3d: delete terminal DownloadEvent rows older than the configured retention window. Never touches pending/running rows. Why terminal-only: pending/running rows represent in-flight work whose owning task may still be alive; deleting them would orphan the task. Retention days comes from ImportSettings.download_event_retention_days so the operator can tune without a code change. """ SessionLocal = _sync_session_factory() with SessionLocal() as session: settings = session.execute( select(ImportSettings).where(ImportSettings.id == 1) ).scalar_one() retention_days = settings.download_event_retention_days cutoff = datetime.now(UTC) - timedelta(days=retention_days) result = session.execute( delete(DownloadEvent) .where(DownloadEvent.status.in_(["ok", "error", "skipped"])) .where(DownloadEvent.started_at < cutoff) ) session.commit() return result.rowcount or 0