FabledCurator/backend/app/tasks/maintenance.py

"""Periodic maintenance: recover stuck import tasks, garbage-collect old finished tasks."""

import logging
import os
import subprocess
from datetime import UTC, datetime, timedelta
from pathlib import Path

from PIL import Image
from sqlalchemy import and_, delete, or_, select, update

from ..celery_app import celery
from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask, TaskRun
from ..utils.phash import compute_phash
from ._sync_engine import sync_session_factory as _sync_session_factory

log = logging.getLogger(__name__)

STUCK_THRESHOLD_MINUTES = 5
# Archive ImportTasks run the per-member pipeline inline for every
# member (import_archive_file: soft=30min/hard=35min). The ImportTask
# 'processing' recovery sweep must give them a longer threshold or it
# re-queues a legitimately-running archive mid-import (double-process).
# 40 min = 5-min buffer past the archive task's hard kill.
# Operator-flagged 2026-05-28 (target 1645019, a big archive).
ARCHIVE_STUCK_THRESHOLD_MINUTES = 40

# Poison-pill cap. After being recovered (re-queued from a stuck
# 'processing' state) MAX_RECOVERY_ATTEMPTS-1 times, the next sweep
# marks the row 'failed' instead of looping. 3 = two recoveries then
# give up. A row reaches this only if it leaves NO terminal flip each
# run — i.e. it hard-crashes the worker (OOM/segfault/SIGKILL), the
# signature of a corrupt or oversized input. Caught exceptions already
# flip to terminal 'failed' and never enter this loop.
MAX_RECOVERY_ATTEMPTS = 3
ORPHAN_PENDING_THRESHOLD_MINUTES = 30
OLD_TASK_DAYS = 7
PHASH_PAGE = 500
VERIFY_PAGE = 200
FFPROBE_TIMEOUT_SECONDS = 10
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600       # 24 h
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600  # 7 days

# Overrides for recover_stalled_task_runs (the TaskRun 'running' sweep).
# Tasks/queues that legitimately run longer than the default 5-min
# threshold need their own larger value, else the sweep marks in-flight
# work 'error' before it finishes. Each value MUST be ≥ the relevant
# task.time_limit + a small buffer. task_name overrides take precedence
# over queue overrides.
#
# ml queue: tag_and_embed video branch (≈20 GPU ops); time_limit=1200.
# import_archive_file: shares the 'import' queue with the fast
#   single-file import_media_file, so it needs a task-name override
#   (the import queue itself stays at the 5-min default for single
#   files); time_limit=2100.
QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
    "ml": 25,
}
TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
    "backend.app.tasks.import_file.import_archive_file": 40,
}


@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
def recover_interrupted_tasks() -> int:
    """Recover stuck ImportTask rows. Two distinct stuck states:

    1. 'processing' too long — worker crash mid-import. Re-queue via
       enqueue_import (routing media vs archive) and let the import
       retry. Threshold is task-type-aware: media files are sub-second
       and capped at the 5-min soft limit, so STUCK_THRESHOLD_MINUTES
       (5) means a confirmed crash; archives run the per-member
       pipeline inline (import_archive_file, 35-min hard limit) so they
       get ARCHIVE_STUCK_THRESHOLD_MINUTES (40) to avoid re-queueing a
       still-running archive. (Media was tightened from 30 min to 5
       2026-05-24 after a 2224-row zombie pile; archive split out
       2026-05-28.)

    2. 'pending' or 'queued' > 30 min — enqueue-phase crash. scan_directory
       creates rows with status='pending' (commit), then in a second pass
       transitions to 'queued' and calls .delay() (commit). If the scanner
       crashes between those two commits, rows are orphaned in 'pending'
       (never enqueued) with no recovery path — invisible to the
       'processing' sweep above. Flagged 2026-05-25 by operator hitting a
       5490-row orphan pile. Flip these to 'failed' (not re-enqueue) so
       the operator drains them via /api/import/retry-failed at their own
       pace; bulk-re-enqueueing 5000+ rows would thundering-herd the
       import worker.

    Returns total rows touched (recovered + marked failed).
    """
    SessionLocal = _sync_session_factory()
    now = datetime.now(UTC)
    media_cutoff = now - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
    archive_cutoff = now - timedelta(minutes=ARCHIVE_STUCK_THRESHOLD_MINUTES)
    orphan_cutoff = now - timedelta(minutes=ORPHAN_PENDING_THRESHOLD_MINUTES)
    with SessionLocal() as session:
        # Both sweeps used to be SELECT ids → UPDATE WHERE id IN (...) which
        # blew past psycopg's 65535-parameter ceiling once a sweep covered
        # tens of thousands of rows (operator hit it 2026-05-26 after the
        # /import deep scan piled up orphans). Folding the SELECT into the
        # UPDATE eliminates the IN-list entirely. RETURNING gives us back
        # exactly the (id, task_type) pairs that flipped so the requeue
        # can route media vs archive correctly.
        #
        # Media + archive get separate cutoffs: a single media file is
        # sub-second so 5 min means crash; an archive runs the per-member
        # pipeline inline and can legitimately take up to its 35-min hard
        # limit, so it gets ARCHIVE_STUCK_THRESHOLD_MINUTES (40) to avoid
        # re-queueing a still-running archive.
        stuck_predicate = and_(
            ImportTask.status == "processing",
            or_(
                and_(ImportTask.task_type != "archive",
                     ImportTask.started_at < media_cutoff),
                and_(ImportTask.task_type == "archive",
                     ImportTask.started_at < archive_cutoff),
            ),
        )

        # POISON-PILL CIRCUIT BREAKER (Layer 1, 2026-05-28). A row that
        # leaves no terminal flip (hard worker crash: OOM/segfault/SIGKILL
        # on a corrupt or oversized input) gets re-queued by this sweep —
        # and would loop forever, re-crashing the worker each pass,
        # without a cap. Once a row has already been recovered
        # MAX_RECOVERY_ATTEMPTS-1 times, stop re-queueing it and mark it
        # 'failed' with a diagnostic so the operator can find + replace
        # the offending file. This UPDATE runs FIRST so the rows it
        # claims drop out of 'processing' before the re-queue pass.
        poison_result = session.execute(
            update(ImportTask)
            .where(stuck_predicate)
            .where(ImportTask.recovery_count >= MAX_RECOVERY_ATTEMPTS - 1)
            .values(
                status="failed",
                finished_at=now,
                error=(
                    f"crashed or stalled the worker {MAX_RECOVERY_ATTEMPTS} "
                    f"times without completing — likely a corrupt or "
                    f"oversized input. Not re-queued. Inspect/replace the "
                    f"file, then retry via /api/import/retry-failed."
                ),
            )
            .returning(ImportTask.id)
        )
        poison_ids = [r[0] for r in poison_result.all()]

        # Re-queue the remaining stuck rows (under the cap) and bump
        # their recovery_count. RETURNING (id, task_type) so the requeue
        # routes media vs archive correctly.
        stuck_result = session.execute(
            update(ImportTask)
            .where(stuck_predicate)
            .where(ImportTask.recovery_count < MAX_RECOVERY_ATTEMPTS - 1)
            .values(
                status="queued",
                started_at=None,
                recovery_count=ImportTask.recovery_count + 1,
                error="recovered from stuck state",
            )
            .returning(ImportTask.id, ImportTask.task_type)
        )
        stuck = stuck_result.all()

        orphan_result = session.execute(
            update(ImportTask)
            .where(ImportTask.status.in_(["pending", "queued"]))
            .where(ImportTask.created_at < orphan_cutoff)
            .values(
                status="failed",
                error=(
                    "orphan pending/queued swept by recover_interrupted_tasks "
                    "(scanner likely crashed mid-enqueue); retry via "
                    "/api/import/retry-failed"
                ),
            )
        )
        orphan_count = orphan_result.rowcount or 0

        session.commit()

        if stuck:
            from .import_file import enqueue_import
            for tid, task_type in stuck:
                enqueue_import(tid, task_type)

        # Layer-2 auto re-download (env-gated, default OFF). For each
        # poison-pill row that resolves to a pollable Source, delete the
        # bad file and trigger ONE source re-check to fetch a fresh
        # copy. Bounded by ImportTask.refetched so source-side
        # corruption can't loop. The 'failed' row stays as history; the
        # re-downloaded file re-imports as a fresh task on the next scan.
        if poison_ids and os.environ.get("FC_AUTO_REFETCH_CORRUPT", "0") == "1":
            from ..models import ImportSettings
            from ..services.refetch_service import attempt_refetch
            import_root = Path(session.execute(
                select(ImportSettings.import_scan_path)
                .where(ImportSettings.id == 1)
            ).scalar_one())
            for pid in poison_ids:
                ptask = session.get(ImportTask, pid)
                if ptask is None:
                    continue
                try:
                    attempt_refetch(session, ptask, import_root)
                except Exception as exc:  # noqa: BLE001 — best-effort
                    log.warning("auto-refetch failed for task %s: %s", pid, exc)

        return len(stuck) + len(poison_ids) + orphan_count


@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks")
def cleanup_old_tasks() -> int:
    """Delete completed/skipped/failed ImportTask rows older than 7 days.

    Why 7 days: long enough to debug an issue an operator only notices days
    later; short enough that the task table stays a useful operational view
    rather than an archive. Matches IR's default.
    """
    SessionLocal = _sync_session_factory()
    cutoff = datetime.now(UTC) - timedelta(days=OLD_TASK_DAYS)
    with SessionLocal() as session:
        result = session.execute(
            delete(ImportTask)
            .where(ImportTask.status.in_(["complete", "skipped", "failed"]))
            .where(ImportTask.finished_at < cutoff)
        )
        session.commit()
        return result.rowcount or 0


@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
def recover_stalled_task_runs() -> int:
    """Flip task_run rows stuck in 'running' past their queue-specific
    threshold to 'error'. FC-3i.

    A row gets stuck when the worker dies without emitting
    task_postrun / task_failure (e.g. OOM, container restart between
    signals, signal handler raised+logged). The default 5-min threshold
    fits short-lived queues (import/thumbnail/download); queues that
    legitimately run longer tasks (ml-video, deep scans) get their
    own larger threshold via QUEUE_STUCK_THRESHOLD_MINUTES so the
    sweep doesn't preempt them.

    Runs once per distinct threshold value: each pass updates rows
    whose queue maps to that threshold.
    """
    SessionLocal = _sync_session_factory()
    now = datetime.now(UTC)
    override_tasks = set(TASK_STUCK_THRESHOLD_MINUTES.keys())
    override_queues = set(QUEUE_STUCK_THRESHOLD_MINUTES.keys())
    total = 0

    def _flag(minutes, *extra_where):
        cutoff = now - timedelta(minutes=minutes)
        stmt = (
            update(TaskRun)
            .where(TaskRun.status == "running")
            .where(TaskRun.started_at < cutoff)
            .values(
                status="error",
                error_type="RecoverySweep",
                error_message=(
                    f"no completion signal received within {minutes} min"
                ),
                finished_at=now,
            )
        )
        for w in extra_where:
            stmt = stmt.where(w)
        return session.execute(stmt).rowcount or 0

    with SessionLocal() as session:
        # Precedence: task_name override → queue override → default.
        # Each pass excludes rows claimed by a higher-precedence pass so
        # every row is touched at most once.

        # 1. Per-task-name overrides (e.g. import_archive_file, which
        #    shares the 'import' queue with fast single-file imports).
        for task_name, minutes in TASK_STUCK_THRESHOLD_MINUTES.items():
            total += _flag(minutes, TaskRun.task_name == task_name)

        # 2. Per-queue overrides, excluding the override task-names.
        for queue, minutes in QUEUE_STUCK_THRESHOLD_MINUTES.items():
            wheres = [TaskRun.queue == queue]
            if override_tasks:
                wheres.append(TaskRun.task_name.notin_(override_tasks))
            total += _flag(minutes, *wheres)

        # 3. Default — everything not claimed above.
        default_wheres = []
        if override_queues:
            default_wheres.append(TaskRun.queue.notin_(override_queues))
        if override_tasks:
            default_wheres.append(TaskRun.task_name.notin_(override_tasks))
        total += _flag(STUCK_THRESHOLD_MINUTES, *default_wheres)

        session.commit()
        return total


@celery.task(name="backend.app.tasks.maintenance.prune_task_runs")
def prune_task_runs() -> dict:
    """Daily retention for task_run rows. FC-3i.

    - 'ok' rows: deleted after TASK_RUN_KEEP_OK_SECONDS (24h default).
      Success is high-volume, not interesting after a day.
    - 'error' / 'timeout' rows: deleted after TASK_RUN_KEEP_FAILURE_SECONDS
      (7 days default). Failures are operationally interesting longer.
    - 'running' rows: NEVER deleted by this task. The recovery sweep
      (recover_stalled_task_runs) is the mechanism that flips them to
      terminal state; prune doesn't touch in-flight state.
    - 'retry' rows: treated as failures (>7d).

    Returns dict of how many rows were deleted in each bucket.
    """
    SessionLocal = _sync_session_factory()
    now = datetime.now(UTC)
    ok_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_OK_SECONDS)
    fail_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_FAILURE_SECONDS)
    with SessionLocal() as session:
        ok_deleted = session.execute(
            delete(TaskRun)
            .where(TaskRun.status == "ok")
            .where(TaskRun.finished_at < ok_cutoff)
        ).rowcount or 0
        fail_deleted = session.execute(
            delete(TaskRun)
            .where(TaskRun.status.in_(["error", "timeout", "retry"]))
            .where(TaskRun.finished_at < fail_cutoff)
        ).rowcount or 0
        session.commit()
        return {"ok_deleted": ok_deleted, "failures_deleted": fail_deleted}


@celery.task(name="backend.app.tasks.maintenance.backfill_phash")
def backfill_phash() -> int:
    """Recompute phash for stored images that have none (imported before
    FC-2d-i+ii). Keyset-paginated by id (restart-safe), NULL-only fill,
    idempotent. Videos legitimately keep phash NULL. A missing/unreadable
    file is logged and left NULL — never fails the task."""
    SessionLocal = _sync_session_factory()
    updated = 0
    last_id = 0
    with SessionLocal() as session:
        while True:
            rows = session.execute(
                select(ImageRecord)
                .where(ImageRecord.id > last_id)
                .where(ImageRecord.phash.is_(None))
                .where(ImageRecord.mime.like("image/%"))
                .order_by(ImageRecord.id.asc())
                .limit(PHASH_PAGE)
            ).scalars().all()
            if not rows:
                break
            for rec in rows:
                try:
                    with Image.open(rec.path) as im:
                        ph = compute_phash(im)
                except Exception as exc:
                    log.warning(
                        "backfill_phash: unreadable %s: %s", rec.path, exc
                    )
                    ph = None
                if ph is not None and rec.phash is None:
                    rec.phash = ph
                    updated += 1
            session.commit()
            last_id = rows[-1].id
    return updated


def _verify_one(path: Path, expected_sha: str, mime: str, sha_fn) -> str:
    """Compute the integrity verdict for one file. Status precedence:
    failed_verification (can't run) > corrupt (sha mismatch / decode
    fails) > ok (passes both). Never raises."""
    try:
        if not path.is_file():
            return "failed_verification"
        try:
            actual_sha = sha_fn(path)
        except (OSError, PermissionError):
            return "failed_verification"
        if actual_sha != expected_sha:
            return "corrupt"
        if mime and mime.startswith("image/"):
            try:
                with Image.open(path) as im:
                    im.verify()
            except Exception:
                return "corrupt"
            return "ok"
        if mime and mime.startswith("video/"):
            try:
                proc = subprocess.run(
                    ["ffprobe", "-v", "error", "-i", str(path)],
                    capture_output=True,
                    timeout=FFPROBE_TIMEOUT_SECONDS,
                )
            except FileNotFoundError:
                # ffprobe binary missing — environment problem, not file.
                return "failed_verification"
            except subprocess.TimeoutExpired:
                return "corrupt"
            return "ok" if proc.returncode == 0 else "corrupt"
        # Unknown mime — sha matched already; trust that.
        return "ok"
    except Exception as exc:
        log.warning("verify_integrity unexpected error for %s: %s", path, exc)
        return "failed_verification"


@celery.task(name="backend.app.tasks.maintenance.verify_integrity")
def verify_integrity() -> int:
    """Verify every ImageRecord file: sha256 recompute + decode/probe
    (PIL for images; ffprobe for videos). Writes integrity_status
    (always — the column reflects the most recent verdict).
    Keyset-paginated, fail-soft per row, idempotent. Returns the total
    count verified."""
    from ..services.importer import _sha256_of  # reuse the importer's helper

    SessionLocal = _sync_session_factory()
    total = 0
    counts = {"ok": 0, "corrupt": 0, "failed_verification": 0}
    last_id = 0
    with SessionLocal() as session:
        while True:
            rows = session.execute(
                select(ImageRecord)
                .where(ImageRecord.id > last_id)
                .order_by(ImageRecord.id.asc())
                .limit(VERIFY_PAGE)
            ).scalars().all()
            if not rows:
                break
            for rec in rows:
                rec.integrity_status = _verify_one(
                    Path(rec.path), rec.sha256, rec.mime, _sha256_of
                )
                counts[rec.integrity_status] = (
                    counts.get(rec.integrity_status, 0) + 1
                )
                total += 1
            session.commit()
            last_id = rows[-1].id
    log.info("verify_integrity verdicts: %s (total %d)", counts, total)
    return total


@celery.task(name="backend.app.tasks.maintenance.cleanup_old_download_events")
def cleanup_old_download_events() -> int:
    """FC-3d: delete terminal DownloadEvent rows older than the configured
    retention window. Never touches pending/running rows.

    Why terminal-only: pending/running rows represent in-flight work whose
    owning task may still be alive; deleting them would orphan the task.
    Retention days comes from ImportSettings.download_event_retention_days
    so the operator can tune without a code change.
    """
    SessionLocal = _sync_session_factory()
    with SessionLocal() as session:
        settings = session.execute(
            select(ImportSettings).where(ImportSettings.id == 1)
        ).scalar_one()
        retention_days = settings.download_event_retention_days
        cutoff = datetime.now(UTC) - timedelta(days=retention_days)
        result = session.execute(
            delete(DownloadEvent)
            .where(DownloadEvent.status.in_(["ok", "error", "skipped"]))
            .where(DownloadEvent.started_at < cutoff)
        )
        session.commit()
        return result.rowcount or 0