imagerepo/app/tasks/import_file.py

# app/tasks/import_file.py
"""
File import tasks for images, videos, and archives.

These tasks handle the actual import of individual files,
including filtering, duplicate detection, and storage.

Supports two modes:
- Quick import (default): Fast duplicate detection by content hash only.
  Skips pHash comparison for speed.
- Deep import (via deep_scan): Full reprocessing with pHash comparison,
  optional thumbnail regeneration, and sidecar re-application.
"""
import os
import shutil
import tempfile
import logging
from datetime import datetime
from functools import lru_cache

import imagehash

from app.celery_app import celery
from app import db
from app.models import ImportTask, ImageRecord, Tag, ArchiveRecord

log = logging.getLogger('celery.tasks.import_file')

# Cache import settings to avoid repeated DB/file reads
_settings_cache = None
_settings_cache_time = None
SETTINGS_CACHE_TTL = 300  # 5 minutes


def _get_cached_settings() -> dict:
    """
    Get import settings with caching.
    Caches for 5 minutes to avoid repeated DB/file reads during batch imports.
    """
    global _settings_cache, _settings_cache_time
    from app.utils.image_importer import load_import_settings

    now = datetime.utcnow()
    if _settings_cache is None or _settings_cache_time is None:
        _settings_cache = load_import_settings()
        _settings_cache_time = now
    elif (now - _settings_cache_time).total_seconds() > SETTINGS_CACHE_TTL:
        _settings_cache = load_import_settings()
        _settings_cache_time = now

    return _settings_cache


def _find_best_sidecar(src_path: str, archive_path: str = None,
                       archive_sidecar_path: str = None) -> str | None:
    """
    Find the best sidecar JSON for a file, checking multiple locations.

    For files extracted from archives, checks:
    1. Per-file sidecar next to extracted file (temp dir)
    2. Per-file sidecar next to original archive (by filename match)
    3. Archive's sidecar as fallback (applies to all files in archive)

    Args:
        src_path: Path to the source file
        archive_path: Path to original archive (if extracted from archive)
        archive_sidecar_path: Path to archive's sidecar JSON

    Returns:
        Path to the best sidecar JSON found, or None
    """
    from app.utils.metadata_enrichment import find_sidecar_json
    import re

    # First try: per-file sidecar next to the source file
    sidecar = find_sidecar_json(src_path)
    if sidecar:
        return sidecar

    # Second try: per-file sidecar next to original archive
    if archive_path:
        from pathlib import Path
        archive_dir = Path(archive_path).parent
        filename = Path(src_path).name
        stem = Path(src_path).stem

        # Check for exact stem match
        per_file_json = archive_dir / f"{stem}.json"
        if per_file_json.exists():
            return str(per_file_json)

        # Check for stem without numeric prefix
        stem_no_prefix = re.sub(r'^\d+_', '', stem)
        if stem_no_prefix != stem:
            alt_json = archive_dir / f"{stem_no_prefix}.json"
            if alt_json.exists():
                return str(alt_json)

    # Third try: archive-level sidecar (fallback for all files)
    if archive_sidecar_path and os.path.exists(archive_sidecar_path):
        return archive_sidecar_path

    return None


@celery.task(bind=True, name='app.tasks.import_file.import_media_file',
             max_retries=3, default_retry_delay=60)
def import_media_file(self, task_id: int):
    """
    Import a single media file (image or video).

    Supports two modes based on context.deep_scan flag:
    - Quick mode (default): Fast duplicate detection by content hash only.
      Skips pHash comparison for speed.
    - Deep mode: Full reprocessing with pHash comparison, optional
      thumbnail regeneration, and sidecar re-application.

    Steps:
    1. Load ImportTask record
    2. Apply filters (dimensions, transparency, single-color)
    3. Check for duplicates (hash, and pHash in deep mode)
    4. Copy to content-addressed storage
    5. Create ImageRecord
    6. Generate thumbnail
    7. Queue sidecar metadata task
    8. Update ImportTask status

    Args:
        task_id: ID of the ImportTask record

    Returns:
        dict with status and result info
    """
    from app.tasks.thumbnail import generate_thumbnail_task
    from app.tasks.sidecar import apply_sidecar_metadata
    from app.utils.image_importer import (
        extract_metadata, calculate_hash, calculate_perceptual_hash,
        is_mostly_transparent, is_mostly_single_color, find_similar_image,
        build_hashed_dest_path, supersede_image,
        generate_thumbnail, generate_video_thumbnail_mirrored,
        transcode_video_to_mp4, needs_transcode
    )
    task = ImportTask.query.get(task_id)
    if not task:
        log.error(f"Task {task_id} not found")
        return {'error': 'Task not found'}

    task.status = 'processing'
    task.started_at = datetime.utcnow()
    db.session.commit()

    try:
        src_path = task.source_path
        context = task.context or {}
        artist = context.get('artist', 'unknown')
        dest_dir = context.get('dest_dir', '/images')

        # Deep scan options
        is_deep_scan = context.get('deep_scan', False)
        regenerate_thumbnails = context.get('regenerate_thumbnails', False)
        reapply_sidecars = context.get('reapply_sidecars', False)
        verify_hashes = context.get('verify_hashes', False)

        # Archive sidecar (for files extracted from archives)
        archive_sidecar_path = context.get('archive_sidecar_path')
        archive_path = context.get('archive_path')

        log.info(f"Processing: {src_path} (deep_scan={is_deep_scan})")

        # Load settings with caching (avoids repeated DB reads)
        settings = _get_cached_settings()

        # Validate file exists
        if not os.path.exists(src_path):
            return _fail_task(task, 'Source file not found')

        filename = os.path.basename(src_path)
        lower = filename.lower()

        # Extract metadata first for filtering
        metadata = extract_metadata(src_path)
        if not metadata.get('width') or not metadata.get('height'):
            return _skip_task(task, 'Missing dimension data')

        # Apply dimension filter
        min_width = settings.get('min_width', 0)
        min_height = settings.get('min_height', 0)
        if min_width > 0 and metadata['width'] < min_width:
            return _skip_task(task, f"Too small: {metadata['width']}x{metadata['height']}")
        if min_height > 0 and metadata['height'] < min_height:
            return _skip_task(task, f"Too small: {metadata['width']}x{metadata['height']}")

        # Apply transparency filter
        if settings.get('skip_transparent') and lower.endswith(('.png', '.gif', '.webp')):
            threshold = settings.get('transparency_threshold', 0.9)
            if is_mostly_transparent(src_path, threshold):
                return _skip_task(task, 'Mostly transparent')

        # Apply single-color filter
        if settings.get('skip_single_color', True) and lower.endswith(('.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp')):
            if is_mostly_single_color(src_path,
                                      settings.get('single_color_threshold', 0.95),
                                      settings.get('single_color_tolerance', 30)):
                return _skip_task(task, 'Mostly single color')

        # Compute content hash
        file_hash = calculate_hash(src_path)
        task.file_hash = file_hash

        # Check for exact duplicate by hash
        existing = ImageRecord.query.filter_by(hash=file_hash).first()
        if existing:
            # Merge archive tag if this file came from an archive
            # This ensures the existing image gets tagged with all archives it appears in
            archive_tag_id = context.get('archive_tag_id')
            if archive_tag_id:
                _add_archive_tag(existing, archive_tag_id)
                db.session.commit()
                log.info(f"Added archive tag {archive_tag_id} to existing image {existing.id}")

            # In deep scan mode with reapply_sidecars, always try to enrich
            # In quick mode, also try if sidecar exists
            # Uses archive sidecar as fallback for files from archives
            sidecar_path = _find_best_sidecar(src_path, archive_path, archive_sidecar_path)
            if sidecar_path and (is_deep_scan and reapply_sidecars or not is_deep_scan):
                apply_sidecar_metadata.delay(existing.id, sidecar_path)

            # In deep scan with regenerate_thumbnails, regenerate even for duplicates
            if is_deep_scan and regenerate_thumbnails:
                _regenerate_thumbnail(existing)

            return _skip_task(task, 'Duplicate by hash', result_image_id=existing.id)

        # Supersede-by-name for previously-flagged corrupt records.
        # If the user (or the downloader) drops a fresh copy of a file we
        # earlier marked corrupt at /import/<artist>/<filename>, the bytes
        # don't match (so the hash dedup above missed it) but the source
        # path does. Treat it as a supersede so tags/series/embeddings
        # carry over and the old corrupt /images/... file gets replaced.
        flagged = (
            ImageRecord.query
            .filter(ImageRecord.integrity_status.in_(('truncated', 'unreadable', 'missing')))
            .filter(ImageRecord.filename == filename)
            .filter(ImageRecord.filepath.like(f"%/{artist}/%"))
            .first()
        )
        if flagged is not None:
            phash = calculate_perceptual_hash(src_path)
            log.info(
                f"Supersede-on-replace: fresh /import/{artist}/{filename} replaces "
                f"flagged image_id={flagged.id} (status={flagged.integrity_status})"
            )
            return _supersede_existing(
                task, flagged, src_path, file_hash, phash,
                metadata, artist, dest_dir, filename,
                archive_path, archive_sidecar_path,
            )

        # pHash similarity check
        # Quick mode: SKIP pHash comparison (relies on content hash for duplicates)
        # Deep mode: Full pHash comparison for similarity detection
        phash = None
        if is_deep_scan:
            phash = calculate_perceptual_hash(src_path)
            phash_threshold = settings.get('phash_threshold', 10)

            if phash and phash_threshold > 0:
                # Load existing pHashes for comparison
                # Note: In production with millions of images, consider using
                # a spatial index or pre-computed pHash buckets
                existing_phashes = [
                    (imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height, img.id)
                    for img in ImageRecord.query.filter(ImageRecord.perceptual_hash.isnot(None)).all()
                ]

                relationship, similar_id = find_similar_image(
                    phash, metadata['width'], metadata['height'],
                    existing_phashes, threshold=phash_threshold
                )

                if relationship == 'larger_exists':
                    # Merge archive tag if this file came from an archive
                    archive_tag_id = context.get('archive_tag_id')
                    if archive_tag_id and similar_id:
                        similar_record = ImageRecord.query.get(similar_id)
                        if similar_record:
                            _add_archive_tag(similar_record, archive_tag_id)
                            db.session.commit()
                            log.info(f"Added archive tag {archive_tag_id} to similar image {similar_id}")

                    # Still try to enrich the existing larger image with sidecar metadata
                    # Uses archive sidecar as fallback for files from archives
                    sidecar_path = _find_best_sidecar(src_path, archive_path, archive_sidecar_path)
                    if sidecar_path and similar_id:
                        apply_sidecar_metadata.delay(similar_id, sidecar_path)
                    return _skip_task(task, 'Similar larger image exists', result_image_id=similar_id)

                # Handle supersede case - new image is larger than existing
                if relationship == 'smaller_exists' and settings.get('supersede_smaller', True) and similar_id:
                    old_record = ImageRecord.query.get(similar_id)
                    if old_record:
                        return _supersede_existing(
                            task, old_record, src_path, file_hash, phash,
                            metadata, artist, dest_dir, filename,
                            archive_path, archive_sidecar_path
                        )
        else:
            # Quick mode: only compute pHash for storage, not comparison
            phash = calculate_perceptual_hash(src_path)

        # Normal import - copy to destination
        target_dir = os.path.join(dest_dir, artist)
        os.makedirs(target_dir, exist_ok=True)

        # Transcode video if needed (non-MP4 formats to H.264 MP4)
        transcoded_temp = None
        actual_src = src_path
        actual_filename = filename

        if needs_transcode(src_path):
            log.info(f"Transcoding video: {filename}")
            # Transcode to temp location first
            transcoded_temp = transcode_video_to_mp4(src_path)
            if transcoded_temp:
                actual_src = transcoded_temp
                # Update filename to .mp4 extension
                actual_filename = os.path.splitext(filename)[0] + '.mp4'
                # Recalculate hash for transcoded file
                file_hash = calculate_hash(transcoded_temp)
                task.file_hash = file_hash
                log.info(f"Transcoded successfully: {actual_filename}")
            else:
                log.warning(f"Transcode failed for {filename}, importing original")

        dest_path = build_hashed_dest_path(target_dir, actual_filename, file_hash)
        shutil.copy2(actual_src, dest_path)

        # Clean up transcoded temp file
        if transcoded_temp and os.path.exists(transcoded_temp):
            os.remove(transcoded_temp)

        # Generate thumbnail
        try:
            if dest_path.lower().endswith(('.mp4', '.mov', '.webm', '.avi', '.mkv')):
                thumb_path = generate_video_thumbnail_mirrored(dest_path)
            else:
                thumb_path = generate_thumbnail(dest_path)
        except Exception as e:
            log.warning(f"Thumbnail generation failed for {actual_filename}: {e}")
            thumb_path = None

        # Create ImageRecord
        # Use actual_filename in case video was transcoded to .mp4
        record = ImageRecord(
            filename=actual_filename,
            filepath=dest_path,
            thumb_path=thumb_path,
            hash=file_hash,
            perceptual_hash=str(phash) if phash else None,
            file_size=os.path.getsize(dest_path),  # Use actual file size after transcode
            width=metadata['width'],
            height=metadata['height'],
            format=metadata['format'] if not needs_transcode(src_path) else 'mp4',
            camera_model=metadata.get('camera_model'),
            taken_at=metadata.get('taken_at'),
            imported_at=datetime.utcnow()
        )

        # Add artist tag
        _add_artist_tag(record, artist)

        # Add archive tag if this file came from an archive
        archive_tag_id = context.get('archive_tag_id')
        if archive_tag_id:
            _add_archive_tag(record, archive_tag_id)

        db.session.add(record)
        db.session.commit()

        # Complete task
        task.status = 'complete'
        task.result_image_id = record.id
        task.completed_at = datetime.utcnow()
        db.session.commit()

        # Queue sidecar metadata enrichment
        # Uses archive sidecar as fallback for files from archives
        sidecar_path = _find_best_sidecar(src_path, archive_path, archive_sidecar_path)
        if sidecar_path:
            apply_sidecar_metadata.delay(record.id, sidecar_path)

        # Queue ML inference (WD14 tagging + SigLIP embedding)
        try:
            from app.tasks.ml import tag_and_embed
            tag_and_embed.apply_async(args=[record.id], queue='ml')
        except Exception as ml_err:
            # Never let an enqueue failure roll back an otherwise-successful import.
            log.warning(f"Could not enqueue tag_and_embed for image {record.id}: {ml_err}")

        # Queue structural integrity verification. Runs cheaply on the
        # maintenance queue; result lands as integrity_status on the row.
        try:
            from app.tasks.maintenance import verify_media_integrity
            verify_media_integrity.delay(record.id)
        except Exception as v_err:
            log.warning(f"Could not enqueue verify_media_integrity for image {record.id}: {v_err}")

        # Update batch stats
        if task.batch_id:
            from app.tasks.scan import update_batch_stats
            update_batch_stats.delay(task.batch_id)

        log.info(f"Imported: {actual_filename} -> {dest_path}")

        return {
            'status': 'imported',
            'image_id': record.id,
            'dest_path': dest_path
        }

    except Exception as e:
        db.session.rollback()
        log.error(f"Import failed for task {task_id}: {e}", exc_info=True)

        task.retry_count += 1
        if task.retry_count < 3:
            task.status = 'queued'
            db.session.commit()
            raise self.retry(exc=e)
        else:
            return _fail_task(task, str(e))


@celery.task(bind=True, name='app.tasks.import_file.import_archive',
             max_retries=2, default_retry_delay=120,
             soft_time_limit=1800, time_limit=3600)  # 30min soft, 60min hard for large archives
def import_archive(self, task_id: int):
    """
    Import an archive file.

    Steps:
    1. Check if archive already processed (by hash)
    2. Find archive sidecar JSON if present
    3. Extract to temp directory
    4. Create ImportTask for each media file inside
    5. Queue import tasks for extracted files (with archive sidecar path)
    6. Create archive tag if files were found
    7. Clean up is deferred until child tasks complete

    Args:
        task_id: ID of the ImportTask record

    Returns:
        dict with status and counts
    """
    from app.utils.image_importer import (
        calculate_hash, compute_archive_tag_name, extract_archive_resilient, ExtractError
    )
    from app.utils.metadata_enrichment import find_archive_sidecar

    task = ImportTask.query.get(task_id)
    if not task:
        return {'error': 'Task not found'}

    task.status = 'processing'
    task.started_at = datetime.utcnow()
    db.session.commit()

    tmpdir = None
    try:
        archive_path = task.source_path
        context = task.context or {}
        artist = context.get('artist', 'unknown')
        dest_dir = context.get('dest_dir', '/images')

        # Deep scan options
        is_deep_scan = context.get('deep_scan', False)

        log.info(f"Processing archive: {archive_path} (deep_scan={is_deep_scan})")

        # Find archive sidecar JSON (next to the archive file, not inside it)
        # This contains post metadata from Gallery-DL for the entire archive
        archive_sidecar_path = find_archive_sidecar(archive_path)
        if archive_sidecar_path:
            log.info(f"Found archive sidecar: {archive_sidecar_path}")
        else:
            log.debug(f"No sidecar found for archive: {archive_path}")

        # Compute archive hash
        archive_hash = calculate_hash(archive_path)
        task.file_hash = archive_hash

        # Check if already processed
        existing_archive = ArchiveRecord.query.filter_by(hash=archive_hash).first()
        if existing_archive and not is_deep_scan:
            return _skip_task(task, 'Archive already imported (use deep scan to reprocess)')

        if existing_archive and is_deep_scan:
            log.info(f"Deep scan: re-processing already-imported archive to update metadata")

        # Get archive size
        archive_size = os.path.getsize(archive_path)

        # Extract archive using resilient extractor (tries unar then 7z for RAR)
        tmpdir = tempfile.mkdtemp(prefix='extract_')
        log.info(f"Extracting to: {tmpdir}")

        try:
            extract_archive_resilient(archive_path, tmpdir)
        except ExtractError as e:
            log.error(f"Extraction failed: {e}")
            if tmpdir and os.path.exists(tmpdir):
                shutil.rmtree(tmpdir, ignore_errors=True)
            return _fail_task(task, f'Extraction failed: {e}')

        # Find media files in extracted content
        media_files = []
        for root, _, files in os.walk(tmpdir):
            for filename in files:
                lower = filename.lower()
                if lower.endswith(('.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp',
                                   '.mp4', '.mov', '.webm', '.avi', '.mkv')):
                    media_files.append(os.path.join(root, filename))

        if not media_files:
            log.info(f"No media files found in archive: {archive_path}")
            if tmpdir and os.path.exists(tmpdir):
                shutil.rmtree(tmpdir, ignore_errors=True)
            return _skip_task(task, 'No media files in archive')

        log.info(f"Found {len(media_files)} media files in archive")

        # Get or create archive tag
        # For deep scan of existing archives, reuse the existing tag
        if existing_archive and existing_archive.tag_id:
            archive_tag = Tag.query.get(existing_archive.tag_id)
            if not archive_tag:
                # Tag was deleted, recreate it
                archive_filename = os.path.basename(archive_path)
                tag_name = compute_archive_tag_name(artist, archive_filename)
                archive_tag = Tag(name=tag_name, kind='archive')
                db.session.add(archive_tag)
                db.session.flush()
        else:
            # New archive - create tag using artist:archive_name format
            archive_filename = os.path.basename(archive_path)
            tag_name = compute_archive_tag_name(artist, archive_filename)
            archive_tag = Tag.query.filter_by(name=tag_name).first()
            if not archive_tag:
                archive_tag = Tag(name=tag_name, kind='archive')
                db.session.add(archive_tag)
                db.session.flush()

        # Create ArchiveRecord only if it doesn't exist
        if not existing_archive:
            archive_record = ArchiveRecord(
                filename=archive_path,
                file_size=archive_size,
                hash=archive_hash,
                artist=artist,
                tag_id=archive_tag.id
            )
            db.session.add(archive_record)
            db.session.commit()
        else:
            # Deep scan - commit any tag changes
            db.session.commit()

        # Process each media file immediately (synchronous processing)
        # This ensures all files from the archive are processed together
        # with their archive tag and sidecar metadata applied immediately
        imported_count = 0
        skipped_count = 0
        failed_count = 0

        for media_path in media_files:
            try:
                result = _process_archive_file(
                    src_path=media_path,
                    artist=artist,
                    dest_dir=dest_dir,
                    archive_tag=archive_tag,
                    archive_path=archive_path,
                    archive_sidecar_path=archive_sidecar_path,
                    is_deep_scan=is_deep_scan,
                    reapply_sidecars=context.get('reapply_sidecars', False),
                    regenerate_thumbnails=context.get('regenerate_thumbnails', False)
                )
                if result['status'] == 'imported' or result['status'] == 'superseded':
                    imported_count += 1
                elif result['status'] == 'skipped':
                    skipped_count += 1
                else:
                    failed_count += 1
            except Exception as e:
                log.error(f"Failed to process {media_path}: {e}")
                failed_count += 1

        # Clean up temp directory immediately since we're done processing
        if tmpdir and os.path.exists(tmpdir):
            shutil.rmtree(tmpdir, ignore_errors=True)

        # Update task context with results
        task.context = task.context or {}
        task.context['archive_tag_id'] = archive_tag.id
        task.context['imported_count'] = imported_count
        task.context['skipped_count'] = skipped_count
        task.context['failed_count'] = failed_count
        if archive_sidecar_path:
            task.context['archive_sidecar_path'] = archive_sidecar_path

        task.status = 'complete'
        task.completed_at = datetime.utcnow()
        db.session.commit()

        log.info(f"Archive processed: {imported_count} imported, {skipped_count} skipped, {failed_count} failed from {archive_path}")

        # Update batch stats
        if task.batch_id:
            from app.tasks.scan import update_batch_stats
            update_batch_stats.delay(task.batch_id)

        return {
            'status': 'processed',
            'imported': imported_count,
            'skipped': skipped_count,
            'failed': failed_count,
            'archive_tag': tag_name
        }

    except Exception as e:
        db.session.rollback()
        log.error(f"Archive import failed: {e}", exc_info=True)
        if tmpdir and os.path.exists(tmpdir):
            shutil.rmtree(tmpdir, ignore_errors=True)
        return _fail_task(task, str(e))


def _process_archive_file(src_path: str, artist: str, dest_dir: str,
                          archive_tag: Tag, archive_path: str,
                          archive_sidecar_path: str = None,
                          is_deep_scan: bool = False,
                          reapply_sidecars: bool = False,
                          regenerate_thumbnails: bool = False) -> dict:
    """
    Process a single file from an archive synchronously.

    This is called directly from import_archive for each extracted file,
    ensuring all files from an archive are processed together with their
    archive tag and sidecar metadata applied immediately.

    Returns:
        dict with 'status' key: 'imported', 'skipped', 'superseded', or 'failed'
    """
    from app.utils.image_importer import (
        extract_metadata, calculate_hash, calculate_perceptual_hash,
        is_mostly_transparent, is_mostly_single_color,
        build_hashed_dest_path, generate_thumbnail, generate_video_thumbnail_mirrored,
        transcode_video_to_mp4, needs_transcode
    )
    from app.utils.metadata_enrichment import load_sidecar_metadata, enrich_from_sidecar
    from app.tasks.sidecar import apply_sidecar_metadata

    filename = os.path.basename(src_path)
    lower = filename.lower()

    # Load settings (use cached settings from module)
    settings = _get_cached_settings()

    # Extract metadata first for filtering
    metadata = extract_metadata(src_path)
    if not metadata.get('width') or not metadata.get('height'):
        log.debug(f"Skipping {filename}: Missing dimension data")
        return {'status': 'skipped', 'reason': 'Missing dimension data'}

    # Apply dimension filter
    min_width = settings.get('min_width', 0)
    min_height = settings.get('min_height', 0)
    if min_width > 0 and metadata['width'] < min_width:
        log.debug(f"Skipping {filename}: Too small")
        return {'status': 'skipped', 'reason': 'Too small'}
    if min_height > 0 and metadata['height'] < min_height:
        log.debug(f"Skipping {filename}: Too small")
        return {'status': 'skipped', 'reason': 'Too small'}

    # Apply transparency filter
    if settings.get('skip_transparent') and lower.endswith(('.png', '.gif', '.webp')):
        threshold = settings.get('transparency_threshold', 0.9)
        if is_mostly_transparent(src_path, threshold):
            log.debug(f"Skipping {filename}: Mostly transparent")
            return {'status': 'skipped', 'reason': 'Mostly transparent'}

    # Apply single-color filter
    if settings.get('skip_single_color', True) and lower.endswith(('.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp')):
        if is_mostly_single_color(src_path,
                                  settings.get('single_color_threshold', 0.95),
                                  settings.get('single_color_tolerance', 30)):
            log.debug(f"Skipping {filename}: Mostly single color")
            return {'status': 'skipped', 'reason': 'Mostly single color'}

    # Compute content hash
    file_hash = calculate_hash(src_path)

    # Find best sidecar for this file
    sidecar_path = _find_best_sidecar(src_path, archive_path, archive_sidecar_path)

    # Check for exact duplicate by hash
    existing = ImageRecord.query.filter_by(hash=file_hash).first()
    if existing:
        # Add archive tag to existing image
        if archive_tag and archive_tag not in existing.tags:
            existing.tags.append(archive_tag)
            db.session.commit()
            log.debug(f"Added archive tag to existing image {existing.id}")

        # Apply sidecar metadata
        if sidecar_path and (is_deep_scan and reapply_sidecars or not is_deep_scan):
            apply_sidecar_metadata.delay(existing.id, sidecar_path)

        # Regenerate thumbnail if requested
        if is_deep_scan and regenerate_thumbnails:
            _regenerate_thumbnail(existing)

        return {'status': 'skipped', 'reason': 'Duplicate', 'image_id': existing.id}

    # Compute perceptual hash
    phash = calculate_perceptual_hash(src_path)

    # Copy to destination
    target_dir = os.path.join(dest_dir, artist)
    os.makedirs(target_dir, exist_ok=True)

    # Transcode video if needed
    transcoded_temp = None
    actual_src = src_path
    actual_filename = filename

    if needs_transcode(src_path):
        log.info(f"Transcoding video: {filename}")
        transcoded_temp = transcode_video_to_mp4(src_path)
        if transcoded_temp:
            actual_src = transcoded_temp
            actual_filename = os.path.splitext(filename)[0] + '.mp4'
            file_hash = calculate_hash(transcoded_temp)
            log.info(f"Transcoded successfully: {actual_filename}")
        else:
            log.warning(f"Transcode failed for {filename}, importing original")

    dest_path = build_hashed_dest_path(target_dir, actual_filename, file_hash)
    shutil.copy2(actual_src, dest_path)

    # Clean up transcoded temp file
    if transcoded_temp and os.path.exists(transcoded_temp):
        os.remove(transcoded_temp)

    # Generate thumbnail
    try:
        if dest_path.lower().endswith(('.mp4', '.mov', '.webm', '.avi', '.mkv')):
            thumb_path = generate_video_thumbnail_mirrored(dest_path)
        else:
            thumb_path = generate_thumbnail(dest_path)
    except Exception as e:
        log.warning(f"Thumbnail generation failed for {actual_filename}: {e}")
        thumb_path = None

    # Create ImageRecord
    record = ImageRecord(
        filename=actual_filename,
        filepath=dest_path,
        thumb_path=thumb_path,
        hash=file_hash,
        perceptual_hash=str(phash) if phash else None,
        file_size=os.path.getsize(dest_path),
        width=metadata['width'],
        height=metadata['height'],
        format=metadata['format'] if not needs_transcode(src_path) else 'mp4',
        camera_model=metadata.get('camera_model'),
        taken_at=metadata.get('taken_at'),
        imported_at=datetime.utcnow()
    )

    # Add artist tag
    _add_artist_tag(record, artist)

    # Add archive tag
    if archive_tag:
        record.tags.append(archive_tag)

    db.session.add(record)
    db.session.commit()

    # Apply sidecar metadata asynchronously
    if sidecar_path:
        apply_sidecar_metadata.delay(record.id, sidecar_path)

    # Queue integrity verification for the extracted file too.
    try:
        from app.tasks.maintenance import verify_media_integrity
        verify_media_integrity.delay(record.id)
    except Exception as v_err:
        log.warning(f"Could not enqueue verify_media_integrity for image {record.id}: {v_err}")

    log.debug(f"Imported from archive: {actual_filename} -> {dest_path}")

    return {
        'status': 'imported',
        'image_id': record.id,
        'dest_path': dest_path
    }


def _fail_task(task: ImportTask, error: str, **kwargs) -> dict:
    """Mark task as failed with error message."""
    task.status = 'failed'
    task.error_message = error
    task.completed_at = datetime.utcnow()
    for key, value in kwargs.items():
        setattr(task, key, value)
    db.session.commit()

    # Update batch stats
    if task.batch_id:
        from app.tasks.scan import update_batch_stats
        update_batch_stats.delay(task.batch_id)

    log.error(f"Task {task.id} failed: {error}")
    return {'status': 'failed', 'error': error}


def _skip_task(task: ImportTask, reason: str, **kwargs) -> dict:
    """Mark task as skipped with reason."""
    task.status = 'skipped'
    task.error_message = reason
    task.completed_at = datetime.utcnow()
    for key, value in kwargs.items():
        setattr(task, key, value)
    db.session.commit()

    # Update batch stats
    if task.batch_id:
        from app.tasks.scan import update_batch_stats
        update_batch_stats.delay(task.batch_id)

    log.info(f"Task {task.id} skipped: {reason}")
    return {'status': 'skipped', 'reason': reason}


def _add_artist_tag(record: ImageRecord, artist: str):
    """Add artist tag to image record."""
    if not artist:
        return

    tag = Tag.query.filter_by(kind='artist', name=artist).first()
    if not tag:
        tag = Tag(kind='artist', name=artist)
        db.session.add(tag)
        db.session.flush()

    if tag not in record.tags:
        record.tags.append(tag)


def _add_archive_tag(record: ImageRecord, archive_tag_id: int):
    """Add archive tag to image record."""
    if not archive_tag_id:
        return

    tag = Tag.query.get(archive_tag_id)
    if tag and tag not in record.tags:
        record.tags.append(tag)


def _regenerate_thumbnail(record: ImageRecord) -> bool:
    """
    Regenerate thumbnail for an existing image record.
    Used by deep scan to refresh thumbnails.

    Returns:
        True if thumbnail was regenerated, False on error
    """
    from app.utils.image_importer import generate_thumbnail, generate_video_thumbnail_mirrored

    try:
        filepath = record.filepath
        if not filepath or not os.path.exists(filepath):
            log.warning(f"Cannot regenerate thumbnail: file not found {filepath}")
            return False

        lower = filepath.lower()
        if lower.endswith(('.mp4', '.mov', '.webm', '.avi', '.mkv')):
            new_thumb = generate_video_thumbnail_mirrored(filepath)
        else:
            new_thumb = generate_thumbnail(filepath, overwrite=True)

        if new_thumb:
            record.thumb_path = new_thumb
            db.session.commit()
            log.info(f"Regenerated thumbnail for image {record.id}")
            return True
        return False

    except Exception as e:
        log.warning(f"Failed to regenerate thumbnail for image {record.id}: {e}")
        return False


def _supersede_existing(task: ImportTask, old_record: ImageRecord, src_path: str,
                        file_hash: str, phash, metadata: dict, artist: str,
                        dest_dir: str, filename: str,
                        archive_path: str = None,
                        archive_sidecar_path: str = None) -> dict:
    """
    Replace an existing smaller image with a larger version.

    Preserves tags and metadata from the old record.
    """
    from app.utils.image_importer import (
        build_hashed_dest_path, generate_thumbnail, generate_video_thumbnail_mirrored,
        supersede_image
    )
    from app.utils.metadata_enrichment import enrich_from_sidecar
    from app.tasks.sidecar import apply_sidecar_metadata

    log.info(f"Superseding {old_record.filename} ({old_record.width}x{old_record.height}) "
             f"with {filename} ({metadata['width']}x{metadata['height']})")

    # Enrich metadata from sidecar if available
    # Uses archive sidecar as fallback for files from archives
    sidecar_path = _find_best_sidecar(src_path, archive_path, archive_sidecar_path)
    if sidecar_path:
        enriched = enrich_from_sidecar(sidecar_path, {'taken_at': metadata.get('taken_at')})
        if enriched.get('taken_at'):
            metadata['taken_at'] = enriched['taken_at']

    # Copy new file to destination
    target_dir = os.path.join(dest_dir, artist)
    os.makedirs(target_dir, exist_ok=True)
    dest_path = build_hashed_dest_path(target_dir, filename, file_hash)
    shutil.copy2(src_path, dest_path)

    # Generate new thumbnail
    try:
        if dest_path.lower().endswith(('.mp4', '.mov', '.webm', '.avi', '.mkv')):
            thumb_path = generate_video_thumbnail_mirrored(dest_path)
        else:
            thumb_path = generate_thumbnail(dest_path)
    except Exception as e:
        log.warning(f"Thumbnail generation failed: {e}")
        thumb_path = None

    # Supersede the old record
    record = supersede_image(
        old_record, dest_path, thumb_path, file_hash, phash, metadata, filename
    )

    # Add archive tag if this file came from an archive
    context = task.context or {}
    archive_tag_id = context.get('archive_tag_id')
    if archive_tag_id:
        _add_archive_tag(record, archive_tag_id)

    db.session.commit()

    # Complete task
    task.status = 'complete'
    task.result_image_id = record.id
    task.completed_at = datetime.utcnow()
    db.session.commit()

    # Apply sidecar metadata if available
    if sidecar_path:
        apply_sidecar_metadata.delay(record.id, sidecar_path)

    # Update batch stats
    if task.batch_id:
        from app.tasks.scan import update_batch_stats
        update_batch_stats.delay(task.batch_id)

    # Re-verify the superseded file. supersede_image already cleared
    # integrity_status to 'unknown'; this resolves it back to a real value.
    try:
        from app.tasks.maintenance import verify_media_integrity
        verify_media_integrity.delay(record.id)
    except Exception as v_err:
        log.warning(f"Could not enqueue verify_media_integrity (supersede) for image {record.id}: {v_err}")

    return {
        'status': 'superseded',
        'image_id': record.id,
        'dest_path': dest_path
    }