changes to mobile styling in modal view, complete reword of backend worker system

app/__init__.py

@@ -13,6 +13,9 @@ import sqlite3
 db = SQLAlchemy()
 migrate = Migrate()
 
+# Celery instance - will be configured with app context in create_app
+celery = None
+
 
 # =============================================================================
 # CLI Commands
@@ -25,6 +28,34 @@ def version_check_command():
     from app.utils.version_migration import check_and_run_migrations
     check_and_run_migrations()
 
+
+@click.command("celery-worker")
+@click.option('--queues', '-Q', default='scan,import,thumbnail,sidecar,default',
+              help='Comma-separated list of queues to consume')
+@click.option('--concurrency', '-c', default=2, type=int,
+              help='Number of worker processes')
+@with_appcontext
+def celery_worker_command(queues, concurrency):
+    """Start a Celery worker for processing import tasks."""
+    from app.celery_app import celery as celery_app
+    celery_app.worker_main([
+        'worker',
+        f'--queues={queues}',
+        f'--concurrency={concurrency}',
+        '--loglevel=info'
+    ])
+
+
+@click.command("celery-beat")
+@with_appcontext
+def celery_beat_command():
+    """Start Celery Beat scheduler for periodic tasks."""
+    from app.celery_app import celery as celery_app
+    celery_app.worker_main([
+        'beat',
+        '--loglevel=info'
+    ])
+
 @event.listens_for(Engine, "connect")
 def set_sqlite_pragma(dbapi_connection, connection_record):
     # Only apply to SQLite connections
@@ -54,5 +85,13 @@ def create_app(config_class='config.Config'):
 
     # Register CLI commands
     app.cli.add_command(version_check_command)
+    app.cli.add_command(celery_worker_command)
+    app.cli.add_command(celery_beat_command)
+
+    # Initialize Celery with Flask app context
+    from app.celery_app import make_celery
+    global celery
+    celery = make_celery(app)
+    app.celery = celery
 
     return app

app/celery_app.py

@@ -0,0 +1,123 @@
+# app/celery_app.py
+"""
+Celery application factory and configuration.
+Provides task queue functionality for the ImageRepo import pipeline.
+"""
+import os
+from celery import Celery
+
+# Default broker/backend URLs
+DEFAULT_BROKER = 'redis://redis:6379/0'
+DEFAULT_BACKEND = 'redis://redis:6379/0'
+
+
+def make_celery(app=None):
+    """
+    Create Celery instance with Flask app context support.
+
+    Args:
+        app: Flask application instance (optional)
+
+    Returns:
+        Configured Celery instance
+    """
+    # Get broker/backend from app config or environment
+    if app:
+        broker = app.config.get('CELERY_BROKER_URL', DEFAULT_BROKER)
+        backend = app.config.get('CELERY_RESULT_BACKEND', DEFAULT_BACKEND)
+    else:
+        broker = os.environ.get('CELERY_BROKER_URL', DEFAULT_BROKER)
+        backend = os.environ.get('CELERY_RESULT_BACKEND', DEFAULT_BACKEND)
+
+    celery = Celery(
+        'imagerepo',
+        broker=broker,
+        backend=backend,
+        include=[
+            'app.tasks.scan',
+            'app.tasks.import_file',
+            'app.tasks.thumbnail',
+            'app.tasks.sidecar'
+        ]
+    )
+
+    # Worker concurrency from environment
+    concurrency = int(os.environ.get('CELERY_WORKER_CONCURRENCY', '2'))
+
+    celery.conf.update(
+        # Serialization
+        task_serializer='json',
+        accept_content=['json'],
+        result_serializer='json',
+        timezone='UTC',
+        enable_utc=True,
+
+        # Concurrency - intentionally low for steady background processing
+        worker_concurrency=concurrency,
+        worker_prefetch_multiplier=1,  # One task at a time per worker process
+
+        # Task routing - separate queues for different task types
+        task_routes={
+            'app.tasks.scan.*': {'queue': 'scan'},
+            'app.tasks.import_file.*': {'queue': 'import'},
+            'app.tasks.thumbnail.*': {'queue': 'thumbnail'},
+            'app.tasks.sidecar.*': {'queue': 'sidecar'},
+        },
+
+        # Task default queue for unrouted tasks
+        task_default_queue='default',
+
+        # Result backend settings
+        result_expires=86400,  # 24 hours
+
+        # Task execution settings for resume capability
+        task_acks_late=True,  # Acknowledge after task completes
+        task_reject_on_worker_lost=True,  # Requeue if worker dies
+
+        # Time limits
+        task_soft_time_limit=300,  # 5 minutes soft limit
+        task_time_limit=600,  # 10 minutes hard limit (archives may need more)
+
+        # Periodic task schedule (Celery Beat)
+        beat_schedule={
+            'periodic-import-scan': {
+                'task': 'app.tasks.scan.scan_directory',
+                'schedule': int(os.environ.get('IMPORT_EVERY_SECONDS', '28800')),  # 8 hours default
+                'args': ('/import', '/images'),
+            },
+            'recover-interrupted-tasks': {
+                'task': 'app.tasks.scan.recover_interrupted_tasks',
+                'schedule': 300,  # Every 5 minutes
+            },
+        },
+    )
+
+    if app:
+        # Don't pass Flask config directly to Celery - it contains old-style keys
+        # that conflict with Celery's new lowercase format.
+        # The broker/backend are already set above from app.config.
+
+        # Create a task base class that runs within Flask app context
+        class ContextTask(celery.Task):
+            def __call__(self, *args, **kwargs):
+                with app.app_context():
+                    return self.run(*args, **kwargs)
+
+        celery.Task = ContextTask
+
+    return celery
+
+
+def create_celery_with_app():
+    """
+    Create Celery instance with Flask app context for standalone workers.
+    This is used when running `celery -A app.celery_app:celery worker`.
+    """
+    from app import create_app
+    flask_app = create_app()
+    return make_celery(flask_app)
+
+
+# Create celery instance with Flask app context
+# This ensures workers have access to the database and Flask extensions
+celery = create_celery_with_app()

app/main.py

@@ -7,7 +7,7 @@ from sqlalchemy.orm import joinedload
 from collections import OrderedDict
 from datetime import datetime
 
-from app.models import ImageRecord, Tag, ArchiveRecord, image_tags
+from app.models import ImageRecord, Tag, ArchiveRecord, image_tags, ImportTask, ImportBatch
 from app import db
 import os
 import shutil
@@ -987,3 +987,299 @@ def delete_filtered_images():
         deleted_count=deleted_count,
         errors=errors if errors else None
     )
+
+
+# ----------------------------
+# Import Task Queue API (Celery)
+# ----------------------------
+
+@main.route('/api/import/trigger', methods=['POST'])
+def trigger_import_celery():
+    """
+    Trigger a directory scan and import via Celery.
+    Replaces the old flag-file mechanism.
+    """
+    try:
+        from app.tasks.scan import scan_directory
+
+        source_dir = request.form.get('source_dir', '/import')
+        dest_dir = request.form.get('dest_dir', '/images')
+
+        result = scan_directory.delay(source_dir, dest_dir)
+
+        return jsonify({
+            'ok': True,
+            'task_id': result.id,
+            'message': 'Import scan started'
+        })
+    except Exception as e:
+        return jsonify({'ok': False, 'error': str(e)}), 500
+
+
+@main.route('/api/import/status')
+def import_queue_status():
+    """
+    Get current import queue status.
+    Returns counts by task status and recent task info.
+    """
+    try:
+        # Count tasks by status
+        status_counts = dict(
+            db.session.query(
+                ImportTask.status,
+                func.count(ImportTask.id)
+            ).group_by(ImportTask.status).all()
+        )
+
+        # Get active batch info
+        active_batch = ImportBatch.query.filter_by(status='running').order_by(
+            ImportBatch.started_at.desc()
+        ).first()
+
+        # Recent tasks (last 20)
+        recent_tasks = ImportTask.query.order_by(
+            ImportTask.created_at.desc()
+        ).limit(20).all()
+
+        return jsonify({
+            'ok': True,
+            'status_counts': {
+                'pending': status_counts.get('pending', 0),
+                'queued': status_counts.get('queued', 0),
+                'processing': status_counts.get('processing', 0),
+                'complete': status_counts.get('complete', 0),
+                'failed': status_counts.get('failed', 0),
+                'skipped': status_counts.get('skipped', 0),
+            },
+            'active_batch': {
+                'id': active_batch.id,
+                'source_directory': active_batch.source_directory,
+                'total_files': active_batch.total_files,
+                'processed_files': active_batch.processed_files,
+                'imported_count': active_batch.imported_count,
+                'skipped_count': active_batch.skipped_count,
+                'failed_count': active_batch.failed_count,
+                'started_at': active_batch.started_at.isoformat() if active_batch.started_at else None
+            } if active_batch else None,
+            'recent_tasks': [
+                {
+                    'id': t.id,
+                    'source_path': os.path.basename(t.source_path) if t.source_path else None,
+                    'task_type': t.task_type,
+                    'status': t.status,
+                    'error_message': t.error_message[:100] if t.error_message else None,
+                    'created_at': t.created_at.isoformat() if t.created_at else None
+                }
+                for t in recent_tasks
+            ]
+        })
+    except Exception as e:
+        return jsonify({'ok': False, 'error': str(e)}), 500
+
+
+@main.route('/api/import/task/<int:task_id>')
+def get_import_task(task_id):
+    """
+    Get detailed status for a specific import task.
+    """
+    task = ImportTask.query.get_or_404(task_id)
+
+    # Check Celery task status if still processing
+    celery_status = None
+    if task.celery_task_id:
+        try:
+            from app.celery_app import celery
+            result = celery.AsyncResult(task.celery_task_id)
+            celery_status = result.status
+        except Exception:
+            pass
+
+    return jsonify({
+        'ok': True,
+        'task': {
+            'id': task.id,
+            'source_path': task.source_path,
+            'file_hash': task.file_hash,
+            'file_size': task.file_size,
+            'task_type': task.task_type,
+            'status': task.status,
+            'celery_status': celery_status,
+            'celery_task_id': task.celery_task_id,
+            'error_message': task.error_message,
+            'retry_count': task.retry_count,
+            'context': task.context,
+            'batch_id': task.batch_id,
+            'result_image_id': task.result_image_id,
+            'created_at': task.created_at.isoformat() if task.created_at else None,
+            'started_at': task.started_at.isoformat() if task.started_at else None,
+            'completed_at': task.completed_at.isoformat() if task.completed_at else None
+        }
+    })
+
+
+@main.route('/api/import/retry-failed', methods=['POST'])
+def retry_failed_import_tasks():
+    """
+    Re-queue all failed import tasks.
+    """
+    try:
+        from app.tasks.import_file import import_media_file, import_archive
+
+        failed_tasks = ImportTask.query.filter_by(status='failed').all()
+        retried = 0
+
+        for task in failed_tasks:
+            task.status = 'pending'
+            task.error_message = None
+            task.retry_count = 0
+            task.started_at = None
+            task.completed_at = None
+
+            if task.task_type == 'import_image':
+                result = import_media_file.delay(task.id)
+            elif task.task_type == 'import_archive':
+                result = import_archive.delay(task.id)
+            else:
+                continue
+
+            task.celery_task_id = result.id
+            task.status = 'queued'
+            retried += 1
+
+        db.session.commit()
+
+        return jsonify({
+            'ok': True,
+            'retried': retried
+        })
+    except Exception as e:
+        db.session.rollback()
+        return jsonify({'ok': False, 'error': str(e)}), 500
+
+
+@main.route('/api/import/clear-completed', methods=['POST'])
+def clear_completed_tasks():
+    """
+    Clear completed and skipped import tasks from the database.
+    Keeps failed tasks for review.
+    """
+    try:
+        deleted = ImportTask.query.filter(
+            ImportTask.status.in_(['complete', 'skipped'])
+        ).delete(synchronize_session=False)
+
+        db.session.commit()
+
+        return jsonify({
+            'ok': True,
+            'deleted': deleted
+        })
+    except Exception as e:
+        db.session.rollback()
+        return jsonify({'ok': False, 'error': str(e)}), 500
+
+
+@main.route('/api/thumbnails/regenerate', methods=['POST'])
+def regenerate_thumbnails_celery():
+    """
+    Trigger thumbnail regeneration for all images via Celery.
+    """
+    try:
+        from app.tasks.thumbnail import regenerate_all_thumbnails
+
+        overwrite = request.form.get('overwrite', 'true').lower() == 'true'
+        result = regenerate_all_thumbnails.delay(overwrite=overwrite)
+
+        return jsonify({
+            'ok': True,
+            'task_id': result.id,
+            'message': 'Thumbnail regeneration started'
+        })
+    except Exception as e:
+        return jsonify({'ok': False, 'error': str(e)}), 500
+
+
+@main.route('/api/thumbnails/regenerate-missing', methods=['POST'])
+def regenerate_missing_thumbnails_celery():
+    """
+    Generate thumbnails only for images that don't have them.
+    """
+    try:
+        from app.tasks.thumbnail import regenerate_missing_thumbnails
+
+        result = regenerate_missing_thumbnails.delay()
+
+        return jsonify({
+            'ok': True,
+            'task_id': result.id,
+            'message': 'Missing thumbnail generation started'
+        })
+    except Exception as e:
+        return jsonify({'ok': False, 'error': str(e)}), 500
+
+
+@main.route('/api/celery/status')
+def celery_cluster_status():
+    """
+    Get Celery cluster status (workers, queues, etc.).
+    """
+    try:
+        from app.celery_app import celery
+
+        inspect = celery.control.inspect()
+
+        # Get worker stats
+        active = inspect.active() or {}
+        scheduled = inspect.scheduled() or {}
+        reserved = inspect.reserved() or {}
+        stats = inspect.stats() or {}
+
+        # Count active tasks
+        active_count = sum(len(tasks) for tasks in active.values())
+
+        return jsonify({
+            'ok': True,
+            'workers': list(stats.keys()),
+            'worker_count': len(stats),
+            'active_tasks': active_count,
+            'active': active,
+            'scheduled': scheduled,
+            'reserved': reserved
+        })
+    except Exception as e:
+        return jsonify({
+            'ok': False,
+            'error': str(e),
+            'message': 'Celery may not be running'
+        }), 503
+
+
+@main.route('/api/import/batches')
+def list_import_batches():
+    """
+    List recent import batches with statistics.
+    """
+    limit = request.args.get('limit', 10, type=int)
+
+    batches = ImportBatch.query.order_by(
+        ImportBatch.started_at.desc()
+    ).limit(limit).all()
+
+    return jsonify({
+        'ok': True,
+        'batches': [
+            {
+                'id': b.id,
+                'source_directory': b.source_directory,
+                'status': b.status,
+                'total_files': b.total_files,
+                'processed_files': b.processed_files,
+                'imported_count': b.imported_count,
+                'skipped_count': b.skipped_count,
+                'failed_count': b.failed_count,
+                'started_at': b.started_at.isoformat() if b.started_at else None,
+                'completed_at': b.completed_at.isoformat() if b.completed_at else None
+            }
+            for b in batches
+        ]
+    })

app/models.py

@@ -76,3 +76,80 @@ class AppSettings(db.Model):
     key = db.Column(db.String(64), unique=True, nullable=False)
     value = db.Column(db.Text, nullable=True)
     updated_at = db.Column(db.DateTime, default=db.func.now(), onupdate=db.func.now())
+
+
+class ImportBatch(db.Model):
+    """
+    Groups related import tasks for batch tracking and reporting.
+    One batch per scan operation.
+    """
+    __tablename__ = "import_batch"
+
+    id = db.Column(db.Integer, primary_key=True)
+    source_directory = db.Column(db.String(1024), nullable=False)
+    status = db.Column(db.String(32), default='running', index=True)  # running, complete, failed, cancelled
+
+    # Statistics
+    total_files = db.Column(db.Integer, default=0)
+    processed_files = db.Column(db.Integer, default=0)
+    imported_count = db.Column(db.Integer, default=0)
+    skipped_count = db.Column(db.Integer, default=0)
+    failed_count = db.Column(db.Integer, default=0)
+
+    # Timestamps
+    started_at = db.Column(db.DateTime, default=db.func.now())
+    completed_at = db.Column(db.DateTime, nullable=True)
+
+
+class ImportTask(db.Model):
+    """
+    Tracks the state of each file being processed through the import pipeline.
+    Enables resume capability and parallel processing.
+    """
+    __tablename__ = "import_task"
+
+    id = db.Column(db.Integer, primary_key=True)
+
+    # File identification
+    source_path = db.Column(db.String(1024), nullable=False, index=True)
+    file_hash = db.Column(db.String(64), nullable=True, index=True)  # SHA256, computed during processing
+    file_size = db.Column(db.BigInteger, nullable=True)
+
+    # Classification
+    # Values: 'scan', 'import_image', 'import_archive', 'thumbnail', 'sidecar'
+    task_type = db.Column(db.String(32), nullable=False, index=True)
+
+    # State tracking
+    # Values: 'pending', 'queued', 'processing', 'complete', 'failed', 'skipped'
+    status = db.Column(db.String(32), nullable=False, default='pending', index=True)
+
+    # Celery task tracking
+    celery_task_id = db.Column(db.String(64), nullable=True, index=True)
+
+    # Context/metadata (JSON)
+    # For images: {"artist": "...", "dest_dir": "..."}
+    # For archives: {"artist": "...", "dest_dir": "...", "archive_tag": "..."}
+    # For sidecars: {"image_id": 123, "sidecar_path": "..."}
+    context = db.Column(db.JSON, nullable=True)
+
+    # Batch relationship
+    batch_id = db.Column(db.Integer, db.ForeignKey('import_batch.id', ondelete='SET NULL'), nullable=True)
+    batch = db.relationship('ImportBatch', backref=db.backref('tasks', lazy='dynamic'))
+
+    # Result tracking
+    result_image_id = db.Column(db.Integer, db.ForeignKey('image_record.id', ondelete='SET NULL'), nullable=True)
+    result_image = db.relationship('ImageRecord', foreign_keys=[result_image_id])
+    error_message = db.Column(db.Text, nullable=True)
+    retry_count = db.Column(db.Integer, default=0)
+
+    # Timestamps
+    created_at = db.Column(db.DateTime, default=db.func.now(), index=True)
+    started_at = db.Column(db.DateTime, nullable=True)
+    completed_at = db.Column(db.DateTime, nullable=True)
+
+    # Unique constraint: prevent duplicate tasks for same file+type combination
+    # Note: file_hash can be NULL initially (computed during processing)
+    __table_args__ = (
+        db.Index('ix_import_task_status_type', 'status', 'task_type'),
+        db.Index('ix_import_task_batch_status', 'batch_id', 'status'),
+    )

app/static/js/modal-pagination.js

@@ -53,7 +53,9 @@ document.addEventListener('DOMContentLoaded', () => {
       archive: '🗜️',
       character: '👤',
       series: '📺',
-      rating: '⚠️'
+      rating: '⚠️',
+      source: '🌐',
+      post: '📌'
     };
     return icons[kind] || '#';
   }
@@ -303,12 +305,17 @@ document.addEventListener('DOMContentLoaded', () => {
     modal.classList.add('active');
     updateImage(index);
     history.replaceState({ modalIndex: index }, '', window.location.href);
-    // Focus the tag input after modal opens
+    // Focus the tag input after modal opens (desktop only - avoids keyboard popup on mobile)
-    if (tagInput) {
+    if (tagInput && !isTouchDevice()) {
       setTimeout(() => tagInput.focus(), 100);
     }
   }
 
+  // Detect touch devices to avoid auto-focus keyboard popup
+  function isTouchDevice() {
+    return ('ontouchstart' in window) || (navigator.maxTouchPoints > 0);
+  }
+
   function closeModal() {
     modal.classList.remove('active');
     modalImageWrapper.innerHTML = '';

app/static/js/showcase.js

@@ -286,7 +286,9 @@
       archive: '🗜️',
       character: '👤',
       series: '📺',
-      rating: '⚠️'
+      rating: '⚠️',
+      source: '🌐',
+      post: '📌'
     };
 
     const icon = icons[tag.kind];

app/static/style.css

@@ -369,6 +369,28 @@ header {
   .modal-body {
     flex-direction: column;
     align-items: center;
+    height: 100%;
+    overflow-y: auto;
+    -webkit-overflow-scrolling: touch;
+  }
+}
+
+/* Mobile landscape: prioritize image, minimize tag editor */
+@media (max-width: 900px) and (orientation: landscape) {
+  .modal-body {
+    flex-direction: row;
+    align-items: stretch;
+  }
+  .modal-image-wrapper {
+    max-width: calc(100% - 200px);
+    max-height: calc(100vh - 2rem);
+    height: calc(100vh - 2rem);
+  }
+  .tag-editor {
+    width: 180px;
+    min-width: 180px;
+    max-height: calc(100vh - 2rem);
+    overflow-y: auto;
   }
 }
 
@@ -415,12 +437,17 @@ header {
   width: 90vw; height: auto; max-height: none;
 }
 
-/* On smaller screens, full width for image wrapper */
+/* On smaller screens (portrait), full width for image wrapper */
-@media (max-width: 900px) {
+@media (max-width: 900px) and (orientation: portrait) {
   .modal-image-wrapper {
     max-width: 100%;
     height: auto;
-    max-height: 60vh;
+    max-height: 55vh;
+    flex-shrink: 0;
+  }
+  .modal-image-wrapper img,
+  .modal-image-wrapper video {
+    max-height: 55vh;
   }
 }
 
@@ -453,6 +480,17 @@ header {
 .modal-prev { left: 1rem; }
 .modal-next { right: 1rem; }
 
+/* Mobile: always show nav buttons, larger touch targets */
+@media (max-width: 900px) {
+  .modal-button {
+    opacity: 0.8;
+    width: 44px;
+    height: 44px;
+  }
+  .modal-prev { left: 0.5rem; }
+  .modal-next { right: 0.5rem; }
+}
+
 /* Close button */
 .modal-close-btn {
   position: absolute;
@@ -535,13 +573,15 @@ header {
   flex-shrink: 0;
 }
 
-/* On smaller screens, make tag editor full width */
+/* On smaller screens (portrait), make tag editor full width and scrollable */
-@media (max-width: 900px) {
+@media (max-width: 900px) and (orientation: portrait) {
   .tag-editor {
     width: 100%;
     max-width: 500px;
     min-width: unset;
-    max-height: unset;
+    max-height: 35vh;
+    overflow-y: auto;
+    flex-shrink: 0;
   }
 }
 
@@ -594,7 +634,7 @@ header {
   border-radius: 6px;
   background: rgba(255, 255, 255, 0.08);
   color: var(--text);
-  font-size: 0.9rem;
+  font-size: 16px; /* Must be 16px+ to prevent iOS auto-zoom on focus */
 }
 .tag-form input::placeholder {
   color: var(--text-muted);
@@ -714,8 +754,9 @@ header {
 
 .tag-thumb {
   flex: 1;
-  background-size: cover;
+  min-width: 0;
-  background-position: center;
+  height: 100%;
+  object-fit: cover;
   background-color: var(--bg-elevated);
   transition: filter 0.2s ease;
 }

app/tasks/__init__.py

@@ -0,0 +1,10 @@
+# app/tasks/__init__.py
+"""
+Celery task modules for the ImageRepo import pipeline.
+
+Task Types:
+- scan: Directory scanning and task queuing
+- import_file: Individual file import (images, videos, archives)
+- thumbnail: Thumbnail generation
+- sidecar: Gallery-DL metadata extraction and application
+"""

app/tasks/import_file.py

@@ -0,0 +1,507 @@
+# app/tasks/import_file.py
+"""
+File import tasks for images, videos, and archives.
+
+These tasks handle the actual import of individual files,
+including filtering, duplicate detection, and storage.
+"""
+import os
+import shutil
+import tempfile
+import logging
+from datetime import datetime
+
+import imagehash
+
+from app.celery_app import celery
+from app import db
+from app.models import ImportTask, ImageRecord, Tag, ArchiveRecord
+
+log = logging.getLogger('celery.tasks.import_file')
+
+
+@celery.task(bind=True, name='app.tasks.import_file.import_media_file',
+             max_retries=3, default_retry_delay=60)
+def import_media_file(self, task_id: int):
+    """
+    Import a single media file (image or video).
+
+    Steps:
+    1. Load ImportTask record
+    2. Apply filters (dimensions, transparency, single-color)
+    3. Check for duplicates (hash and pHash)
+    4. Copy to content-addressed storage
+    5. Create ImageRecord
+    6. Queue thumbnail task
+    7. Queue sidecar metadata task
+    8. Update ImportTask status
+
+    Args:
+        task_id: ID of the ImportTask record
+
+    Returns:
+        dict with status and result info
+    """
+    from app.tasks.thumbnail import generate_thumbnail_task
+    from app.tasks.sidecar import apply_sidecar_metadata
+    from app.utils.image_importer import (
+        extract_metadata, calculate_hash, calculate_perceptual_hash,
+        is_mostly_transparent, is_mostly_single_color, find_similar_image,
+        build_hashed_dest_path, load_import_settings, supersede_image,
+        generate_thumbnail, generate_video_thumbnail_mirrored
+    )
+    from app.utils.metadata_enrichment import find_sidecar_json
+
+    task = ImportTask.query.get(task_id)
+    if not task:
+        log.error(f"Task {task_id} not found")
+        return {'error': 'Task not found'}
+
+    task.status = 'processing'
+    task.started_at = datetime.utcnow()
+    db.session.commit()
+
+    try:
+        src_path = task.source_path
+        context = task.context or {}
+        artist = context.get('artist', 'unknown')
+        dest_dir = context.get('dest_dir', '/images')
+
+        log.info(f"Processing: {src_path}")
+
+        # Load settings
+        settings = load_import_settings()
+
+        # Validate file exists
+        if not os.path.exists(src_path):
+            return _fail_task(task, 'Source file not found')
+
+        filename = os.path.basename(src_path)
+        lower = filename.lower()
+
+        # Extract metadata first for filtering
+        metadata = extract_metadata(src_path)
+        if not metadata.get('width') or not metadata.get('height'):
+            return _skip_task(task, 'Missing dimension data')
+
+        # Apply dimension filter
+        min_width = settings.get('min_width', 0)
+        min_height = settings.get('min_height', 0)
+        if min_width > 0 and metadata['width'] < min_width:
+            return _skip_task(task, f"Too small: {metadata['width']}x{metadata['height']}")
+        if min_height > 0 and metadata['height'] < min_height:
+            return _skip_task(task, f"Too small: {metadata['width']}x{metadata['height']}")
+
+        # Apply transparency filter
+        if settings.get('skip_transparent') and lower.endswith(('.png', '.gif', '.webp')):
+            threshold = settings.get('transparency_threshold', 0.9)
+            if is_mostly_transparent(src_path, threshold):
+                return _skip_task(task, 'Mostly transparent')
+
+        # Apply single-color filter
+        if settings.get('skip_single_color', True) and lower.endswith(('.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp')):
+            if is_mostly_single_color(src_path,
+                                      settings.get('single_color_threshold', 0.95),
+                                      settings.get('single_color_tolerance', 30)):
+                return _skip_task(task, 'Mostly single color')
+
+        # Compute content hash
+        file_hash = calculate_hash(src_path)
+        task.file_hash = file_hash
+
+        # Check for exact duplicate by hash
+        existing = ImageRecord.query.filter_by(hash=file_hash).first()
+        if existing:
+            # Still try to enrich with sidecar metadata
+            sidecar_path = find_sidecar_json(src_path)
+            if sidecar_path:
+                apply_sidecar_metadata.delay(existing.id, sidecar_path)
+            return _skip_task(task, 'Duplicate by hash', result_image_id=existing.id)
+
+        # Compute pHash for similarity check
+        phash = calculate_perceptual_hash(src_path)
+        phash_threshold = settings.get('phash_threshold', 10)
+
+        if phash and phash_threshold > 0:
+            # Load existing pHashes for comparison
+            existing_phashes = [
+                (imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height, img.id)
+                for img in ImageRecord.query.filter(ImageRecord.perceptual_hash.isnot(None)).all()
+            ]
+
+            relationship, similar_id = find_similar_image(
+                phash, metadata['width'], metadata['height'],
+                existing_phashes, threshold=phash_threshold
+            )
+
+            if relationship == 'larger_exists':
+                return _skip_task(task, 'Similar larger image exists')
+
+            # Handle supersede case - new image is larger than existing
+            if relationship == 'smaller_exists' and settings.get('supersede_smaller', True) and similar_id:
+                old_record = ImageRecord.query.get(similar_id)
+                if old_record:
+                    return _supersede_existing(
+                        task, old_record, src_path, file_hash, phash,
+                        metadata, artist, dest_dir, filename
+                    )
+
+        # Normal import - copy to destination
+        target_dir = os.path.join(dest_dir, artist)
+        os.makedirs(target_dir, exist_ok=True)
+        dest_path = build_hashed_dest_path(target_dir, filename, file_hash)
+        shutil.copy2(src_path, dest_path)
+
+        # Generate thumbnail
+        try:
+            if dest_path.lower().endswith(('.mp4', '.mov', '.webm', '.avi', '.mkv')):
+                thumb_path = generate_video_thumbnail_mirrored(dest_path)
+            else:
+                thumb_path = generate_thumbnail(dest_path)
+        except Exception as e:
+            log.warning(f"Thumbnail generation failed for {filename}: {e}")
+            thumb_path = None
+
+        # Create ImageRecord
+        record = ImageRecord(
+            filename=filename,
+            filepath=dest_path,
+            thumb_path=thumb_path,
+            hash=file_hash,
+            perceptual_hash=str(phash) if phash else None,
+            file_size=metadata['file_size'],
+            width=metadata['width'],
+            height=metadata['height'],
+            format=metadata['format'],
+            camera_model=metadata.get('camera_model'),
+            taken_at=metadata.get('taken_at'),
+            imported_at=datetime.utcnow()
+        )
+
+        # Add artist tag
+        _add_artist_tag(record, artist)
+
+        db.session.add(record)
+        db.session.commit()
+
+        # Complete task
+        task.status = 'complete'
+        task.result_image_id = record.id
+        task.completed_at = datetime.utcnow()
+        db.session.commit()
+
+        # Queue sidecar metadata enrichment
+        sidecar_path = find_sidecar_json(src_path)
+        if sidecar_path:
+            apply_sidecar_metadata.delay(record.id, sidecar_path)
+
+        # Update batch stats
+        if task.batch_id:
+            from app.tasks.scan import update_batch_stats
+            update_batch_stats.delay(task.batch_id)
+
+        log.info(f"Imported: {filename} -> {dest_path}")
+
+        return {
+            'status': 'imported',
+            'image_id': record.id,
+            'dest_path': dest_path
+        }
+
+    except Exception as e:
+        db.session.rollback()
+        log.error(f"Import failed for task {task_id}: {e}", exc_info=True)
+
+        task.retry_count += 1
+        if task.retry_count < 3:
+            task.status = 'queued'
+            db.session.commit()
+            raise self.retry(exc=e)
+        else:
+            return _fail_task(task, str(e))
+
+
+@celery.task(bind=True, name='app.tasks.import_file.import_archive',
+             max_retries=2, default_retry_delay=120,
+             soft_time_limit=1800, time_limit=3600)  # 30min soft, 60min hard for large archives
+def import_archive(self, task_id: int):
+    """
+    Import an archive file.
+
+    Steps:
+    1. Check if archive already processed (by hash)
+    2. Extract to temp directory
+    3. Create ImportTask for each media file inside
+    4. Queue import tasks for extracted files
+    5. Create archive tag if files were found
+    6. Clean up is deferred until child tasks complete
+
+    Args:
+        task_id: ID of the ImportTask record
+
+    Returns:
+        dict with status and counts
+    """
+    from app.utils.image_importer import calculate_hash, compute_next_archive_tag_name
+    from pyunpack import Archive
+
+    task = ImportTask.query.get(task_id)
+    if not task:
+        return {'error': 'Task not found'}
+
+    task.status = 'processing'
+    task.started_at = datetime.utcnow()
+    db.session.commit()
+
+    tmpdir = None
+    try:
+        archive_path = task.source_path
+        context = task.context or {}
+        artist = context.get('artist', 'unknown')
+        dest_dir = context.get('dest_dir', '/images')
+
+        log.info(f"Processing archive: {archive_path}")
+
+        # Compute archive hash
+        archive_hash = calculate_hash(archive_path)
+        task.file_hash = archive_hash
+
+        # Check if already processed
+        existing = ArchiveRecord.query.filter_by(hash=archive_hash).first()
+        if existing:
+            return _skip_task(task, 'Archive already imported')
+
+        # Get archive size
+        archive_size = os.path.getsize(archive_path)
+
+        # Extract archive
+        tmpdir = tempfile.mkdtemp(prefix='extract_')
+        log.info(f"Extracting to: {tmpdir}")
+
+        try:
+            Archive(archive_path).extractall(tmpdir)
+        except Exception as e:
+            log.error(f"Extraction failed: {e}")
+            if tmpdir and os.path.exists(tmpdir):
+                shutil.rmtree(tmpdir, ignore_errors=True)
+            return _fail_task(task, f'Extraction failed: {e}')
+
+        # Find media files in extracted content
+        media_files = []
+        for root, _, files in os.walk(tmpdir):
+            for filename in files:
+                lower = filename.lower()
+                if lower.endswith(('.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp',
+                                   '.mp4', '.mov', '.webm', '.avi', '.mkv')):
+                    media_files.append(os.path.join(root, filename))
+
+        if not media_files:
+            log.info(f"No media files found in archive: {archive_path}")
+            if tmpdir and os.path.exists(tmpdir):
+                shutil.rmtree(tmpdir, ignore_errors=True)
+            return _skip_task(task, 'No media files in archive')
+
+        log.info(f"Found {len(media_files)} media files in archive")
+
+        # Create archive tag
+        tag_name = compute_next_archive_tag_name(artist)
+        archive_tag = Tag.query.filter_by(name=tag_name).first()
+        if not archive_tag:
+            archive_tag = Tag(name=tag_name, kind='archive')
+            db.session.add(archive_tag)
+            db.session.flush()
+
+        # Create ArchiveRecord
+        archive_record = ArchiveRecord(
+            filename=archive_path,
+            file_size=archive_size,
+            hash=archive_hash,
+            artist=artist,
+            tag_id=archive_tag.id
+        )
+        db.session.add(archive_record)
+        db.session.commit()
+
+        # Queue tasks for each media file
+        queued_count = 0
+        for media_path in media_files:
+            child_task = ImportTask(
+                source_path=media_path,
+                task_type='import_image',
+                batch_id=task.batch_id,
+                context={
+                    'artist': artist,
+                    'dest_dir': dest_dir,
+                    'archive_task_id': task.id,
+                    'archive_tag_id': archive_tag.id,
+                    'temp_dir': tmpdir
+                },
+                status='pending'
+            )
+            db.session.add(child_task)
+            db.session.flush()
+
+            result = import_media_file.delay(child_task.id)
+            child_task.celery_task_id = result.id
+            child_task.status = 'queued'
+            queued_count += 1
+
+        # Store child task info for cleanup tracking
+        task.context = task.context or {}
+        task.context['temp_dir'] = tmpdir
+        task.context['archive_tag_id'] = archive_tag.id
+        task.context['child_count'] = queued_count
+
+        task.status = 'complete'
+        task.completed_at = datetime.utcnow()
+        db.session.commit()
+
+        log.info(f"Archive processed: queued {queued_count} files from {archive_path}")
+
+        # Note: temp directory cleanup happens via a separate periodic task
+        # or when all child tasks complete
+
+        return {
+            'status': 'extracted',
+            'queued_files': queued_count,
+            'archive_tag': tag_name
+        }
+
+    except Exception as e:
+        db.session.rollback()
+        log.error(f"Archive import failed: {e}", exc_info=True)
+        if tmpdir and os.path.exists(tmpdir):
+            shutil.rmtree(tmpdir, ignore_errors=True)
+        return _fail_task(task, str(e))
+
+
+def _fail_task(task: ImportTask, error: str, **kwargs) -> dict:
+    """Mark task as failed with error message."""
+    task.status = 'failed'
+    task.error_message = error
+    task.completed_at = datetime.utcnow()
+    for key, value in kwargs.items():
+        setattr(task, key, value)
+    db.session.commit()
+
+    # Update batch stats
+    if task.batch_id:
+        from app.tasks.scan import update_batch_stats
+        update_batch_stats.delay(task.batch_id)
+
+    log.error(f"Task {task.id} failed: {error}")
+    return {'status': 'failed', 'error': error}
+
+
+def _skip_task(task: ImportTask, reason: str, **kwargs) -> dict:
+    """Mark task as skipped with reason."""
+    task.status = 'skipped'
+    task.error_message = reason
+    task.completed_at = datetime.utcnow()
+    for key, value in kwargs.items():
+        setattr(task, key, value)
+    db.session.commit()
+
+    # Update batch stats
+    if task.batch_id:
+        from app.tasks.scan import update_batch_stats
+        update_batch_stats.delay(task.batch_id)
+
+    log.info(f"Task {task.id} skipped: {reason}")
+    return {'status': 'skipped', 'reason': reason}
+
+
+def _add_artist_tag(record: ImageRecord, artist: str):
+    """Add artist tag to image record."""
+    if not artist:
+        return
+
+    tag_name = f"artist:{artist}"
+    tag = Tag.query.filter_by(name=tag_name).first()
+    if not tag:
+        tag = Tag(name=tag_name, kind='artist')
+        db.session.add(tag)
+        db.session.flush()
+
+    if tag not in record.tags:
+        record.tags.append(tag)
+
+
+def _add_archive_tag(record: ImageRecord, archive_tag_id: int):
+    """Add archive tag to image record."""
+    if not archive_tag_id:
+        return
+
+    tag = Tag.query.get(archive_tag_id)
+    if tag and tag not in record.tags:
+        record.tags.append(tag)
+
+
+def _supersede_existing(task: ImportTask, old_record: ImageRecord, src_path: str,
+                        file_hash: str, phash, metadata: dict, artist: str,
+                        dest_dir: str, filename: str) -> dict:
+    """
+    Replace an existing smaller image with a larger version.
+
+    Preserves tags and metadata from the old record.
+    """
+    from app.utils.image_importer import (
+        build_hashed_dest_path, generate_thumbnail, generate_video_thumbnail_mirrored,
+        supersede_image
+    )
+    from app.utils.metadata_enrichment import find_sidecar_json, enrich_from_sidecar
+    from app.tasks.sidecar import apply_sidecar_metadata
+
+    log.info(f"Superseding {old_record.filename} ({old_record.width}x{old_record.height}) "
+             f"with {filename} ({metadata['width']}x{metadata['height']})")
+
+    # Enrich metadata from sidecar if available
+    sidecar_path = find_sidecar_json(src_path)
+    if sidecar_path:
+        enriched = enrich_from_sidecar(sidecar_path, {'taken_at': metadata.get('taken_at')})
+        if enriched.get('taken_at'):
+            metadata['taken_at'] = enriched['taken_at']
+
+    # Copy new file to destination
+    target_dir = os.path.join(dest_dir, artist)
+    os.makedirs(target_dir, exist_ok=True)
+    dest_path = build_hashed_dest_path(target_dir, filename, file_hash)
+    shutil.copy2(src_path, dest_path)
+
+    # Generate new thumbnail
+    try:
+        if dest_path.lower().endswith(('.mp4', '.mov', '.webm', '.avi', '.mkv')):
+            thumb_path = generate_video_thumbnail_mirrored(dest_path)
+        else:
+            thumb_path = generate_thumbnail(dest_path)
+    except Exception as e:
+        log.warning(f"Thumbnail generation failed: {e}")
+        thumb_path = None
+
+    # Supersede the old record
+    record = supersede_image(
+        old_record, dest_path, thumb_path, file_hash, phash, metadata, filename
+    )
+
+    db.session.commit()
+
+    # Complete task
+    task.status = 'complete'
+    task.result_image_id = record.id
+    task.completed_at = datetime.utcnow()
+    db.session.commit()
+
+    # Apply sidecar metadata if available
+    if sidecar_path:
+        apply_sidecar_metadata.delay(record.id, sidecar_path)
+
+    # Update batch stats
+    if task.batch_id:
+        from app.tasks.scan import update_batch_stats
+        update_batch_stats.delay(task.batch_id)
+
+    return {
+        'status': 'superseded',
+        'image_id': record.id,
+        'dest_path': dest_path
+    }

app/tasks/scan.py

@@ -0,0 +1,349 @@
+# app/tasks/scan.py
+"""
+Directory scanning task for the import pipeline.
+
+This task walks the import directory, detects new/changed files,
+creates ImportTask records, and queues appropriate processing tasks.
+"""
+import os
+import logging
+from datetime import datetime
+
+from app.celery_app import celery
+from app import db
+from app.models import ImportTask, ImportBatch
+
+log = logging.getLogger('celery.tasks.scan')
+
+# Supported file extensions (imported from image_importer for consistency)
+ALLOWED_MEDIA_EXTS = (
+    '.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff', '.tif',
+    '.mp4', '.mov', '.webm', '.avi', '.mkv'
+)
+ALLOWED_ARCHIVE_EXTS = (
+    '.zip', '.rar', '.7z', '.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tbz2',
+    '.cbr', '.cbz'
+)
+
+
+@celery.task(bind=True, name='app.tasks.scan.scan_directory')
+def scan_directory(self, source_dir: str = '/import', dest_dir: str = '/images'):
+    """
+    Scan source directory for new/changed files.
+
+    Creates ImportTask records for each file and queues them for processing.
+
+    Strategy:
+    1. Walk directory tree under each artist folder
+    2. For each file, check if already in ImportTask with same path+size
+    3. If new/changed, create ImportTask and queue appropriate task
+    4. Handle archives specially (first volume only, queue as atomic unit)
+
+    Args:
+        source_dir: Root directory to scan (default: /import)
+        dest_dir: Destination for imported files (default: /images)
+
+    Returns:
+        dict with batch_id and statistics
+    """
+    from app.tasks.import_file import import_media_file, import_archive
+    from app.utils.image_importer import is_first_volume
+
+    log.info(f"Starting directory scan: {source_dir}")
+
+    # Create batch record
+    batch = ImportBatch(source_directory=source_dir)
+    db.session.add(batch)
+    db.session.commit()
+
+    stats = {
+        'total_files': 0,
+        'queued_media': 0,
+        'queued_archives': 0,
+        'skipped_existing': 0,
+        'skipped_non_first_volume': 0,
+        'skipped_non_media': 0,
+    }
+
+    try:
+        # Iterate through artist directories
+        for artist_dir in os.listdir(source_dir):
+            artist_path = os.path.join(source_dir, artist_dir)
+            if not os.path.isdir(artist_path):
+                continue
+
+            log.debug(f"Scanning artist directory: {artist_dir}")
+
+            # Walk all files in artist directory (including subdirectories)
+            for root, _, files in os.walk(artist_path):
+                for filename in files:
+                    full_path = os.path.join(root, filename)
+                    lower = filename.lower()
+                    stats['total_files'] += 1
+
+                    # Skip sidecar JSON files (handled during import)
+                    if lower.endswith('.json'):
+                        continue
+
+                    # Check if archive
+                    if lower.endswith(ALLOWED_ARCHIVE_EXTS):
+                        # Only process first volume of multi-part archives
+                        if not is_first_volume(full_path):
+                            stats['skipped_non_first_volume'] += 1
+                            continue
+
+                        # Create and queue archive task
+                        task_record = _create_task_if_new(
+                            source_path=full_path,
+                            task_type='import_archive',
+                            batch_id=batch.id,
+                            context={'artist': artist_dir, 'dest_dir': dest_dir}
+                        )
+
+                        if task_record:
+                            result = import_archive.delay(task_record.id)
+                            task_record.celery_task_id = result.id
+                            task_record.status = 'queued'
+                            db.session.commit()
+                            stats['queued_archives'] += 1
+                            log.debug(f"Queued archive: {filename}")
+                        else:
+                            stats['skipped_existing'] += 1
+                        continue
+
+                    # Check if media file
+                    if not lower.endswith(ALLOWED_MEDIA_EXTS):
+                        stats['skipped_non_media'] += 1
+                        continue
+
+                    # Create and queue media file task
+                    task_record = _create_task_if_new(
+                        source_path=full_path,
+                        task_type='import_image',
+                        batch_id=batch.id,
+                        context={'artist': artist_dir, 'dest_dir': dest_dir}
+                    )
+
+                    if task_record:
+                        result = import_media_file.delay(task_record.id)
+                        task_record.celery_task_id = result.id
+                        task_record.status = 'queued'
+                        db.session.commit()
+                        stats['queued_media'] += 1
+                        log.debug(f"Queued media: {filename}")
+                    else:
+                        stats['skipped_existing'] += 1
+
+        # Update batch with totals
+        batch.total_files = stats['total_files']
+        db.session.commit()
+
+        log.info(f"Scan complete. Queued {stats['queued_media']} media, "
+                 f"{stats['queued_archives']} archives. "
+                 f"Skipped {stats['skipped_existing']} existing.")
+
+        return {
+            'batch_id': batch.id,
+            'stats': stats
+        }
+
+    except Exception as e:
+        log.error(f"Scan failed: {e}", exc_info=True)
+        batch.status = 'failed'
+        db.session.commit()
+        raise
+
+
+def _create_task_if_new(source_path: str, task_type: str, batch_id: int, context: dict) -> ImportTask:
+    """
+    Create ImportTask if no existing task for this file.
+
+    Uses file path and size for change detection.
+    Hash is computed later during actual processing.
+
+    Args:
+        source_path: Full path to source file
+        task_type: Type of task ('import_image', 'import_archive', etc.)
+        batch_id: ID of the current ImportBatch
+        context: Context dict with artist, dest_dir, etc.
+
+    Returns:
+        New ImportTask record if created, None if already exists
+    """
+    try:
+        file_size = os.path.getsize(source_path)
+    except OSError as e:
+        log.warning(f"Cannot access file {source_path}: {e}")
+        return None
+
+    # Check for existing pending/queued/processing task for same file
+    existing_active = ImportTask.query.filter_by(
+        source_path=source_path,
+        task_type=task_type
+    ).filter(
+        ImportTask.status.in_(['pending', 'queued', 'processing'])
+    ).first()
+
+    if existing_active:
+        log.debug(f"Task already active for: {source_path}")
+        return None
+
+    # Check for completed task with same file path and size
+    # If size changed, we should re-import
+    completed_same = ImportTask.query.filter_by(
+        source_path=source_path,
+        task_type=task_type,
+        file_size=file_size,
+        status='complete'
+    ).first()
+
+    if completed_same:
+        log.debug(f"Already imported (same size): {source_path}")
+        return None
+
+    # Also check 'skipped' status - don't re-queue skipped files unless size changed
+    skipped_same = ImportTask.query.filter_by(
+        source_path=source_path,
+        task_type=task_type,
+        file_size=file_size,
+        status='skipped'
+    ).first()
+
+    if skipped_same:
+        log.debug(f"Previously skipped (same size): {source_path}")
+        return None
+
+    # Create new task
+    task = ImportTask(
+        source_path=source_path,
+        task_type=task_type,
+        file_size=file_size,
+        batch_id=batch_id,
+        context=context,
+        status='pending'
+    )
+    db.session.add(task)
+    db.session.commit()
+
+    return task
+
+
+@celery.task(name='app.tasks.scan.recover_interrupted_tasks')
+def recover_interrupted_tasks():
+    """
+    Find and re-queue tasks that were interrupted.
+
+    Called periodically by Celery Beat to recover from worker crashes.
+    Tasks with status 'processing' or 'queued' that have no active
+    Celery task are re-queued.
+
+    Returns:
+        dict with count of recovered tasks
+    """
+    from app.tasks.import_file import import_media_file, import_archive
+    from app.tasks.thumbnail import generate_thumbnail_task
+
+    log.info("Checking for interrupted tasks...")
+
+    # Find tasks that are stuck in processing/queued state
+    # We look for tasks that have been in this state for more than 10 minutes
+    cutoff = datetime.utcnow()
+
+    interrupted = ImportTask.query.filter(
+        ImportTask.status.in_(['processing', 'queued'])
+    ).all()
+
+    recovered = 0
+    for task in interrupted:
+        # Check if Celery task is still active
+        if task.celery_task_id:
+            from app.celery_app import celery as celery_app
+            result = celery_app.AsyncResult(task.celery_task_id)
+            # If task is still pending/started, don't re-queue
+            if result.state in ('PENDING', 'STARTED', 'RETRY'):
+                continue
+
+        log.info(f"Recovering task {task.id}: {task.source_path}")
+
+        # Reset task state
+        task.status = 'pending'
+        task.celery_task_id = None
+        task.started_at = None
+        task.retry_count += 1
+
+        # Re-queue based on type
+        if task.task_type == 'import_image':
+            result = import_media_file.delay(task.id)
+        elif task.task_type == 'import_archive':
+            result = import_archive.delay(task.id)
+        elif task.task_type == 'thumbnail':
+            # Thumbnail tasks store image_id in context
+            image_id = task.context.get('image_id') if task.context else None
+            if image_id:
+                result = generate_thumbnail_task.delay(image_id)
+            else:
+                task.status = 'failed'
+                task.error_message = 'Missing image_id in context'
+                db.session.commit()
+                continue
+        else:
+            log.warning(f"Unknown task type: {task.task_type}")
+            continue
+
+        task.celery_task_id = result.id
+        task.status = 'queued'
+        recovered += 1
+
+    db.session.commit()
+
+    if recovered > 0:
+        log.info(f"Recovered {recovered} interrupted tasks")
+
+    return {'recovered': recovered}
+
+
+@celery.task(name='app.tasks.scan.update_batch_stats')
+def update_batch_stats(batch_id: int):
+    """
+    Update statistics for an ImportBatch based on its tasks.
+
+    Args:
+        batch_id: ID of the batch to update
+    """
+    batch = ImportBatch.query.get(batch_id)
+    if not batch:
+        return
+
+    from sqlalchemy import func
+
+    # Count tasks by status
+    status_counts = dict(
+        db.session.query(
+            ImportTask.status,
+            func.count(ImportTask.id)
+        ).filter(
+            ImportTask.batch_id == batch_id
+        ).group_by(ImportTask.status).all()
+    )
+
+    batch.processed_files = sum(
+        status_counts.get(s, 0)
+        for s in ['complete', 'skipped', 'failed']
+    )
+    batch.imported_count = status_counts.get('complete', 0)
+    batch.skipped_count = status_counts.get('skipped', 0)
+    batch.failed_count = status_counts.get('failed', 0)
+
+    # Check if batch is complete
+    pending_or_active = sum(
+        status_counts.get(s, 0)
+        for s in ['pending', 'queued', 'processing']
+    )
+
+    if pending_or_active == 0 and batch.status == 'running':
+        batch.status = 'complete'
+        batch.completed_at = datetime.utcnow()
+        log.info(f"Batch {batch_id} complete: {batch.imported_count} imported, "
+                 f"{batch.skipped_count} skipped, {batch.failed_count} failed")
+
+    db.session.commit()

app/tasks/sidecar.py

@@ -0,0 +1,181 @@
+# app/tasks/sidecar.py
+"""
+Sidecar metadata extraction and application tasks.
+
+Handles finding and applying metadata from Gallery-DL sidecar JSON files
+to imported images.
+"""
+import logging
+from datetime import datetime
+
+from app.celery_app import celery
+from app import db
+from app.models import ImageRecord, Tag
+
+log = logging.getLogger('celery.tasks.sidecar')
+
+
+@celery.task(bind=True, name='app.tasks.sidecar.apply_sidecar_metadata',
+             max_retries=2, default_retry_delay=30)
+def apply_sidecar_metadata(self, image_id: int, sidecar_path: str):
+    """
+    Apply metadata from a sidecar JSON file to an image record.
+
+    Extracts platform info, artist, post ID, dates, and generates
+    appropriate tags (source:platform, post:platform:artist:id).
+
+    Args:
+        image_id: ID of the ImageRecord to update
+        sidecar_path: Path to the sidecar JSON file
+
+    Returns:
+        dict with status and applied metadata info
+    """
+    from app.utils.metadata_enrichment import (
+        enrich_from_sidecar, resolve_date_conflict, load_sidecar_metadata
+    )
+
+    record = ImageRecord.query.get(image_id)
+    if not record:
+        log.warning(f"Image {image_id} not found")
+        return {'error': 'Image not found'}
+
+    try:
+        # Load sidecar data
+        sidecar_data = load_sidecar_metadata(sidecar_path)
+        if not sidecar_data:
+            return {'status': 'no_metadata', 'reason': 'Failed to load sidecar'}
+
+        # Get enriched metadata
+        enriched = enrich_from_sidecar(sidecar_path, {'taken_at': record.taken_at})
+
+        if not enriched.get('raw_metadata'):
+            return {'status': 'no_metadata', 'reason': 'No metadata extracted'}
+
+        tags_added = 0
+
+        # Update date if earlier
+        new_date = resolve_date_conflict(record.taken_at, enriched.get('taken_at'))
+        if new_date != record.taken_at:
+            record.taken_at = new_date
+            log.debug(f"Updated date for {image_id}: {new_date}")
+
+        # Apply tags from metadata
+        for tag_info in enriched.get('tags', []):
+            tag_name = tag_info.get('name')
+            tag_kind = tag_info.get('kind')
+            if not tag_name:
+                continue
+
+            tag = Tag.query.filter_by(name=tag_name).first()
+            if not tag:
+                tag = Tag(name=tag_name, kind=tag_kind)
+                db.session.add(tag)
+                db.session.flush()
+                log.debug(f"Created tag: {tag_name} ({tag_kind})")
+
+            if tag not in record.tags:
+                record.tags.append(tag)
+                tags_added += 1
+
+        db.session.commit()
+
+        log.info(f"Applied sidecar metadata to {image_id}: "
+                 f"platform={enriched.get('platform')}, tags_added={tags_added}")
+
+        return {
+            'status': 'applied',
+            'platform': enriched.get('platform'),
+            'artist': enriched.get('artist'),
+            'post_id': enriched.get('post_id'),
+            'tags_added': tags_added
+        }
+
+    except Exception as e:
+        db.session.rollback()
+        log.error(f"Sidecar application failed for {image_id}: {e}", exc_info=True)
+        raise self.retry(exc=e)
+
+
+@celery.task(name='app.tasks.sidecar.scan_and_match_sidecars')
+def scan_and_match_sidecars(batch_id: int = None, limit: int = 1000):
+    """
+    Scan for sidecar files and match them to imported images.
+
+    Looks at recently imported images and tries to find corresponding
+    sidecar JSON files for metadata enrichment.
+
+    Args:
+        batch_id: Optional batch ID to limit scope
+        limit: Maximum number of images to process
+
+    Returns:
+        dict with count of matched sidecars
+    """
+    from app.utils.metadata_enrichment import find_sidecar_json
+
+    log.info(f"Scanning for unmatched sidecars (batch={batch_id}, limit={limit})")
+
+    # Build query for images that might need sidecar matching
+    query = ImageRecord.query.filter(
+        ImageRecord.filepath.isnot(None)
+    ).order_by(ImageRecord.imported_at.desc())
+
+    if limit:
+        query = query.limit(limit)
+
+    matched = 0
+    already_has_source = 0
+
+    for record in query.all():
+        # Check if already has source tag (indicates sidecar was processed)
+        has_source_tag = any(
+            tag.kind == 'source' or tag.kind == 'post'
+            for tag in record.tags
+        )
+        if has_source_tag:
+            already_has_source += 1
+            continue
+
+        # Try to find sidecar
+        sidecar_path = find_sidecar_json(record.filepath)
+        if sidecar_path:
+            apply_sidecar_metadata.delay(record.id, sidecar_path)
+            matched += 1
+
+    log.info(f"Sidecar scan complete: matched={matched}, already_tagged={already_has_source}")
+    return {'matched': matched, 'already_tagged': already_has_source}
+
+
+@celery.task(name='app.tasks.sidecar.reprocess_sidecars_for_tag')
+def reprocess_sidecars_for_tag(tag_name: str):
+    """
+    Reprocess sidecar metadata for all images with a specific tag.
+
+    Useful for re-running sidecar extraction after updates to
+    the metadata extraction logic.
+
+    Args:
+        tag_name: Name of the tag to filter by
+
+    Returns:
+        dict with count of queued tasks
+    """
+    from app.utils.metadata_enrichment import find_sidecar_json
+
+    tag = Tag.query.filter_by(name=tag_name).first()
+    if not tag:
+        return {'error': f'Tag not found: {tag_name}'}
+
+    queued = 0
+    for record in tag.images:
+        if not record.filepath:
+            continue
+
+        sidecar_path = find_sidecar_json(record.filepath)
+        if sidecar_path:
+            apply_sidecar_metadata.delay(record.id, sidecar_path)
+            queued += 1
+
+    log.info(f"Queued {queued} sidecar reprocessing tasks for tag: {tag_name}")
+    return {'queued': queued, 'tag': tag_name}

app/tasks/thumbnail.py

@@ -0,0 +1,153 @@
+# app/tasks/thumbnail.py
+"""
+Thumbnail generation tasks.
+
+Handles generating thumbnails for images and videos,
+both individually and in bulk.
+"""
+import logging
+from datetime import datetime
+
+from app.celery_app import celery
+from app import db
+from app.models import ImageRecord, ImportTask
+
+log = logging.getLogger('celery.tasks.thumbnail')
+
+
+@celery.task(bind=True, name='app.tasks.thumbnail.generate_thumbnail_task',
+             max_retries=2, default_retry_delay=30)
+def generate_thumbnail_task(self, image_id: int, overwrite: bool = False):
+    """
+    Generate thumbnail for a single image/video.
+
+    Args:
+        image_id: ID of the ImageRecord
+        overwrite: Whether to overwrite existing thumbnail
+
+    Returns:
+        dict with status and thumbnail path
+    """
+    from app.utils.image_importer import generate_thumbnail, generate_video_thumbnail_mirrored
+
+    record = ImageRecord.query.get(image_id)
+    if not record:
+        log.warning(f"Image {image_id} not found")
+        return {'error': 'Image not found'}
+
+    try:
+        filepath = record.filepath
+        if not filepath:
+            return {'error': 'No filepath'}
+
+        # Check if thumbnail already exists and we're not overwriting
+        if record.thumb_path and not overwrite:
+            import os
+            if os.path.exists(record.thumb_path):
+                return {'status': 'exists', 'thumb_path': record.thumb_path}
+
+        log.debug(f"Generating thumbnail for: {filepath}")
+
+        is_video = filepath.lower().endswith(('.mp4', '.mov', '.webm', '.avi', '.mkv'))
+
+        if is_video:
+            thumb_path = generate_video_thumbnail_mirrored(filepath)
+        else:
+            thumb_path = generate_thumbnail(filepath, overwrite=overwrite)
+
+        if thumb_path:
+            record.thumb_path = thumb_path
+            db.session.commit()
+            log.info(f"Generated thumbnail: {thumb_path}")
+            return {'status': 'success', 'thumb_path': thumb_path}
+        else:
+            return {'status': 'failed', 'error': 'Thumbnail generation returned None'}
+
+    except Exception as e:
+        db.session.rollback()
+        log.error(f"Thumbnail generation failed for {image_id}: {e}", exc_info=True)
+        raise self.retry(exc=e)
+
+
+@celery.task(name='app.tasks.thumbnail.regenerate_all_thumbnails')
+def regenerate_all_thumbnails(batch_size: int = 100, overwrite: bool = True):
+    """
+    Queue thumbnail regeneration for all images.
+
+    Processes images in batches to avoid memory issues.
+
+    Args:
+        batch_size: Number of images to process per batch
+        overwrite: Whether to overwrite existing thumbnails
+
+    Returns:
+        dict with count of queued tasks
+    """
+    log.info("Starting bulk thumbnail regeneration")
+
+    count = 0
+    last_id = 0
+
+    while True:
+        records = ImageRecord.query.filter(
+            ImageRecord.id > last_id,
+            ImageRecord.filepath.isnot(None)
+        ).order_by(ImageRecord.id.asc()).limit(batch_size).all()
+
+        if not records:
+            break
+
+        for record in records:
+            generate_thumbnail_task.delay(record.id, overwrite=overwrite)
+            count += 1
+
+        last_id = records[-1].id
+        log.info(f"Queued {count} thumbnail tasks so far...")
+
+    log.info(f"Thumbnail regeneration complete: queued {count} tasks")
+    return {'queued': count}
+
+
+@celery.task(name='app.tasks.thumbnail.regenerate_missing_thumbnails')
+def regenerate_missing_thumbnails(batch_size: int = 100):
+    """
+    Generate thumbnails only for images that don't have them.
+
+    Args:
+        batch_size: Number of images to process per batch
+
+    Returns:
+        dict with count of queued tasks
+    """
+    import os
+
+    log.info("Checking for missing thumbnails")
+
+    count = 0
+    last_id = 0
+
+    while True:
+        records = ImageRecord.query.filter(
+            ImageRecord.id > last_id,
+            ImageRecord.filepath.isnot(None)
+        ).order_by(ImageRecord.id.asc()).limit(batch_size).all()
+
+        if not records:
+            break
+
+        for record in records:
+            # Check if thumbnail is missing
+            needs_thumb = False
+            if not record.thumb_path:
+                needs_thumb = True
+            elif not os.path.exists(record.thumb_path):
+                needs_thumb = True
+
+            if needs_thumb:
+                generate_thumbnail_task.delay(record.id, overwrite=False)
+                count += 1
+
+        last_id = records[-1].id
+
+    log.info(f"Missing thumbnail check complete: queued {count} tasks")
+    return {'queued': count}

app/templates/settings.html

@@ -206,7 +206,140 @@
   </div>
 </div><!-- end settings-grid -->
 
+<!-- Import Queue Status (Full Width) -->
+<div class="settings-container" style="max-width: 1200px; margin: 2rem auto;">
+  <div class="settings-section">
+    <h2>Import Queue Status</h2>
+    <p class="settings-description">Monitor and control the Celery-based import queue system.</p>
+
+    <div id="queueStatusSection">
+      <div class="stats-grid" style="grid-template-columns: repeat(6, 1fr);">
+        <div class="stat-item">
+          <span class="stat-value" id="queuePending">-</span>
+          <span class="stat-label">Pending</span>
+        </div>
+        <div class="stat-item">
+          <span class="stat-value" id="queueQueued">-</span>
+          <span class="stat-label">Queued</span>
+        </div>
+        <div class="stat-item">
+          <span class="stat-value" id="queueProcessing">-</span>
+          <span class="stat-label">Processing</span>
+        </div>
+        <div class="stat-item">
+          <span class="stat-value" id="queueComplete">-</span>
+          <span class="stat-label">Complete</span>
+        </div>
+        <div class="stat-item">
+          <span class="stat-value" id="queueSkipped">-</span>
+          <span class="stat-label">Skipped</span>
+        </div>
+        <div class="stat-item">
+          <span class="stat-value" id="queueFailed">-</span>
+          <span class="stat-label">Failed</span>
+        </div>
+      </div>
+
+      <div id="activeBatchInfo" class="import-stats" style="margin-top: 1rem; display: none;">
+        <h3>Active Batch</h3>
+        <p>Processing <strong id="batchDir">-</strong></p>
+        <div class="batch-progress">
+          <div class="progress-bar">
+            <div class="progress-fill" id="batchProgressFill" style="width: 0%"></div>
+          </div>
+          <span id="batchProgressText">0 / 0</span>
+        </div>
+      </div>
+
+      <div class="queue-actions" style="margin-top: 1rem; display: flex; gap: 0.75rem; flex-wrap: wrap;">
+        <button id="triggerImportBtn" class="btn primary-btn">Trigger Import Scan</button>
+        <button id="retryFailedBtn" class="btn warning-btn">Retry Failed Tasks</button>
+        <button id="clearCompletedBtn" class="btn secondary-btn">Clear Completed</button>
+        <button id="regenMissingThumbsBtn" class="btn secondary-btn">Regenerate Missing Thumbnails</button>
+        <a href="/flower" target="_blank" class="btn secondary-btn" style="text-decoration: none;">Open Flower Dashboard</a>
+      </div>
+
+      <div id="celeryWorkerInfo" style="margin-top: 1rem;">
+        <p style="color: var(--text-muted); font-size: 0.85rem;">
+          Workers: <strong id="workerCount">-</strong> |
+          Active Tasks: <strong id="activeTasks">-</strong>
+        </p>
+      </div>
+    </div>
+
+    <!-- Recent Tasks Table -->
+    <div id="recentTasksSection" style="margin-top: 1.5rem;">
+      <h3>Recent Tasks</h3>
+      <div class="tasks-table-wrapper" style="max-height: 300px; overflow-y: auto;">
+        <table class="tasks-table">
+          <thead>
+            <tr>
+              <th>File</th>
+              <th>Type</th>
+              <th>Status</th>
+              <th>Error</th>
+              <th>Time</th>
+            </tr>
+          </thead>
+          <tbody id="recentTasksBody">
+            <tr><td colspan="5" style="text-align: center; color: var(--text-muted);">Loading...</td></tr>
+          </tbody>
+        </table>
+      </div>
+    </div>
+  </div>
+</div>
+
 <style>
+  .tasks-table {
+    width: 100%;
+    border-collapse: collapse;
+    font-size: 0.85rem;
+  }
+  .tasks-table th,
+  .tasks-table td {
+    padding: 0.5rem 0.75rem;
+    text-align: left;
+    border-bottom: 1px solid rgba(255, 255, 255, 0.08);
+  }
+  .tasks-table th {
+    background: rgba(255, 255, 255, 0.05);
+    color: var(--text-dim);
+    font-weight: 500;
+    position: sticky;
+    top: 0;
+  }
+  .tasks-table td {
+    color: var(--text);
+  }
+  .tasks-table .status-pending { color: #9ca3af; }
+  .tasks-table .status-queued { color: #60a5fa; }
+  .tasks-table .status-processing { color: #fbbf24; }
+  .tasks-table .status-complete { color: #34d399; }
+  .tasks-table .status-skipped { color: #a78bfa; }
+  .tasks-table .status-failed { color: #f87171; }
+  .batch-progress {
+    display: flex;
+    align-items: center;
+    gap: 1rem;
+    margin-top: 0.5rem;
+  }
+  .progress-bar {
+    flex: 1;
+    height: 8px;
+    background: rgba(255, 255, 255, 0.1);
+    border-radius: 4px;
+    overflow: hidden;
+  }
+  .progress-fill {
+    height: 100%;
+    background: var(--btn-primary);
+    transition: width 0.3s ease;
+  }
+  .queue-actions .btn {
+    min-width: auto;
+    white-space: nowrap;
+  }
   .settings-description {
     color: var(--text-muted);
     margin-bottom: 1rem;
@@ -363,6 +496,177 @@
 
 <script>
 document.addEventListener('DOMContentLoaded', () => {
+  // ========================================
+  // Celery Queue Status
+  // ========================================
+  async function loadQueueStatus() {
+    try {
+      const r = await fetch('/api/import/status');
+      const data = await r.json();
+
+      if (data.ok) {
+        // Update status counts
+        document.getElementById('queuePending').textContent = data.status_counts.pending || 0;
+        document.getElementById('queueQueued').textContent = data.status_counts.queued || 0;
+        document.getElementById('queueProcessing').textContent = data.status_counts.processing || 0;
+        document.getElementById('queueComplete').textContent = data.status_counts.complete || 0;
+        document.getElementById('queueSkipped').textContent = data.status_counts.skipped || 0;
+        document.getElementById('queueFailed').textContent = data.status_counts.failed || 0;
+
+        // Update active batch info
+        const batchEl = document.getElementById('activeBatchInfo');
+        if (data.active_batch) {
+          batchEl.style.display = 'block';
+          document.getElementById('batchDir').textContent = data.active_batch.source_directory;
+          const processed = data.active_batch.processed_files || 0;
+          const total = data.active_batch.total_files || 1;
+          const pct = Math.round((processed / total) * 100);
+          document.getElementById('batchProgressFill').style.width = pct + '%';
+          document.getElementById('batchProgressText').textContent = `${processed} / ${total}`;
+        } else {
+          batchEl.style.display = 'none';
+        }
+
+        // Update recent tasks table
+        const tbody = document.getElementById('recentTasksBody');
+        if (data.recent_tasks && data.recent_tasks.length > 0) {
+          tbody.innerHTML = data.recent_tasks.map(t => `
+            <tr>
+              <td title="${t.source_path || ''}">${t.source_path || '-'}</td>
+              <td>${t.task_type || '-'}</td>
+              <td class="status-${t.status}">${t.status || '-'}</td>
+              <td title="${t.error_message || ''}">${t.error_message ? t.error_message.substring(0, 30) + '...' : '-'}</td>
+              <td>${t.created_at ? new Date(t.created_at).toLocaleString() : '-'}</td>
+            </tr>
+          `).join('');
+        } else {
+          tbody.innerHTML = '<tr><td colspan="5" style="text-align: center; color: var(--text-muted);">No recent tasks</td></tr>';
+        }
+      }
+    } catch (e) {
+      console.error('Failed to load queue status:', e);
+    }
+  }
+
+  async function loadCeleryStatus() {
+    try {
+      const r = await fetch('/api/celery/status');
+      const data = await r.json();
+
+      if (data.ok) {
+        document.getElementById('workerCount').textContent = data.worker_count || 0;
+        document.getElementById('activeTasks').textContent = data.active_tasks || 0;
+      } else {
+        document.getElementById('workerCount').textContent = '?';
+        document.getElementById('activeTasks').textContent = '?';
+      }
+    } catch (e) {
+      document.getElementById('workerCount').textContent = 'offline';
+      document.getElementById('activeTasks').textContent = '-';
+    }
+  }
+
+  // Load queue status on page load and refresh periodically
+  loadQueueStatus();
+  loadCeleryStatus();
+  setInterval(loadQueueStatus, 5000);  // Refresh every 5 seconds
+  setInterval(loadCeleryStatus, 10000);  // Refresh every 10 seconds
+
+  // Trigger Import Scan
+  document.getElementById('triggerImportBtn').addEventListener('click', async () => {
+    const btn = document.getElementById('triggerImportBtn');
+    btn.disabled = true;
+    btn.textContent = 'Starting...';
+
+    try {
+      const r = await fetch('/api/import/trigger', { method: 'POST' });
+      const data = await r.json();
+
+      if (data.ok) {
+        alert('Import scan started! Task ID: ' + data.task_id);
+        loadQueueStatus();
+      } else {
+        alert('Failed to start import: ' + (data.error || 'Unknown error'));
+      }
+    } catch (e) {
+      alert('Failed to start import: ' + e.message);
+    } finally {
+      btn.disabled = false;
+      btn.textContent = 'Trigger Import Scan';
+    }
+  });
+
+  // Retry Failed Tasks
+  document.getElementById('retryFailedBtn').addEventListener('click', async () => {
+    const btn = document.getElementById('retryFailedBtn');
+    btn.disabled = true;
+    btn.textContent = 'Retrying...';
+
+    try {
+      const r = await fetch('/api/import/retry-failed', { method: 'POST' });
+      const data = await r.json();
+
+      if (data.ok) {
+        alert(`Retried ${data.retried} failed task(s).`);
+        loadQueueStatus();
+      } else {
+        alert('Failed to retry: ' + (data.error || 'Unknown error'));
+      }
+    } catch (e) {
+      alert('Failed to retry: ' + e.message);
+    } finally {
+      btn.disabled = false;
+      btn.textContent = 'Retry Failed Tasks';
+    }
+  });
+
+  // Clear Completed Tasks
+  document.getElementById('clearCompletedBtn').addEventListener('click', async () => {
+    const btn = document.getElementById('clearCompletedBtn');
+    btn.disabled = true;
+    btn.textContent = 'Clearing...';
+
+    try {
+      const r = await fetch('/api/import/clear-completed', { method: 'POST' });
+      const data = await r.json();
+
+      if (data.ok) {
+        alert(`Cleared ${data.deleted} completed/skipped task(s).`);
+        loadQueueStatus();
+      } else {
+        alert('Failed to clear: ' + (data.error || 'Unknown error'));
+      }
+    } catch (e) {
+      alert('Failed to clear: ' + e.message);
+    } finally {
+      btn.disabled = false;
+      btn.textContent = 'Clear Completed';
+    }
+  });
+
+  // Regenerate Missing Thumbnails
+  document.getElementById('regenMissingThumbsBtn').addEventListener('click', async () => {
+    const btn = document.getElementById('regenMissingThumbsBtn');
+    btn.disabled = true;
+    btn.textContent = 'Starting...';
+
+    try {
+      const r = await fetch('/api/thumbnails/regenerate-missing', { method: 'POST' });
+      const data = await r.json();
+
+      if (data.ok) {
+        alert('Missing thumbnail generation started! Task ID: ' + data.task_id);
+      } else {
+        alert('Failed to start: ' + (data.error || 'Unknown error'));
+      }
+    } catch (e) {
+      alert('Failed to start: ' + e.message);
+    } finally {
+      btn.disabled = false;
+      btn.textContent = 'Regenerate Missing Thumbnails';
+    }
+  });
+
   // Load import settings
   fetch('/api/import-settings')
     .then(r => r.json())

app/templates/tags_list.html

@@ -5,6 +5,8 @@
   {%- elif active_kind == 'series' -%}Series
   {%- elif active_kind == 'rating' -%}Ratings
   {%- elif active_kind == 'archive' -%}Archives
+  {%- elif active_kind == 'source' -%}Sources
+  {%- elif active_kind == 'post' -%}Posts
   {%- elif active_kind == 'user' -%}User Tags
   {%- else -%}Tags{%- endif -%}
 {% endblock %}
@@ -16,6 +18,8 @@
   {%- elif active_kind == 'series' -%}Series
   {%- elif active_kind == 'rating' -%}Ratings
   {%- elif active_kind == 'archive' -%}Archives
+  {%- elif active_kind == 'source' -%}Sources
+  {%- elif active_kind == 'post' -%}Posts
   {%- elif active_kind == 'user' -%}User Tags
   {%- else -%}All Tags{%- endif -%}
 </h1>
@@ -46,6 +50,14 @@
      href="{{ url_for('main.tag_list', kind='archive') }}"
      aria-current="{{ 'page' if active_kind=='archive' else 'false' }}">🗜️ Archives</a>
 
+  <a class="filter-btn {{ 'is-active' if active_kind=='source' }}"
+     href="{{ url_for('main.tag_list', kind='source') }}"
+     aria-current="{{ 'page' if active_kind=='source' else 'false' }}">🌐 Sources</a>
+
+  <a class="filter-btn {{ 'is-active' if active_kind=='post' }}"
+     href="{{ url_for('main.tag_list', kind='post') }}"
+     aria-current="{{ 'page' if active_kind=='post' else 'false' }}">📌 Posts</a>
+
   <a class="filter-btn {{ 'is-active' if active_kind=='user' }}"
      href="{{ url_for('main.tag_list', kind='user') }}"
      aria-current="{{ 'page' if active_kind=='user' else 'false' }}"># User</a>
@@ -58,9 +70,10 @@
       <a href="{{ url_for('main.gallery', tag=tag.tag) }}" class="tag-card-link">
         <div class="tag-preview">
           {% for image in tag.images %}
-            <div class="tag-thumb"
+            <img class="tag-thumb"
-                 style="background-image: url('{{ url_for('main.serve_image', filename=(image.thumb_path or image.filepath) | replace('/images/', '') ) }}');">
+                 src="{{ url_for('main.serve_image', filename=(image.thumb_path or image.filepath) | replace('/images/', '') ) }}"
-            </div>
+                 loading="lazy"
+                 alt="">
           {% endfor %}
           {% if tag.images|length == 0 %}
             <div class="tag-thumb tag-thumb-empty"></div>
@@ -73,6 +86,8 @@
             {%- elif tag.kind == 'series' -%}📺
             {%- elif tag.kind == 'rating' -%}⚠️
             {%- elif tag.kind == 'archive' -%}🗜️
+            {%- elif tag.kind == 'source' -%}🌐
+            {%- elif tag.kind == 'post' -%}📌
             {%- else -%}#{%- endif -%}
           </span>
           <span class="tag-name" title="{{ tag.name }}">{{ tag.name }}</span>
@@ -109,6 +124,8 @@
           <option value="series">📺 Series</option>
           <option value="rating">⚠️ Rating</option>
           <option value="archive">🗜️ Archive</option>
+          <option value="source">🌐 Source</option>
+          <option value="post">📌 Post</option>
         </select>
       </div>
 

app/utils/metadata_enrichment.py

@@ -28,12 +28,10 @@ def find_sidecar_json(image_path: str) -> Optional[str]:
     """
     Find the Gallery-DL sidecar JSON file for an image.
 
-    Gallery-DL creates JSON files with the UUID portion of the filename.
+    Gallery-DL creates JSON files with various naming patterns:
-    For example:
+    - Patreon: {filename}.json in same directory
-        Image: 01_BBEFD45F-8775-4DF9-8BCA-99E6063E3616.jpg
+    - HentaiFoundry: {artist}-{index}-{title}.json (may differ from image filename)
-        JSON:  BBEFD45F-8775-4DF9-8BCA-99E6063E3616.json
+    - UUID-based: {uuid}.json where UUID is extracted from filename
-
-    Also checks for direct {filename}.json pattern.
 
     Returns the path to the sidecar JSON if found, None otherwise.
     """
@@ -41,12 +39,18 @@ def find_sidecar_json(image_path: str) -> Optional[str]:
     parent_dir = image_path.parent
     stem = image_path.stem  # filename without extension
 
-    # Pattern 1: Direct {filename}.json (e.g., image.jpg -> image.jpg.json)
+    # Pattern 1: Same stem with .json extension (most common)
+    # e.g., "Artist-12345-title.jpg" -> "Artist-12345-title.json"
+    stem_json = parent_dir / f"{stem}.json"
+    if stem_json.exists():
+        return str(stem_json)
+
+    # Pattern 2: Direct {filename}.json (e.g., image.jpg -> image.jpg.json)
     direct_json = image_path.with_suffix(image_path.suffix + ".json")
     if direct_json.exists():
         return str(direct_json)
 
-    # Pattern 2: Extract UUID from filename and look for {uuid}.json
+    # Pattern 3: Extract UUID from filename and look for {uuid}.json
     # Common patterns: "01_UUID", "UUID", "{num}_{UUID}"
     uuid_pattern = re.compile(r'([A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{12})', re.IGNORECASE)
     match = uuid_pattern.search(stem)
@@ -56,7 +60,7 @@ def find_sidecar_json(image_path: str) -> Optional[str]:
         if uuid_json.exists():
             return str(uuid_json)
 
-    # Pattern 3: Just the stem without prefix numbers
+    # Pattern 4: Just the stem without prefix numbers
     # e.g., "01_filename" -> look for "filename.json"
     stem_no_prefix = re.sub(r'^\d+_', '', stem)
     if stem_no_prefix != stem:
@@ -64,6 +68,18 @@ def find_sidecar_json(image_path: str) -> Optional[str]:
         if alt_json.exists():
             return str(alt_json)
 
+    # Pattern 5: Extract numeric ID from filename and find JSON with same ID
+    # Handles HentaiFoundry where image is "hentaifoundry_1000220_title.jpg"
+    # but JSON is "Artist-1000220-title.json"
+    # Look for a 5-7 digit number that could be a post ID
+    id_match = re.search(r'[_-](\d{5,8})[_-]', stem)
+    if id_match:
+        post_id = id_match.group(1)
+        # Search for any JSON file in the directory containing this ID
+        for json_file in parent_dir.glob("*.json"):
+            if f"-{post_id}-" in json_file.name or f"_{post_id}_" in json_file.name:
+                return str(json_file)
+
     return None
 
 
@@ -122,9 +138,14 @@ def extract_artist(metadata: dict) -> Optional[str]:
 
 def extract_post_id(metadata: dict) -> Optional[str]:
     """Extract the post ID from metadata."""
+    # Patreon, general
     if "id" in metadata:
         return str(metadata["id"])
 
+    # HentaiFoundry uses "index" as the post ID
+    if "index" in metadata:
+        return str(metadata["index"])
+
     if "post_id" in metadata:
         return str(metadata["post_id"])
 

config.py

@@ -28,3 +28,7 @@ class Config:
 
     SQLALCHEMY_TRACK_MODIFICATIONS = False
 
+    # Celery configuration
+    CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://redis:6379/0')
+    CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://redis:6379/0')
+    CELERY_WORKER_CONCURRENCY = int(os.environ.get('CELERY_WORKER_CONCURRENCY', '2'))

docker-compose.sqlite.yml

@@ -0,0 +1,86 @@
+# docker-compose.sqlite.yml
+# Development/testing configuration using SQLite instead of PostgreSQL
+# Usage: docker-compose -f docker-compose.sqlite.yml up
+#
+# NOTE: SQLite does not support concurrent writes well.
+# This configuration is for development/testing only.
+# For production, use the main docker-compose.yml with PostgreSQL.
+
+services:
+  # Redis message broker for Celery
+  redis:
+    image: redis:7-alpine
+    ports:
+      - "6379:6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    restart: unless-stopped
+
+  # Flask web application (SQLite mode)
+  web:
+    build: .
+    ports:
+      - "5000:5000"
+    environment:
+      - DB_TYPE=sqlite
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - RUN_IMAGE_IMPORTER=0  # Disable old polling worker
+    volumes:
+      - ./imagerepo/db/:/db
+      - ./imagerepo/images:/images
+      - ./import:/import
+      - ./migrations:/app/migrations
+    depends_on:
+      redis:
+        condition: service_healthy
+    restart: unless-stopped
+
+  # Single Celery worker (SQLite needs single worker to avoid write conflicts)
+  celery-worker:
+    build: .
+    command: celery -A app.celery_app:celery worker --loglevel=info -Q scan,import,thumbnail,sidecar,default --concurrency=1
+    environment:
+      - DB_TYPE=sqlite
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+    volumes:
+      - ./imagerepo/db/:/db
+      - ./imagerepo/images:/images
+      - ./import:/import
+    depends_on:
+      redis:
+        condition: service_healthy
+    restart: unless-stopped
+
+  # Celery Beat scheduler (optional for development)
+  celery-beat:
+    build: .
+    command: celery -A app.celery_app:celery beat --loglevel=info
+    environment:
+      - DB_TYPE=sqlite
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - IMPORT_EVERY_SECONDS=28800
+    volumes:
+      - ./imagerepo/db/:/db
+    depends_on:
+      - redis
+      - celery-worker
+    restart: unless-stopped
+
+  # Flower monitoring (optional)
+  flower:
+    build: .
+    command: celery -A app.celery_app:celery flower --port=5555
+    ports:
+      - "5555:5555"
+    environment:
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+    depends_on:
+      - redis
+    restart: unless-stopped

docker-compose.yml

@@ -1,10 +1,103 @@
 services:
+  # Redis message broker for Celery
+  redis:
+    image: redis:7-alpine
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis_data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    restart: unless-stopped
+
+  # PostgreSQL database (production-ready)
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER: ${DB_USER:-imagerepo}
+      POSTGRES_PASSWORD: ${DB_PASS:-postgres}
+      POSTGRES_DB: ${DB_NAME:-imagerepo}
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-imagerepo}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    restart: unless-stopped
+
+  # Flask web application
   web:
     build: .
     ports:
       - "5000:5000"
+    environment:
+      - DB_TYPE=postgresql
+      - DB_USER=${DB_USER:-imagerepo}
+      - DB_PASS=${DB_PASS:-postgres}
+      - DB_HOST=postgres
+      - DB_PORT=5432
+      - DB_NAME=${DB_NAME:-imagerepo}
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
     volumes:
-      - ./imagerepo/db/:/db # SQLite db storage
+      - ./imagerepo/db/:/db  # Legacy SQLite path (not used with PostgreSQL)
-      - ./imagerepo/images:/images # where the app will store it's imported images
+      - ./imagerepo/images:/images
-      - ./import:/import # where it will import images from
+      - ./import:/import
-      - ./migrations:/app/migrations # for dev
+      - ./migrations:/app/migrations  # For dev
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    restart: unless-stopped
+
+  # Celery worker for processing import tasks
+  celery-worker:
+    build: .
+    command: celery -A app.celery_app:celery worker --loglevel=info -Q scan,import,thumbnail,sidecar,default --concurrency=${CELERY_WORKER_CONCURRENCY:-2}
+    environment:
+      - DB_TYPE=postgresql
+      - DB_USER=${DB_USER:-imagerepo}
+      - DB_PASS=${DB_PASS:-postgres}
+      - DB_HOST=postgres
+      - DB_PORT=5432
+      - DB_NAME=${DB_NAME:-imagerepo}
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - CELERY_WORKER_CONCURRENCY=${CELERY_WORKER_CONCURRENCY:-2}
+    volumes:
+      - ./imagerepo/images:/images
+      - ./import:/import
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    restart: unless-stopped
+
+  # Celery Beat scheduler for periodic tasks
+  celery-beat:
+    build: .
+    command: celery -A app.celery_app:celery beat --loglevel=info
+    environment:
+      - DB_TYPE=postgresql
+      - DB_USER=${DB_USER:-imagerepo}
+      - DB_PASS=${DB_PASS:-postgres}
+      - DB_HOST=postgres
+      - DB_PORT=5432
+      - DB_NAME=${DB_NAME:-imagerepo}
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - IMPORT_EVERY_SECONDS=${IMPORT_EVERY_SECONDS:-28800}
+    depends_on:
+      - redis
+      - celery-worker
+    restart: unless-stopped
+
+volumes:
+  redis_data:
+  postgres_data:

entrypoint.sh

@@ -5,6 +5,14 @@ set -e
 export FLASK_APP=run.py
 export FLASK_ENV=production
 
+# If arguments are passed and first arg is NOT "web", execute them directly
+# This allows: docker run <image> celery -A app.celery_app:celery worker ...
+if [ $# -gt 0 ] && [ "$1" != "web" ]; then
+  exec "$@"
+fi
+
+# --- Web service mode (default or explicit "web" argument) ---
+
 # --- Migrations with simple retry (no pg_isready needed) ---
 if [ "${RUN_DB_MIGRATIONS:-1}" = "1" ]; then
   ATTEMPTS="${MIGRATION_MAX_ATTEMPTS:-30}"      # ~90s default with 3s sleep
@@ -26,12 +34,6 @@ if [ "${RUN_DB_MIGRATIONS:-1}" = "1" ]; then
   flask version-check
 fi
 
-# --- Start the image import worker exactly once per container ---
-if [ "${RUN_IMAGE_IMPORTER:-1}" = "1" ]; then
-  echo "Starting image import worker (background)…"
-  python image_import_worker.py &
-fi
-
 # --- Start Gunicorn ---
 GUNICORN_BIND="${GUNICORN_BIND:-0.0.0.0:5000}"
 GUNICORN_WORKERS="${GUNICORN_WORKERS:-8}"

image_import_worker.py

@@ -1,451 +0,0 @@
-# image_import_worker.py
-import os
-import time
-import logging
-from dataclasses import dataclass
-from typing import Callable, Optional, Tuple
-
-from sqlalchemy.exc import OperationalError
-
-from app import create_app
-from app.utils.image_importer import import_images_task
-
-# -------------------------------------------------------------------
-# Paths & basic config
-# -------------------------------------------------------------------
-THUMBNAIL_FLAG = "/import/thumbnail.flag"
-TRIGGER_FLAG   = "/import/trigger.flag"
-SOURCE_DIR     = "/import"
-DEST_DIR       = "/images"
-
-CHECK_INTERVAL = int(os.getenv("CHECK_INTERVAL", "10"))  # seconds
-
-# Periodic import interval (default 8 hours)
-IMPORT_EVERY_SECONDS = int(os.getenv("IMPORT_EVERY_SECONDS", str(8 * 60 * 60)))
-
-# Thumbnail verbosity controls
-THUMBS_VERBOSE     = os.getenv("THUMBS_VERBOSE", "0").lower() in ("1", "true", "yes")
-THUMBS_LOG_EVERY   = int(os.getenv("THUMBS_LOG_EVERY", "50"))
-THUMBS_BATCH_SIZE  = int(os.getenv("THUMBS_BATCH_SIZE", "500"))
-
-MAX_RETRIES = int(os.getenv("MAX_RETRIES", "3"))
-
-# -------------------------------------------------------------------
-# Logging
-# -------------------------------------------------------------------
-logging.basicConfig(
-    level=os.getenv("LOG_LEVEL", "INFO").upper(),
-    format="%(asctime)s %(levelname)s %(message)s",
-)
-log = logging.getLogger("image-import-worker")
-
-app = create_app()
-
-
-# -------------------------------------------------------------------
-# Job / flag helpers
-# -------------------------------------------------------------------
-@dataclass(frozen=True)
-class FlagJob:
-    name: str
-    flag_path: str
-
-    @property
-    def run_path(self) -> str:
-        return self.flag_path + ".run"
-
-    @property
-    def err_path(self) -> str:
-        return self.flag_path + ".err"
-
-
-def claim_job(job: FlagJob) -> bool:
-    """
-    Atomically claim a job by renaming <flag> -> <flag>.run.
-    Returns True if claimed, False otherwise.
-    """
-    try:
-        os.replace(job.flag_path, job.run_path)
-        log.info("[%s] Claimed job -> %s", job.name, job.run_path)
-        return True
-    except FileNotFoundError:
-        # Flag disappeared between exists() check and replace()
-        return False
-    except Exception as e:
-        log.warning("[%s] Could not claim job: %s", job.name, e)
-        return False
-
-
-def complete_job(job: FlagJob) -> None:
-    """Remove the .run file on success."""
-    try:
-        os.remove(job.run_path)
-        log.info("[%s] Cleared run flag.", job.name)
-    except FileNotFoundError:
-        log.info("[%s] Run flag already cleared.", job.name)
-    except Exception as e:
-        log.warning("[%s] Could not remove run flag: %s", job.name, e)
-
-
-def fail_job(job: FlagJob, error_text: Optional[str] = None) -> None:
-    """
-    Move .run -> .err and write error details into .err.
-    """
-    import datetime
-
-    try:
-        os.replace(job.run_path, job.err_path)
-        log.error("[%s] Marked failure -> %s", job.name, job.err_path)
-
-        if error_text:
-            try:
-                with open(job.err_path, "a", encoding="utf-8") as f:
-                    f.write("\n")
-                    f.write(datetime.datetime.utcnow().isoformat() + "Z\n")
-                    f.write(error_text)
-                    f.write("\n")
-            except Exception as e:
-                log.warning("[%s] Could not write error details: %s", job.name, e)
-
-    except FileNotFoundError:
-        log.error("[%s] Could not mark failure: run flag missing.", job.name)
-    except Exception as e:
-        log.error("[%s] Could not mark failure: %s", job.name, e)
-
-
-# -------------------------------------------------------------------
-# DB helpers
-# -------------------------------------------------------------------
-def db_ping() -> None:
-    """Validate DB session connection quickly."""
-    from sqlalchemy import text
-    from app import db
-
-    db.session.execute(text("SELECT 1"))
-
-
-def recover_db_session(dispose_engine: bool = False) -> None:
-    """
-    Rollback session and optionally dispose engine pool to force fresh connections.
-    """
-    try:
-        from app import db
-        db.session.rollback()
-        if dispose_engine:
-            db.engine.dispose()
-    except Exception:
-        pass
-
-
-def run_with_retries(
-    job: FlagJob,
-    fn: Callable[[], None],
-    *,
-    dispose_engine_on_operational_error: bool = False,
-    max_retries: int = MAX_RETRIES,
-) -> Tuple[bool, Optional[str]]:
-    """
-    Run fn() with retry/backoff for OperationalError.
-    Returns (success, error_text). error_text contains the last traceback on failure.
-    """
-    import traceback
-
-    attempt = 0
-    last_error_text: Optional[str] = None
-
-    while attempt < max_retries:
-        try:
-            # Catch stale pool conns early
-            db_ping()
-
-            fn()
-            return True, None
-
-        except OperationalError as e:
-            attempt += 1
-            last_error_text = "".join(traceback.format_exception(type(e), e, e.__traceback__))
-            log.warning(
-                "[%s] DB error (attempt %d/%d): %s",
-                job.name,
-                attempt,
-                max_retries,
-                e,
-            )
-
-            recover_db_session(dispose_engine=dispose_engine_on_operational_error)
-
-            if attempt >= max_retries:
-                return False, last_error_text
-
-            sleep_time = 2 ** attempt
-            log.info("[%s] Retrying in %ss...", job.name, sleep_time)
-            time.sleep(sleep_time)
-
-        except Exception as e:
-            last_error_text = "".join(traceback.format_exception(type(e), e, e.__traceback__))
-            log.exception("[%s] Unexpected error: %s", job.name, e)
-            recover_db_session(dispose_engine=True)
-            return False, last_error_text
-
-    return False, last_error_text
-
-
-# -------------------------------------------------------------------
-# Actual job bodies
-# -------------------------------------------------------------------
-def do_import() -> None:
-    result = import_images_task(SOURCE_DIR, DEST_DIR)
-    log.info("[Import] %s", result)
-
-
-def do_thumbnail_regen() -> None:
-    """
-    Your thumbnail regeneration logic, moved into a function.
-    Behavior is unchanged: chunked, progress logging, commit per batch.
-    """
-    from pathlib import Path
-    from app import db
-    from app.models import ImageRecord
-    from app.utils.image_importer import generate_thumbnail, generate_video_thumbnail
-
-    total = db.session.query(ImageRecord).count()
-    log.info("[Thumbs] Starting regen; ImageRecord count = %d", total)
-
-    images_root = Path(DEST_DIR).resolve()
-    batch_size = THUMBS_BATCH_SIZE
-
-    processed = 0
-    writes = 0
-    path_updates = 0
-    failures = 0
-    last_id = 0
-    t_start = time.time()
-
-    while True:
-        rows = (
-            ImageRecord.query
-            .filter(ImageRecord.id > last_id)
-            .order_by(ImageRecord.id.asc())
-            .limit(batch_size)
-            .all()
-        )
-        if not rows:
-            break
-
-        batch_start_id = rows[0].id
-        batch_end_id   = rows[-1].id
-        batch_processed = 0
-        batch_writes = 0
-        batch_updates = 0
-        batch_failures = 0
-        bt0 = time.time()
-
-        for img in rows:
-            try:
-                processed += 1
-                batch_processed += 1
-
-                ext = (img.format or "").lower()
-                is_video = ext in ("mp4", "mov") or img.filepath.lower().endswith((".mp4", ".mov"))
-
-                if is_video:
-                    out_path = img.thumb_path
-                    if not out_path:
-                        try:
-                            rel = Path(img.filepath).resolve().relative_to(images_root)
-                        except Exception:
-                            batch_failures += 1
-                            failures += 1
-                            if THUMBS_VERBOSE:
-                                log.info(
-                                    "[Thumbs] SKIP (not under %s): id=%s %s",
-                                    images_root,
-                                    img.id,
-                                    img.filepath,
-                                )
-                            continue
-                        out_path = str((images_root / "thumbs" / rel).with_suffix(".jpg"))
-
-                    new_path = generate_video_thumbnail(img.filepath, out_path)
-                else:
-                    new_path = generate_thumbnail(img.filepath, overwrite=True)
-
-                if new_path:
-                    writes += 1
-                    batch_writes += 1
-
-                if new_path and new_path != img.thumb_path:
-                    img.thumb_path = new_path
-                    path_updates += 1
-                    batch_updates += 1
-
-                if THUMBS_VERBOSE and (processed <= 10 or processed % THUMBS_LOG_EVERY == 0):
-                    kind = "video" if is_video else "image"
-                    log.info(
-                        "[Thumbs] #%d (%s) id=%s name='%s' -> %s",
-                        processed,
-                        kind,
-                        img.id,
-                        img.filename,
-                        new_path or "None",
-                    )
-
-            except Exception as e:
-                failures += 1
-                batch_failures += 1
-                log.warning(
-                    "[Thumbs] Failed for id=%s name='%s': %s",
-                    img.id,
-                    img.filename,
-                    e,
-                )
-
-        db.session.commit()
-        last_id = rows[-1].id
-        db.session.expunge_all()
-
-        bt = time.time() - bt0
-        rate = batch_processed / bt if bt > 0 else 0
-        log.info(
-            "[Thumbs] Batch %s-%s: processed=%d writes=%d db_updates=%d failures=%d (%.1f items/s) "
-            "— totals: processed=%d writes=%d db_updates=%d failures=%d (last_id=%d)",
-            batch_start_id,
-            batch_end_id,
-            batch_processed,
-            batch_writes,
-            batch_updates,
-            batch_failures,
-            rate,
-            processed,
-            writes,
-            path_updates,
-            failures,
-            last_id,
-        )
-
-    elapsed = time.time() - t_start
-    overall_rate = processed / elapsed if elapsed > 0 else 0
-    log.info(
-        "[Thumbs] COMPLETE in %.1fs — processed=%d writes=%d db_updates=%d failures=%d (%.1f items/s).",
-        elapsed,
-        processed,
-        writes,
-        path_updates,
-        failures,
-        overall_rate,
-    )
-
-
-# -------------------------------------------------------------------
-# Orchestration helpers
-# -------------------------------------------------------------------
-def handle_flag_job(
-    job: FlagJob,
-    fn: Callable[[], None],
-    *,
-    dispose_engine_on_operational_error: bool = False,
-) -> bool:
-    """
-    If <flag> exists, claim it and run fn with retries.
-    Returns True if it did anything (including another worker claiming it).
-    """
-    if not os.path.exists(job.flag_path):
-        return False
-
-    log.info("[%s] Flag detected: %s", job.name, job.flag_path)
-
-    if not claim_job(job):
-        # It existed, but someone else may have claimed it.
-        return True
-
-    ok, err_text = run_with_retries(
-        job,
-        fn,
-        dispose_engine_on_operational_error=dispose_engine_on_operational_error,
-    )
-    if ok:
-        complete_job(job)
-    else:
-        fail_job(job, error_text=err_text)
-
-    return True
-
-
-# -------------------------------------------------------------------
-# Main loop
-# -------------------------------------------------------------------
-def monitor_and_import() -> None:
-    thumbnail_job = FlagJob(name="Thumbs", flag_path=THUMBNAIL_FLAG)
-    import_job    = FlagJob(name="Import", flag_path=TRIGGER_FLAG)
-
-    # next time we should run periodic import (monotonic avoids wall-clock jumps)
-    next_periodic_import = time.monotonic() + IMPORT_EVERY_SECONDS
-
-    with app.app_context():
-        log.info(
-            "[Worker] Started. Poll=%ss, periodic import every=%ss (%.2fh).",
-            CHECK_INTERVAL,
-            IMPORT_EVERY_SECONDS,
-            IMPORT_EVERY_SECONDS / 3600.0,
-        )
-
-        while True:
-            did_work = False
-
-            # 1) Thumbnail regen (priority)
-            did_work |= handle_flag_job(thumbnail_job, do_thumbnail_regen)
-
-            # 2) Flag-triggered import
-            did_work |= handle_flag_job(
-                import_job,
-                do_import,
-                dispose_engine_on_operational_error=True,
-            )
-
-            # 3) Periodic import (every IMPORT_EVERY_SECONDS)
-            now = time.monotonic()
-            if IMPORT_EVERY_SECONDS > 0 and now >= next_periodic_import:
-                next_periodic_import = now + IMPORT_EVERY_SECONDS
-
-                # Skip if an import is already in progress or in error
-                if os.path.exists(import_job.run_path):
-                    log.info("[Import] Periodic run skipped (import already running).")
-                elif os.path.exists(import_job.err_path):
-                    log.info(
-                        "[Import] Periodic run skipped (import in error state: %s).",
-                        import_job.err_path,
-                    )
-                else:
-                    # Create a .run lock for periodic run (no trigger.flag required)
-                    tmp = import_job.run_path + ".tmp"
-                    try:
-                        with open(tmp, "w", encoding="utf-8") as f:
-                            f.write("periodic\n")
-                        os.replace(tmp, import_job.run_path)
-                        log.info("[Import] Periodic run claimed -> %s", import_job.run_path)
-
-                        ok, err_text = run_with_retries(
-                            import_job,
-                            do_import,
-                            dispose_engine_on_operational_error=True,
-                        )
-                        if ok:
-                            complete_job(import_job)
-                        else:
-                            fail_job(import_job, error_text=err_text)
-
-                        did_work = True
-                    except Exception as e:
-                        log.warning("[Import] Could not start periodic run: %s", e)
-                        try:
-                            if os.path.exists(tmp):
-                                os.remove(tmp)
-                        except Exception:
-                            pass
-
-            if not did_work:
-                time.sleep(CHECK_INTERVAL)
-
-
-if __name__ == "__main__":
-    monitor_and_import()

migrations/versions/c26012001_add_import_task_batch.py

@@ -0,0 +1,80 @@
+"""Add ImportTask and ImportBatch tables for Celery task queue
+
+Revision ID: c26012001
+Revises: b26011901
+Create Date: 2026-01-20
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c26012001'
+down_revision = 'b26011901'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Create import_batch table
+    op.create_table(
+        'import_batch',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('source_directory', sa.String(length=1024), nullable=False),
+        sa.Column('status', sa.String(length=32), nullable=True),
+        sa.Column('total_files', sa.Integer(), nullable=True),
+        sa.Column('processed_files', sa.Integer(), nullable=True),
+        sa.Column('imported_count', sa.Integer(), nullable=True),
+        sa.Column('skipped_count', sa.Integer(), nullable=True),
+        sa.Column('failed_count', sa.Integer(), nullable=True),
+        sa.Column('started_at', sa.DateTime(), nullable=True),
+        sa.Column('completed_at', sa.DateTime(), nullable=True),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_import_batch_status'), 'import_batch', ['status'], unique=False)
+
+    # Create import_task table
+    op.create_table(
+        'import_task',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('source_path', sa.String(length=1024), nullable=False),
+        sa.Column('file_hash', sa.String(length=64), nullable=True),
+        sa.Column('file_size', sa.BigInteger(), nullable=True),
+        sa.Column('task_type', sa.String(length=32), nullable=False),
+        sa.Column('status', sa.String(length=32), nullable=False),
+        sa.Column('celery_task_id', sa.String(length=64), nullable=True),
+        sa.Column('context', sa.JSON(), nullable=True),
+        sa.Column('batch_id', sa.Integer(), nullable=True),
+        sa.Column('result_image_id', sa.Integer(), nullable=True),
+        sa.Column('error_message', sa.Text(), nullable=True),
+        sa.Column('retry_count', sa.Integer(), nullable=True),
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('started_at', sa.DateTime(), nullable=True),
+        sa.Column('completed_at', sa.DateTime(), nullable=True),
+        sa.ForeignKeyConstraint(['batch_id'], ['import_batch.id'], ondelete='SET NULL'),
+        sa.ForeignKeyConstraint(['result_image_id'], ['image_record.id'], ondelete='SET NULL'),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_import_task_celery_task_id'), 'import_task', ['celery_task_id'], unique=False)
+    op.create_index(op.f('ix_import_task_created_at'), 'import_task', ['created_at'], unique=False)
+    op.create_index(op.f('ix_import_task_file_hash'), 'import_task', ['file_hash'], unique=False)
+    op.create_index(op.f('ix_import_task_source_path'), 'import_task', ['source_path'], unique=False)
+    op.create_index(op.f('ix_import_task_status'), 'import_task', ['status'], unique=False)
+    op.create_index(op.f('ix_import_task_task_type'), 'import_task', ['task_type'], unique=False)
+    op.create_index('ix_import_task_batch_status', 'import_task', ['batch_id', 'status'], unique=False)
+    op.create_index('ix_import_task_status_type', 'import_task', ['status', 'task_type'], unique=False)
+
+
+def downgrade():
+    op.drop_index('ix_import_task_status_type', table_name='import_task')
+    op.drop_index('ix_import_task_batch_status', table_name='import_task')
+    op.drop_index(op.f('ix_import_task_task_type'), table_name='import_task')
+    op.drop_index(op.f('ix_import_task_status'), table_name='import_task')
+    op.drop_index(op.f('ix_import_task_source_path'), table_name='import_task')
+    op.drop_index(op.f('ix_import_task_file_hash'), table_name='import_task')
+    op.drop_index(op.f('ix_import_task_created_at'), table_name='import_task')
+    op.drop_index(op.f('ix_import_task_celery_task_id'), table_name='import_task')
+    op.drop_table('import_task')
+    op.drop_index(op.f('ix_import_batch_status'), table_name='import_batch')
+    op.drop_table('import_batch')

requirements.txt

@@ -8,3 +8,7 @@ imagehash
 pyunpack
 patool
 gunicorn
+
+# Celery task queue
+celery[redis]>=5.3.0
+redis>=5.0.0