tuning job log views and deep scan archive processing.

2026-02-02 18:48:01 -05:00
parent 042a69f9c3
commit 09883960d4
9 changed files with 1078 additions and 103 deletions
@@ -57,8 +57,20 @@ def make_celery(app=None):
        worker_prefetch_multiplier=1,  # One task at a time per worker process

        # Task routing - separate queues for different task types
+        # Scheduler handles: maintenance (periodic tasks) + scan (directory scans)
+        # Worker handles: import, thumbnail, sidecar, default
        task_routes={
-            'app.tasks.scan.*': {'queue': 'scan'},
+            # Scan tasks - handled by scheduler
+            'app.tasks.scan.scan_directory': {'queue': 'scan'},
+            'app.tasks.scan.deep_scan_directory': {'queue': 'scan'},
+
+            # Maintenance tasks - handled by scheduler (periodic/lightweight)
+            'app.tasks.scan.recover_interrupted_tasks': {'queue': 'maintenance'},
+            'app.tasks.scan.cleanup_old_tasks': {'queue': 'maintenance'},
+            'app.tasks.scan.update_system_stats': {'queue': 'maintenance'},
+            'app.tasks.scan.update_batch_stats': {'queue': 'maintenance'},
+
+            # Import tasks - handled by worker (heavy processing)
            'app.tasks.import_file.*': {'queue': 'import'},
            'app.tasks.thumbnail.*': {'queue': 'thumbnail'},
            'app.tasks.sidecar.*': {'queue': 'sidecar'},
@@ -94,6 +106,10 @@ def make_celery(app=None):
                'schedule': 86400,  # Once per day
                'args': (7,),  # Keep tasks for 7 days
            },
+            'update-system-stats': {
+                'task': 'app.tasks.scan.update_system_stats',
+                'schedule': 21600,  # Every 6 hours
+            },
        },
    )

@@ -7,7 +7,7 @@ from sqlalchemy.orm import joinedload, aliased
 from collections import OrderedDict
 from datetime import datetime

-from app.models import ImageRecord, Tag, ArchiveRecord, image_tags, ImportTask, ImportBatch
+from app.models import ImageRecord, Tag, ArchiveRecord, image_tags, ImportTask, ImportBatch, AppSettings
 from app import db
 import os
 import shutil
@@ -1225,6 +1225,91 @@ def import_queue_status():
        return jsonify({'ok': False, 'error': str(e)}), 500


+@main.route('/api/import/tasks')
+def list_import_tasks():
+    """
+    List import tasks with filtering and pagination.
+
+    Query params:
+        status: Filter by status (comma-separated, e.g., 'failed,skipped')
+        task_type: Filter by task type ('import_image', 'import_archive')
+        search: Search in source_path or error_message
+        limit: Max results (default 50, max 200)
+        offset: Skip first N results
+        sort: Sort field ('created_at', 'completed_at', default: 'created_at')
+        order: Sort order ('asc', 'desc', default: 'desc')
+    """
+    try:
+        # Parse parameters
+        status_filter = request.args.get('status', '')
+        task_type = request.args.get('task_type', '')
+        search = request.args.get('search', '')
+        limit = min(int(request.args.get('limit', 50)), 200)
+        offset = int(request.args.get('offset', 0))
+        sort_field = request.args.get('sort', 'created_at')
+        sort_order = request.args.get('order', 'desc')
+
+        # Build query
+        query = ImportTask.query
+
+        # Filter by status
+        if status_filter:
+            statuses = [s.strip() for s in status_filter.split(',') if s.strip()]
+            if statuses:
+                query = query.filter(ImportTask.status.in_(statuses))
+
+        # Filter by task type
+        if task_type:
+            query = query.filter_by(task_type=task_type)
+
+        # Search in source_path or error_message
+        if search:
+            search_pattern = f'%{search}%'
+            query = query.filter(
+                db.or_(
+                    ImportTask.source_path.ilike(search_pattern),
+                    ImportTask.error_message.ilike(search_pattern)
+                )
+            )
+
+        # Get total count before pagination
+        total = query.count()
+
+        # Sort
+        sort_col = getattr(ImportTask, sort_field, ImportTask.created_at)
+        if sort_order == 'asc':
+            query = query.order_by(sort_col.asc())
+        else:
+            query = query.order_by(sort_col.desc())
+
+        # Paginate
+        tasks = query.offset(offset).limit(limit).all()
+
+        return jsonify({
+            'ok': True,
+            'total': total,
+            'offset': offset,
+            'limit': limit,
+            'tasks': [
+                {
+                    'id': t.id,
+                    'source_path': t.source_path,
+                    'filename': os.path.basename(t.source_path) if t.source_path else None,
+                    'task_type': t.task_type,
+                    'status': t.status,
+                    'error_message': t.error_message,  # Full error message
+                    'file_size': t.file_size,
+                    'retry_count': t.retry_count,
+                    'created_at': t.created_at.isoformat() if t.created_at else None,
+                    'completed_at': t.completed_at.isoformat() if t.completed_at else None
+                }
+                for t in tasks
+            ]
+        })
+    except Exception as e:
+        return jsonify({'ok': False, 'error': str(e)}), 500
+
+
@main.route('/api/import/task/<int:task_id>')
 def get_import_task(task_id):
    """
@@ -1308,25 +1393,129 @@ def retry_failed_import_tasks():
@main.route('/api/import/clear-completed', methods=['POST'])
 def clear_completed_tasks():
    """
-    Clear completed and skipped import tasks from the database.
-    Keeps failed tasks for review.
-    """
-    try:
-        deleted = ImportTask.query.filter(
-            ImportTask.status.in_(['complete', 'skipped'])
-        ).delete(synchronize_session=False)
+    Clear import tasks from the database with optional age filtering.

+    POST params:
+        statuses: Comma-separated statuses to clear (default: complete,skipped)
+        older_than_days: Only clear tasks older than N days (default: 0 = all)
+    """
+    from datetime import timedelta
+
+    try:
+        # Parse parameters
+        statuses_param = request.form.get('statuses', 'complete,skipped')
+        older_than_days = int(request.form.get('older_than_days', 0))
+
+        # Validate statuses
+        allowed_statuses = {'complete', 'skipped', 'failed'}
+        statuses = [s.strip() for s in statuses_param.split(',') if s.strip() in allowed_statuses]
+        if not statuses:
+            statuses = ['complete', 'skipped']
+
+        # Build query
+        query = ImportTask.query.filter(ImportTask.status.in_(statuses))
+
+        # Apply age filter if specified
+        if older_than_days > 0:
+            cutoff = datetime.utcnow() - timedelta(days=older_than_days)
+            query = query.filter(ImportTask.completed_at < cutoff)
+
+        deleted = query.delete(synchronize_session=False)
        db.session.commit()

        return jsonify({
            'ok': True,
-            'deleted': deleted
+            'deleted': deleted,
+            'statuses': statuses,
+            'older_than_days': older_than_days
        })
    except Exception as e:
        db.session.rollback()
        return jsonify({'ok': False, 'error': str(e)}), 500


+@main.route('/api/import/task-stats')
+def import_task_stats():
+    """
+    Get detailed task statistics with age breakdown.
+    """
+    from datetime import timedelta
+
+    try:
+        now = datetime.utcnow()
+        today = now - timedelta(days=1)
+        week_ago = now - timedelta(days=7)
+
+        stats = {}
+        for status in ['complete', 'skipped', 'failed']:
+            total = ImportTask.query.filter_by(status=status).count()
+            today_count = ImportTask.query.filter(
+                ImportTask.status == status,
+                ImportTask.completed_at >= today
+            ).count()
+            week_count = ImportTask.query.filter(
+                ImportTask.status == status,
+                ImportTask.completed_at >= week_ago,
+                ImportTask.completed_at < today
+            ).count()
+            older = total - today_count - week_count
+
+            stats[status] = {
+                'total': total,
+                'today': today_count,
+                'this_week': week_count,
+                'older': older
+            }
+
+        return jsonify({'ok': True, 'stats': stats})
+    except Exception as e:
+        return jsonify({'ok': False, 'error': str(e)}), 500
+
+
+@main.route('/api/system/stats')
+def system_stats():
+    """
+    Get system-wide statistics for the dashboard.
+
+    Returns cached stats computed by the update_system_stats Celery task.
+    Stats are updated every 5 minutes by Celery Beat.
+
+    Query params:
+        refresh: If 'true', triggers an immediate stats update (async)
+    """
+    import json
+
+    try:
+        # Check for refresh request
+        if request.args.get('refresh') == 'true':
+            from app.tasks.scan import update_system_stats
+            update_system_stats.delay()
+
+        # Get cached stats from AppSettings
+        setting = AppSettings.query.filter_by(key='system_stats_cache').first()
+
+        if setting and setting.value:
+            stats = json.loads(setting.value)
+            return jsonify({
+                'ok': True,
+                'cached': True,
+                **stats
+            })
+
+        # No cached stats - return empty with flag to indicate first run
+        return jsonify({
+            'ok': True,
+            'cached': False,
+            'images': {'total': 0, 'storage_bytes': 0, 'storage_human': '0 B'},
+            'tags': {'total': 0, 'by_kind': {}},
+            'import_folder': {'pending_files': 0, 'pending_archives': 0},
+            'message': 'Stats not yet computed. Will be available shortly.'
+        })
+
+    except Exception as e:
+        return jsonify({'ok': False, 'error': str(e)}), 500
+
+
@main.route('/api/thumbnails/regenerate', methods=['POST'])
 def regenerate_thumbnails_celery():
    """
@@ -51,6 +51,58 @@ def _get_cached_settings() -> dict:
    return _settings_cache


+def _find_best_sidecar(src_path: str, archive_path: str = None,
+                       archive_sidecar_path: str = None) -> str | None:
+    """
+    Find the best sidecar JSON for a file, checking multiple locations.
+
+    For files extracted from archives, checks:
+    1. Per-file sidecar next to extracted file (temp dir)
+    2. Per-file sidecar next to original archive (by filename match)
+    3. Archive's sidecar as fallback (applies to all files in archive)
+
+    Args:
+        src_path: Path to the source file
+        archive_path: Path to original archive (if extracted from archive)
+        archive_sidecar_path: Path to archive's sidecar JSON
+
+    Returns:
+        Path to the best sidecar JSON found, or None
+    """
+    from app.utils.metadata_enrichment import find_sidecar_json
+    import re
+
+    # First try: per-file sidecar next to the source file
+    sidecar = find_sidecar_json(src_path)
+    if sidecar:
+        return sidecar
+
+    # Second try: per-file sidecar next to original archive
+    if archive_path:
+        from pathlib import Path
+        archive_dir = Path(archive_path).parent
+        filename = Path(src_path).name
+        stem = Path(src_path).stem
+
+        # Check for exact stem match
+        per_file_json = archive_dir / f"{stem}.json"
+        if per_file_json.exists():
+            return str(per_file_json)
+
+        # Check for stem without numeric prefix
+        stem_no_prefix = re.sub(r'^\d+_', '', stem)
+        if stem_no_prefix != stem:
+            alt_json = archive_dir / f"{stem_no_prefix}.json"
+            if alt_json.exists():
+                return str(alt_json)
+
+    # Third try: archive-level sidecar (fallback for all files)
+    if archive_sidecar_path and os.path.exists(archive_sidecar_path):
+        return archive_sidecar_path
+
+    return None
+
+
@celery.task(bind=True, name='app.tasks.import_file.import_media_file',
             max_retries=3, default_retry_delay=60)
 def import_media_file(self, task_id: int):
@@ -88,8 +140,6 @@ def import_media_file(self, task_id: int):
        generate_thumbnail, generate_video_thumbnail_mirrored,
        transcode_video_to_mp4, needs_transcode
    )
-    from app.utils.metadata_enrichment import find_sidecar_json
-
    task = ImportTask.query.get(task_id)
    if not task:
        log.error(f"Task {task_id} not found")
@@ -111,6 +161,10 @@ def import_media_file(self, task_id: int):
        reapply_sidecars = context.get('reapply_sidecars', False)
        verify_hashes = context.get('verify_hashes', False)

+        # Archive sidecar (for files extracted from archives)
+        archive_sidecar_path = context.get('archive_sidecar_path')
+        archive_path = context.get('archive_path')
+
        log.info(f"Processing: {src_path} (deep_scan={is_deep_scan})")

        # Load settings with caching (avoids repeated DB reads)
@@ -166,7 +220,8 @@ def import_media_file(self, task_id: int):

            # In deep scan mode with reapply_sidecars, always try to enrich
            # In quick mode, also try if sidecar exists
-            sidecar_path = find_sidecar_json(src_path)
+            # Uses archive sidecar as fallback for files from archives
+            sidecar_path = _find_best_sidecar(src_path, archive_path, archive_sidecar_path)
            if sidecar_path and (is_deep_scan and reapply_sidecars or not is_deep_scan):
                apply_sidecar_metadata.delay(existing.id, sidecar_path)

@@ -209,7 +264,8 @@ def import_media_file(self, task_id: int):
                            log.info(f"Added archive tag {archive_tag_id} to similar image {similar_id}")

                    # Still try to enrich the existing larger image with sidecar metadata
-                    sidecar_path = find_sidecar_json(src_path)
+                    # Uses archive sidecar as fallback for files from archives
+                    sidecar_path = _find_best_sidecar(src_path, archive_path, archive_sidecar_path)
                    if sidecar_path and similar_id:
                        apply_sidecar_metadata.delay(similar_id, sidecar_path)
                    return _skip_task(task, 'Similar larger image exists', result_image_id=similar_id)
@@ -220,7 +276,8 @@ def import_media_file(self, task_id: int):
                    if old_record:
                        return _supersede_existing(
                            task, old_record, src_path, file_hash, phash,
-                            metadata, artist, dest_dir, filename
+                            metadata, artist, dest_dir, filename,
+                            archive_path, archive_sidecar_path
                        )
        else:
            # Quick mode: only compute pHash for storage, not comparison
@@ -302,7 +359,8 @@ def import_media_file(self, task_id: int):
        db.session.commit()

        # Queue sidecar metadata enrichment
-        sidecar_path = find_sidecar_json(src_path)
+        # Uses archive sidecar as fallback for files from archives
+        sidecar_path = _find_best_sidecar(src_path, archive_path, archive_sidecar_path)
        if sidecar_path:
            apply_sidecar_metadata.delay(record.id, sidecar_path)

@@ -341,11 +399,12 @@ def import_archive(self, task_id: int):

    Steps:
    1. Check if archive already processed (by hash)
-    2. Extract to temp directory
-    3. Create ImportTask for each media file inside
-    4. Queue import tasks for extracted files
-    5. Create archive tag if files were found
-    6. Clean up is deferred until child tasks complete
+    2. Find archive sidecar JSON if present
+    3. Extract to temp directory
+    4. Create ImportTask for each media file inside
+    5. Queue import tasks for extracted files (with archive sidecar path)
+    6. Create archive tag if files were found
+    7. Clean up is deferred until child tasks complete

    Args:
        task_id: ID of the ImportTask record
@@ -354,8 +413,9 @@ def import_archive(self, task_id: int):
        dict with status and counts
    """
    from app.utils.image_importer import (
-        calculate_hash, compute_next_archive_tag_name, extract_archive_resilient, ExtractError
+        calculate_hash, compute_archive_tag_name, extract_archive_resilient, ExtractError
    )
+    from app.utils.metadata_enrichment import find_archive_sidecar

    task = ImportTask.query.get(task_id)
    if not task:
@@ -372,17 +432,31 @@ def import_archive(self, task_id: int):
        artist = context.get('artist', 'unknown')
        dest_dir = context.get('dest_dir', '/images')

-        log.info(f"Processing archive: {archive_path}")
+        # Deep scan options
+        is_deep_scan = context.get('deep_scan', False)
+
+        log.info(f"Processing archive: {archive_path} (deep_scan={is_deep_scan})")
+
+        # Find archive sidecar JSON (next to the archive file, not inside it)
+        # This contains post metadata from Gallery-DL for the entire archive
+        archive_sidecar_path = find_archive_sidecar(archive_path)
+        if archive_sidecar_path:
+            log.info(f"Found archive sidecar: {archive_sidecar_path}")
+        else:
+            log.debug(f"No sidecar found for archive: {archive_path}")

        # Compute archive hash
        archive_hash = calculate_hash(archive_path)
        task.file_hash = archive_hash

        # Check if already processed
-        existing = ArchiveRecord.query.filter_by(hash=archive_hash).first()
-        if existing:
+        existing_archive = ArchiveRecord.query.filter_by(hash=archive_hash).first()
+        if existing_archive and not is_deep_scan:
            return _skip_task(task, 'Archive already imported')

+        if existing_archive and is_deep_scan:
+            log.info(f"Deep scan: re-processing already-imported archive to update metadata")
+
        # Get archive size
        archive_size = os.path.getsize(archive_path)

@@ -415,39 +489,68 @@ def import_archive(self, task_id: int):

        log.info(f"Found {len(media_files)} media files in archive")

-        # Create archive tag
-        tag_name = compute_next_archive_tag_name(artist)
-        archive_tag = Tag.query.filter_by(name=tag_name).first()
-        if not archive_tag:
-            archive_tag = Tag(name=tag_name, kind='archive')
-            db.session.add(archive_tag)
-            db.session.flush()
+        # Get or create archive tag
+        # For deep scan of existing archives, reuse the existing tag
+        if existing_archive and existing_archive.tag_id:
+            archive_tag = Tag.query.get(existing_archive.tag_id)
+            if not archive_tag:
+                # Tag was deleted, recreate it
+                archive_filename = os.path.basename(archive_path)
+                tag_name = compute_archive_tag_name(artist, archive_filename)
+                archive_tag = Tag(name=tag_name, kind='archive')
+                db.session.add(archive_tag)
+                db.session.flush()
+        else:
+            # New archive - create tag using artist:archive_name format
+            archive_filename = os.path.basename(archive_path)
+            tag_name = compute_archive_tag_name(artist, archive_filename)
+            archive_tag = Tag.query.filter_by(name=tag_name).first()
+            if not archive_tag:
+                archive_tag = Tag(name=tag_name, kind='archive')
+                db.session.add(archive_tag)
+                db.session.flush()

-        # Create ArchiveRecord
-        archive_record = ArchiveRecord(
-            filename=archive_path,
-            file_size=archive_size,
-            hash=archive_hash,
-            artist=artist,
-            tag_id=archive_tag.id
-        )
-        db.session.add(archive_record)
-        db.session.commit()
+        # Create ArchiveRecord only if it doesn't exist
+        if not existing_archive:
+            archive_record = ArchiveRecord(
+                filename=archive_path,
+                file_size=archive_size,
+                hash=archive_hash,
+                artist=artist,
+                tag_id=archive_tag.id
+            )
+            db.session.add(archive_record)
+            db.session.commit()
+        else:
+            # Deep scan - commit any tag changes
+            db.session.commit()

        # Queue tasks for each media file
        queued_count = 0
        for media_path in media_files:
+            # Build child context with archive info
+            child_context = {
+                'artist': artist,
+                'dest_dir': dest_dir,
+                'archive_task_id': task.id,
+                'archive_tag_id': archive_tag.id,
+                'temp_dir': tmpdir,
+                'archive_path': archive_path,  # Original archive location (for sidecar lookups)
+            }
+            # Pass archive sidecar path so child tasks can use it as fallback
+            if archive_sidecar_path:
+                child_context['archive_sidecar_path'] = archive_sidecar_path
+
+            # Pass through deep scan flags from parent task
+            for flag in ('deep_scan', 'reapply_sidecars', 'regenerate_thumbnails', 'verify_hashes'):
+                if context.get(flag):
+                    child_context[flag] = context[flag]
+
            child_task = ImportTask(
                source_path=media_path,
                task_type='import_image',
                batch_id=task.batch_id,
-                context={
-                    'artist': artist,
-                    'dest_dir': dest_dir,
-                    'archive_task_id': task.id,
-                    'archive_tag_id': archive_tag.id,
-                    'temp_dir': tmpdir
-                },
+                context=child_context,
                status='pending'
            )
            db.session.add(child_task)
@@ -463,6 +566,8 @@ def import_archive(self, task_id: int):
        task.context['temp_dir'] = tmpdir
        task.context['archive_tag_id'] = archive_tag.id
        task.context['child_count'] = queued_count
+        if archive_sidecar_path:
+            task.context['archive_sidecar_path'] = archive_sidecar_path

        task.status = 'complete'
        task.completed_at = datetime.utcnow()
@@ -585,7 +690,9 @@ def _regenerate_thumbnail(record: ImageRecord) -> bool:

 def _supersede_existing(task: ImportTask, old_record: ImageRecord, src_path: str,
                        file_hash: str, phash, metadata: dict, artist: str,
-                        dest_dir: str, filename: str) -> dict:
+                        dest_dir: str, filename: str,
+                        archive_path: str = None,
+                        archive_sidecar_path: str = None) -> dict:
    """
    Replace an existing smaller image with a larger version.

@@ -595,14 +702,15 @@ def _supersede_existing(task: ImportTask, old_record: ImageRecord, src_path: str
        build_hashed_dest_path, generate_thumbnail, generate_video_thumbnail_mirrored,
        supersede_image
    )
-    from app.utils.metadata_enrichment import find_sidecar_json, enrich_from_sidecar
+    from app.utils.metadata_enrichment import enrich_from_sidecar
    from app.tasks.sidecar import apply_sidecar_metadata

    log.info(f"Superseding {old_record.filename} ({old_record.width}x{old_record.height}) "
             f"with {filename} ({metadata['width']}x{metadata['height']})")

    # Enrich metadata from sidecar if available
-    sidecar_path = find_sidecar_json(src_path)
+    # Uses archive sidecar as fallback for files from archives
+    sidecar_path = _find_best_sidecar(src_path, archive_path, archive_sidecar_path)
    if sidecar_path:
        enriched = enrich_from_sidecar(sidecar_path, {'taken_at': metadata.get('taken_at')})
        if enriched.get('taken_at'):
@@ -633,3 +633,105 @@ def update_batch_stats(batch_id: int):
                 f"{batch.skipped_count} skipped, {batch.failed_count} failed")

    db.session.commit()
+
+
+@celery.task(name='app.tasks.scan.update_system_stats')
+def update_system_stats():
+    """
+    Compute and cache system-wide statistics for the dashboard.
+
+    Called periodically by Celery Beat to avoid expensive queries on page load.
+    Stores results in AppSettings as JSON for quick retrieval.
+
+    Stats computed:
+    - Total images and storage usage
+    - Tag counts by kind
+    - Pending files/archives in import folder
+
+    Returns:
+        dict with computed stats
+    """
+    import json
+    from sqlalchemy import func
+    from app.models import ImageRecord, Tag, AppSettings
+
+    log.info("Updating system stats cache...")
+
+    try:
+        # Database stats
+        total_images = ImageRecord.query.count()
+        total_tags = Tag.query.count()
+
+        # Tag kind breakdown
+        tag_kinds = dict(
+            db.session.query(Tag.kind, func.count(Tag.id))
+            .group_by(Tag.kind).all()
+        )
+
+        # Storage stats (sum of file_size column)
+        storage_result = db.session.query(func.sum(ImageRecord.file_size)).scalar()
+        total_storage_bytes = storage_result or 0
+
+        # Import folder stats
+        import_dir = '/import'
+        pending_files = 0
+        pending_archives = 0
+        if os.path.exists(import_dir):
+            for artist_dir in os.listdir(import_dir):
+                artist_path = os.path.join(import_dir, artist_dir)
+                if not os.path.isdir(artist_path):
+                    continue
+                for root, _, files in os.walk(artist_path):
+                    for f in files:
+                        lower = f.lower()
+                        if lower.endswith(ALLOWED_MEDIA_EXTS):
+                            pending_files += 1
+                        elif lower.endswith(ALLOWED_ARCHIVE_EXTS):
+                            pending_archives += 1
+
+        # Format storage for display
+        storage_human = _format_bytes(total_storage_bytes)
+
+        # Build stats dict
+        stats = {
+            'images': {
+                'total': total_images,
+                'storage_bytes': total_storage_bytes,
+                'storage_human': storage_human
+            },
+            'tags': {
+                'total': total_tags,
+                'by_kind': tag_kinds
+            },
+            'import_folder': {
+                'pending_files': pending_files,
+                'pending_archives': pending_archives
+            },
+            'updated_at': datetime.utcnow().isoformat()
+        }
+
+        # Store in AppSettings
+        setting = AppSettings.query.filter_by(key='system_stats_cache').first()
+        if not setting:
+            setting = AppSettings(key='system_stats_cache')
+            db.session.add(setting)
+        setting.value = json.dumps(stats)
+        db.session.commit()
+
+        log.info(f"System stats updated: {total_images} images, {total_tags} tags, "
+                 f"{storage_human} storage, {pending_files + pending_archives} pending")
+
+        return stats
+
+    except Exception as e:
+        log.error(f"Failed to update system stats: {e}", exc_info=True)
+        raise
+
+
+def _format_bytes(size_bytes: int) -> str:
+    """Format bytes to human readable string."""
+    for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
+        if size_bytes < 1024:
+            return f"{size_bytes:.1f} {unit}"
+        size_bytes /= 1024
+    return f"{size_bytes:.1f} PB"
@@ -4,7 +4,44 @@
 {% block content %}
 <h1 style="text-align:center; margin-top: 1rem;">Settings</h1>

-<!-- Import Queue Status (Full Width at Top) -->
+<!-- System Stats (Full Width at Top) -->
+<div class="settings-grid-full">
+  <div class="settings-container">
+    <div class="settings-section">
+      <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 0.5rem;">
+        <h2 style="margin: 0;">System Overview</h2>
+        <div style="display: flex; align-items: center; gap: 0.75rem;">
+          <span id="statsUpdatedAt" style="font-size: 0.75rem; color: var(--text-muted);"></span>
+          <button id="refreshStatsBtn" class="btn secondary-btn" style="padding: 0.3rem 0.6rem; font-size: 0.8rem;" title="Refresh stats">Refresh</button>
+        </div>
+      </div>
+      <div class="stats-grid" style="grid-template-columns: repeat(5, 1fr);">
+        <div class="stat-item">
+          <span class="stat-value" id="statTotalImages">-</span>
+          <span class="stat-label">Total Images</span>
+        </div>
+        <div class="stat-item">
+          <span class="stat-value" id="statTotalTags">-</span>
+          <span class="stat-label">Total Tags</span>
+        </div>
+        <div class="stat-item">
+          <span class="stat-value" id="statStorage">-</span>
+          <span class="stat-label">Storage Used</span>
+        </div>
+        <div class="stat-item">
+          <span class="stat-value" id="statPendingFiles">-</span>
+          <span class="stat-label">Files to Import</span>
+        </div>
+        <div class="stat-item">
+          <span class="stat-value" id="statPendingArchives">-</span>
+          <span class="stat-label">Archives to Import</span>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+<!-- Import Queue Status (Full Width) -->
 <div class="settings-grid-full">
  <div class="settings-container">
    <div class="settings-section">
@@ -25,20 +62,28 @@
            <span class="stat-value" id="queueProcessing">-</span>
            <span class="stat-label">Processing</span>
          </div>
-          <div class="stat-item">
+          <div class="stat-item clickable-stat" id="completeStatItem" title="Click for breakdown">
            <span class="stat-value" id="queueComplete">-</span>
            <span class="stat-label">Complete</span>
          </div>
-          <div class="stat-item">
+          <div class="stat-item clickable-stat" id="skippedStatItem" title="Click for breakdown">
            <span class="stat-value" id="queueSkipped">-</span>
            <span class="stat-label">Skipped</span>
          </div>
-          <div class="stat-item">
+          <div class="stat-item clickable-stat" id="failedStatItem" title="Click for breakdown">
            <span class="stat-value" id="queueFailed">-</span>
            <span class="stat-label">Failed</span>
          </div>
        </div>

+        <!-- Age breakdown tooltip -->
+        <div id="ageBreakdown" style="display: none; margin-top: 0.5rem; padding: 0.5rem; background: rgba(255,255,255,0.05); border-radius: 4px; font-size: 0.8rem;">
+          <span id="ageBreakdownTitle" style="color: var(--text-muted);"></span>:
+          <span id="ageBreakdownToday">-</span> today,
+          <span id="ageBreakdownWeek">-</span> this week,
+          <span id="ageBreakdownOlder">-</span> older
+        </div>
+
        <div id="activeBatchInfo" class="import-stats" style="margin-top: 1rem; display: none;">
          <h3>Active Batch</h3>
          <p>Processing <strong id="batchDir">-</strong></p>
@@ -51,9 +96,36 @@
        </div>

        <div class="queue-actions" style="margin-top: 1rem; display: flex; gap: 0.75rem; flex-wrap: wrap;">
-          <button id="triggerImportBtn" class="btn primary-btn">Trigger Import Scan</button>
+          <button id="triggerImportBtn" class="btn primary-btn">Quick Scan</button>
+          <button id="triggerDeepScanBtn" class="btn warning-btn" title="Full reprocessing: re-extracts metadata, runs pHash comparison, merges archive tags">Deep Scan</button>
          <button id="retryFailedBtn" class="btn warning-btn">Retry Failed Tasks</button>
-          <button id="clearCompletedBtn" class="btn secondary-btn">Clear Completed</button>
+        </div>
+
+        <!-- Task Cleanup Controls -->
+        <div class="cleanup-controls" style="margin-top: 1rem; padding: 0.75rem; background: rgba(255,255,255,0.03); border-radius: 6px;">
+          <div style="display: flex; align-items: center; gap: 0.75rem; flex-wrap: wrap;">
+            <span style="color: var(--text-muted); font-size: 0.85rem;">Clear tasks:</span>
+            <select id="clearStatusSelect" class="form-input" style="width: auto; min-width: 140px;">
+              <option value="complete,skipped">Complete + Skipped</option>
+              <option value="complete">Complete only</option>
+              <option value="skipped">Skipped only</option>
+              <option value="failed">Failed only</option>
+              <option value="complete,skipped,failed">All finished</option>
+            </select>
+            <select id="clearAgeSelect" class="form-input" style="width: auto; min-width: 120px;">
+              <option value="0">All</option>
+              <option value="1">Older than 1 day</option>
+              <option value="7" selected>Older than 7 days</option>
+              <option value="30">Older than 30 days</option>
+            </select>
+            <button id="clearTasksBtn" class="btn secondary-btn">Clear</button>
+          </div>
+          <p style="color: var(--text-muted); font-size: 0.75rem; margin-top: 0.5rem;">
+            Note: Failed/skipped tasks older than 7 days are auto-cleaned daily.
+          </p>
+        </div>
+
+        <div class="queue-actions" style="margin-top: 0.75rem; display: flex; gap: 0.75rem; flex-wrap: wrap;">
          <button id="regenMissingThumbsBtn" class="btn secondary-btn">Regenerate Missing Thumbnails</button>
          <button id="reapplyArtistTagsBtn" class="btn secondary-btn">Reapply Artist Tags from Paths</button>
          <button id="cleanupOrphanedTagsBtn" class="btn secondary-btn">Cleanup Orphaned Tags</button>
@@ -67,25 +139,76 @@
        </div>
      </div>

-      <!-- Recent Tasks Table -->
-      <div id="recentTasksSection" style="margin-top: 1.5rem;">
-        <h3>Recent Tasks</h3>
-        <div class="tasks-table-wrapper" style="max-height: 300px; overflow-y: auto;">
+      <!-- Task List with Filters -->
+      <div id="taskListSection" style="margin-top: 1.5rem;">
+        <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 0.75rem; flex-wrap: wrap; gap: 0.5rem;">
+          <h3 style="margin: 0;">Import Tasks</h3>
+          <div style="display: flex; gap: 0.5rem; align-items: center; flex-wrap: wrap;">
+            <select id="taskStatusFilter" class="form-input" style="width: auto; min-width: 120px; padding: 0.4rem;">
+              <option value="">All statuses</option>
+              <option value="failed">Failed</option>
+              <option value="skipped">Skipped</option>
+              <option value="complete">Complete</option>
+              <option value="processing">Processing</option>
+              <option value="queued">Queued</option>
+              <option value="pending">Pending</option>
+            </select>
+            <input type="text" id="taskSearchInput" class="form-input" placeholder="Search..." style="width: 150px; padding: 0.4rem;">
+            <button id="taskSearchBtn" class="btn secondary-btn" style="padding: 0.4rem 0.75rem;">Search</button>
+          </div>
+        </div>
+
+        <div id="taskListInfo" style="font-size: 0.8rem; color: var(--text-muted); margin-bottom: 0.5rem;">
+          Showing <span id="taskListShowing">0</span> of <span id="taskListTotal">0</span> tasks
+        </div>
+
+        <div class="tasks-table-wrapper" style="max-height: 400px; overflow-y: auto;">
          <table class="tasks-table">
            <thead>
              <tr>
-                <th>File</th>
-                <th>Type</th>
-                <th>Status</th>
-                <th>Error</th>
-                <th>Time</th>
+                <th style="width: 40%;">File</th>
+                <th style="width: 10%;">Type</th>
+                <th style="width: 10%;">Status</th>
+                <th style="width: 30%;">Error/Reason</th>
+                <th style="width: 10%;">Time</th>
              </tr>
            </thead>
-            <tbody id="recentTasksBody">
+            <tbody id="taskListBody">
              <tr><td colspan="5" style="text-align: center; color: var(--text-muted);">Loading...</td></tr>
            </tbody>
          </table>
        </div>
+
+        <div style="margin-top: 0.75rem; display: flex; gap: 0.5rem;">
+          <button id="loadMoreTasksBtn" class="btn secondary-btn" style="display: none;">Load More</button>
+        </div>
+      </div>
+
+      <!-- Error Detail Modal -->
+      <div id="errorDetailModal" class="modal" style="display: none;">
+        <div class="modal-content" style="max-width: 600px;">
+          <div class="modal-header">
+            <h3>Task Details</h3>
+            <button class="modal-close" onclick="document.getElementById('errorDetailModal').style.display='none'">&times;</button>
+          </div>
+          <div class="modal-body">
+            <div style="margin-bottom: 1rem;">
+              <strong>File:</strong>
+              <div id="errorDetailPath" style="word-break: break-all; font-family: monospace; font-size: 0.85rem; background: rgba(0,0,0,0.2); padding: 0.5rem; border-radius: 4px; margin-top: 0.25rem;"></div>
+            </div>
+            <div style="margin-bottom: 1rem;">
+              <strong>Status:</strong> <span id="errorDetailStatus"></span>
+            </div>
+            <div style="margin-bottom: 1rem;">
+              <strong>Error/Reason:</strong>
+              <div id="errorDetailMessage" style="word-break: break-word; font-family: monospace; font-size: 0.85rem; background: rgba(255,100,100,0.1); padding: 0.5rem; border-radius: 4px; margin-top: 0.25rem; white-space: pre-wrap;"></div>
+            </div>
+            <div style="display: flex; gap: 1rem; font-size: 0.85rem; color: var(--text-muted);">
+              <span><strong>Retries:</strong> <span id="errorDetailRetries"></span></span>
+              <span><strong>Created:</strong> <span id="errorDetailCreated"></span></span>
+            </div>
+          </div>
+        </div>
      </div>
    </div>
  </div>
@@ -282,6 +405,18 @@
 </div>

 <style>
+  .clickable-stat {
+    cursor: pointer;
+    transition: background 0.2s;
+    border-radius: 4px;
+  }
+  .clickable-stat:hover {
+    background: rgba(255, 255, 255, 0.08);
+  }
+  .cleanup-controls select {
+    padding: 0.4rem 0.6rem;
+    font-size: 0.85rem;
+  }
  .tasks-table {
    width: 100%;
    border-collapse: collapse;
@@ -560,6 +695,55 @@
    background: var(--btn-danger);
    color: #fff;
  }
+  /* Error detail modal */
+  #errorDetailModal {
+    position: fixed;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100%;
+    background: rgba(0, 0, 0, 0.7);
+    z-index: 1000;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+  }
+  #errorDetailModal .modal-content {
+    background: var(--surface-elevated);
+    border-radius: 8px;
+    max-height: 80vh;
+    overflow-y: auto;
+  }
+  #errorDetailModal .modal-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    padding: 1rem;
+    border-bottom: 1px solid rgba(255,255,255,0.1);
+  }
+  #errorDetailModal .modal-header h3 {
+    margin: 0;
+  }
+  #errorDetailModal .modal-close {
+    background: none;
+    border: none;
+    font-size: 1.5rem;
+    cursor: pointer;
+    color: var(--text-muted);
+  }
+  #errorDetailModal .modal-body {
+    padding: 1rem;
+  }
+  .tasks-table .error-cell {
+    max-width: 250px;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+    cursor: pointer;
+  }
+  .tasks-table .error-cell:hover {
+    text-decoration: underline;
+  }
 </style>

 <script>
@@ -595,27 +779,121 @@ document.addEventListener('DOMContentLoaded', () => {
          batchEl.style.display = 'none';
        }

-        // Update recent tasks table
-        const tbody = document.getElementById('recentTasksBody');
-        if (data.recent_tasks && data.recent_tasks.length > 0) {
-          tbody.innerHTML = data.recent_tasks.map(t => `
-            <tr>
-              <td title="${t.source_path || ''}">${t.source_path || '-'}</td>
-              <td>${t.task_type || '-'}</td>
-              <td class="status-${t.status}">${t.status || '-'}</td>
-              <td title="${t.error_message || ''}">${t.error_message ? t.error_message.substring(0, 30) + '...' : '-'}</td>
-              <td>${t.created_at ? new Date(t.created_at).toLocaleString() : '-'}</td>
-            </tr>
-          `).join('');
-        } else {
-          tbody.innerHTML = '<tr><td colspan="5" style="text-align: center; color: var(--text-muted);">No recent tasks</td></tr>';
-        }
      }
    } catch (e) {
      console.error('Failed to load queue status:', e);
    }
  }

+  // ========================================
+  // Task List with Filtering
+  // ========================================
+  let taskListOffset = 0;
+  const taskListLimit = 50;
+  let taskListTasks = [];
+
+  async function loadTaskList(append = false) {
+    const status = document.getElementById('taskStatusFilter').value;
+    const search = document.getElementById('taskSearchInput').value;
+
+    if (!append) {
+      taskListOffset = 0;
+      taskListTasks = [];
+    }
+
+    try {
+      const params = new URLSearchParams({
+        limit: taskListLimit,
+        offset: taskListOffset,
+        order: 'desc'
+      });
+      if (status) params.append('status', status);
+      if (search) params.append('search', search);
+
+      const r = await fetch('/api/import/tasks?' + params.toString());
+      const data = await r.json();
+
+      if (data.ok) {
+        taskListTasks = append ? [...taskListTasks, ...data.tasks] : data.tasks;
+
+        const tbody = document.getElementById('taskListBody');
+        if (taskListTasks.length > 0) {
+          tbody.innerHTML = taskListTasks.map(t => `
+            <tr>
+              <td title="${escapeHtml(t.source_path || '')}">${escapeHtml(t.filename || '-')}</td>
+              <td>${t.task_type === 'import_image' ? 'Image' : t.task_type === 'import_archive' ? 'Archive' : t.task_type || '-'}</td>
+              <td class="status-${t.status}">${t.status || '-'}</td>
+              <td class="error-cell" onclick="window.showErrorDetail(${t.id})" title="${t.error_message ? escapeHtml(t.error_message) : 'Click for details'}">${t.error_message ? escapeHtml(t.error_message.substring(0, 50)) + (t.error_message.length > 50 ? '...' : '') : '-'}</td>
+              <td>${t.completed_at ? new Date(t.completed_at).toLocaleString() : (t.created_at ? new Date(t.created_at).toLocaleString() : '-')}</td>
+            </tr>
+          `).join('');
+        } else {
+          tbody.innerHTML = '<tr><td colspan="5" style="text-align: center; color: var(--text-muted);">No tasks found</td></tr>';
+        }
+
+        // Update info
+        document.getElementById('taskListShowing').textContent = taskListTasks.length;
+        document.getElementById('taskListTotal').textContent = data.total;
+
+        // Show/hide load more button
+        const loadMoreBtn = document.getElementById('loadMoreTasksBtn');
+        if (taskListTasks.length < data.total) {
+          loadMoreBtn.style.display = 'inline-block';
+        } else {
+          loadMoreBtn.style.display = 'none';
+        }
+      }
+    } catch (e) {
+      console.error('Failed to load task list:', e);
+    }
+  }
+
+  function escapeHtml(text) {
+    if (!text) return '';
+    const div = document.createElement('div');
+    div.textContent = text;
+    return div.innerHTML;
+  }
+
+  // Store tasks for detail view
+  let taskDetailCache = {};
+
+  // Make showErrorDetail globally accessible for onclick handlers
+  window.showErrorDetail = function(taskId) {
+    const task = taskListTasks.find(t => t.id === taskId);
+    if (!task) return;
+
+    document.getElementById('errorDetailPath').textContent = task.source_path || '-';
+    document.getElementById('errorDetailStatus').textContent = task.status || '-';
+    document.getElementById('errorDetailStatus').className = 'status-' + task.status;
+    document.getElementById('errorDetailMessage').textContent = task.error_message || 'No error message';
+    document.getElementById('errorDetailRetries').textContent = task.retry_count || 0;
+    document.getElementById('errorDetailCreated').textContent = task.created_at ? new Date(task.created_at).toLocaleString() : '-';
+
+    document.getElementById('errorDetailModal').style.display = 'flex';
+  };
+
+  // Close modal on backdrop click
+  document.getElementById('errorDetailModal').addEventListener('click', (e) => {
+    if (e.target.id === 'errorDetailModal') {
+      e.target.style.display = 'none';
+    }
+  });
+
+  // Task list event listeners
+  document.getElementById('taskStatusFilter').addEventListener('change', () => loadTaskList());
+  document.getElementById('taskSearchBtn').addEventListener('click', () => loadTaskList());
+  document.getElementById('taskSearchInput').addEventListener('keypress', (e) => {
+    if (e.key === 'Enter') loadTaskList();
+  });
+  document.getElementById('loadMoreTasksBtn').addEventListener('click', () => {
+    taskListOffset += taskListLimit;
+    loadTaskList(true);
+  });
+
+  // Initial load
+  loadTaskList();
+
  async function loadCeleryStatus() {
    try {
      const r = await fetch('/api/celery/status');
@@ -634,13 +912,106 @@ document.addEventListener('DOMContentLoaded', () => {
    }
  }

+  // System stats
+  let taskAgeStats = {};
+
+  async function loadSystemStats(triggerRefresh = false) {
+    try {
+      const url = triggerRefresh ? '/api/system/stats?refresh=true' : '/api/system/stats';
+      const r = await fetch(url);
+      const data = await r.json();
+
+      if (data.ok) {
+        document.getElementById('statTotalImages').textContent = data.images.total.toLocaleString();
+        document.getElementById('statTotalTags').textContent = data.tags.total.toLocaleString();
+        document.getElementById('statStorage').textContent = data.images.storage_human;
+        document.getElementById('statPendingFiles').textContent = data.import_folder.pending_files.toLocaleString();
+        document.getElementById('statPendingArchives').textContent = data.import_folder.pending_archives.toLocaleString();
+
+        // Show last updated time
+        const updatedEl = document.getElementById('statsUpdatedAt');
+        if (data.updated_at) {
+          const updated = new Date(data.updated_at);
+          const now = new Date();
+          const diffMs = now - updated;
+          const diffMins = Math.floor(diffMs / 60000);
+
+          if (diffMins < 1) {
+            updatedEl.textContent = 'Updated just now';
+          } else if (diffMins < 60) {
+            updatedEl.textContent = `Updated ${diffMins}m ago`;
+          } else {
+            const diffHours = Math.floor(diffMins / 60);
+            updatedEl.textContent = `Updated ${diffHours}h ago`;
+          }
+        } else if (data.message) {
+          updatedEl.textContent = data.message;
+        } else {
+          updatedEl.textContent = '';
+        }
+      }
+    } catch (e) {
+      console.error('Failed to load system stats:', e);
+    }
+  }
+
+  // Refresh stats button handler
+  document.getElementById('refreshStatsBtn').addEventListener('click', async () => {
+    const btn = document.getElementById('refreshStatsBtn');
+    btn.disabled = true;
+    btn.textContent = 'Refreshing...';
+
+    // Trigger refresh and reload after a short delay
+    await loadSystemStats(true);
+    await new Promise(r => setTimeout(r, 2000)); // Wait for task to run
+    await loadSystemStats();
+
+    btn.disabled = false;
+    btn.textContent = 'Refresh';
+  });
+
+  async function loadTaskAgeStats() {
+    try {
+      const r = await fetch('/api/import/task-stats');
+      const data = await r.json();
+
+      if (data.ok) {
+        taskAgeStats = data.stats;
+      }
+    } catch (e) {
+      console.error('Failed to load task age stats:', e);
+    }
+  }
+
+  function showAgeBreakdown(status) {
+    const stats = taskAgeStats[status];
+    if (!stats) return;
+
+    const breakdown = document.getElementById('ageBreakdown');
+    const title = status.charAt(0).toUpperCase() + status.slice(1);
+    document.getElementById('ageBreakdownTitle').textContent = title;
+    document.getElementById('ageBreakdownToday').textContent = stats.today;
+    document.getElementById('ageBreakdownWeek').textContent = stats.this_week;
+    document.getElementById('ageBreakdownOlder').textContent = stats.older;
+    breakdown.style.display = 'block';
+  }
+
+  // Add click handlers for stat items
+  document.getElementById('completeStatItem').addEventListener('click', () => showAgeBreakdown('complete'));
+  document.getElementById('skippedStatItem').addEventListener('click', () => showAgeBreakdown('skipped'));
+  document.getElementById('failedStatItem').addEventListener('click', () => showAgeBreakdown('failed'));
+
  // Load queue status on page load and refresh periodically
  loadQueueStatus();
  loadCeleryStatus();
+  loadSystemStats();
+  loadTaskAgeStats();
  setInterval(loadQueueStatus, 5000);  // Refresh every 5 seconds
  setInterval(loadCeleryStatus, 10000);  // Refresh every 10 seconds
+  setInterval(loadSystemStats, 30000);  // Refresh every 30 seconds
+  setInterval(loadTaskAgeStats, 30000);  // Refresh every 30 seconds

-  // Trigger Import Scan
+  // Quick Scan (default)
  document.getElementById('triggerImportBtn').addEventListener('click', async () => {
    const btn = document.getElementById('triggerImportBtn');
    btn.disabled = true;
@@ -651,7 +1022,7 @@ document.addEventListener('DOMContentLoaded', () => {
      const data = await r.json();

      if (data.ok) {
-        alert('Import scan started! Task ID: ' + data.task_id);
+        alert('Quick scan started! Task ID: ' + data.task_id);
        loadQueueStatus();
      } else {
        alert('Failed to start import: ' + (data.error || 'Unknown error'));
@@ -660,7 +1031,38 @@ document.addEventListener('DOMContentLoaded', () => {
      alert('Failed to start import: ' + e.message);
    } finally {
      btn.disabled = false;
-      btn.textContent = 'Trigger Import Scan';
+      btn.textContent = 'Quick Scan';
+    }
+  });
+
+  // Deep Scan (full reprocessing)
+  document.getElementById('triggerDeepScanBtn').addEventListener('click', async () => {
+    if (!confirm('Deep scan will reprocess ALL files in the import folder.\n\nThis includes:\n- Full pHash similarity detection\n- Archive tag merging for duplicates\n- Thumbnail regeneration\n- Sidecar metadata re-application\n\nThis may take a long time. Continue?')) {
+      return;
+    }
+
+    const btn = document.getElementById('triggerDeepScanBtn');
+    btn.disabled = true;
+    btn.textContent = 'Starting...';
+
+    try {
+      const formData = new FormData();
+      formData.append('deep', 'true');
+
+      const r = await fetch('/api/import/trigger', { method: 'POST', body: formData });
+      const data = await r.json();
+
+      if (data.ok) {
+        alert('Deep scan started! Task ID: ' + data.task_id + '\n\nThis will reprocess all files and may take a while.');
+        loadQueueStatus();
+      } else {
+        alert('Failed to start deep scan: ' + (data.error || 'Unknown error'));
+      }
+    } catch (e) {
+      alert('Failed to start deep scan: ' + e.message);
+    } finally {
+      btn.disabled = false;
+      btn.textContent = 'Deep Scan';
    }
  });

@@ -688,19 +1090,34 @@ document.addEventListener('DOMContentLoaded', () => {
    }
  });

-  // Clear Completed Tasks
-  document.getElementById('clearCompletedBtn').addEventListener('click', async () => {
-    const btn = document.getElementById('clearCompletedBtn');
+  // Clear Tasks with age/status options
+  document.getElementById('clearTasksBtn').addEventListener('click', async () => {
+    const btn = document.getElementById('clearTasksBtn');
+    const statuses = document.getElementById('clearStatusSelect').value;
+    const olderThanDays = document.getElementById('clearAgeSelect').value;
+
+    const statusLabel = document.getElementById('clearStatusSelect').selectedOptions[0].text;
+    const ageLabel = document.getElementById('clearAgeSelect').selectedOptions[0].text;
+
+    if (!confirm(`Clear ${statusLabel} tasks (${ageLabel.toLowerCase()})?`)) {
+      return;
+    }
+
    btn.disabled = true;
    btn.textContent = 'Clearing...';

    try {
-      const r = await fetch('/api/import/clear-completed', { method: 'POST' });
+      const formData = new FormData();
+      formData.append('statuses', statuses);
+      formData.append('older_than_days', olderThanDays);
+
+      const r = await fetch('/api/import/clear-completed', { method: 'POST', body: formData });
      const data = await r.json();

      if (data.ok) {
-        alert(`Cleared ${data.deleted} completed/skipped task(s).`);
+        alert(`Cleared ${data.deleted} task(s).`);
        loadQueueStatus();
+        loadTaskAgeStats();
      } else {
        alert('Failed to clear: ' + (data.error || 'Unknown error'));
      }
@@ -708,7 +1125,7 @@ document.addEventListener('DOMContentLoaded', () => {
      alert('Failed to clear: ' + e.message);
    } finally {
      btn.disabled = false;
-      btn.textContent = 'Clear Completed';
+      btn.textContent = 'Clear';
    }
  });

@@ -846,8 +846,69 @@ def is_first_volume(path: str) -> bool:

    return True

+def extract_archive_name(filename: str) -> str:
+    """
+    Extract a clean archive name from a filename by removing common prefixes.
+
+    Handles patterns like:
+    - "43387617_attachment_83109081_October 2020 Rewards.rar" -> "October 2020 Rewards"
+    - "01_Archive Name.zip" -> "Archive Name"
+    - "Archive Name.rar" -> "Archive Name"
+
+    Returns the cleaned name without extension.
+    """
+    import re
+    from pathlib import Path
+
+    # Remove extension
+    stem = Path(filename).stem
+
+    # Pattern 1: Gallery-DL style "{id}_attachment_{id}_{name}" or "{id}_{name}"
+    # Match one or more numeric_prefix patterns at the start
+    cleaned = re.sub(r'^(\d+_)+(attachment_)?(\d+_)?', '', stem)
+
+    # Pattern 2: Simple numeric prefix "01_name" or "001_name"
+    if cleaned == stem:  # No match from pattern 1
+        cleaned = re.sub(r'^\d+_', '', stem)
+
+    # If we stripped everything or got empty, fall back to original stem
+    if not cleaned.strip():
+        cleaned = stem
+
+    return cleaned.strip()
+
+
+def compute_archive_tag_name(artist: str, archive_filename: str) -> str:
+    """
+    Generate an archive tag name in the format: '{artist}:{archive_name}'
+
+    Examples:
+    - artist="InCaseArt", filename="43387617_attachment_83109081_October 2020 Rewards.rar"
+      -> "InCaseArt:October 2020 Rewards"
+
+    If a tag with the same name already exists, appends a numeric suffix.
+    """
+    archive_name = extract_archive_name(archive_filename)
+    base_tag = f"{artist}:{archive_name}"
+
+    # Check if tag already exists
+    existing = Tag.query.filter_by(name=base_tag, kind='archive').first()
+    if not existing:
+        return base_tag
+
+    # Tag exists, add numeric suffix
+    n = 2
+    while True:
+        candidate = f"{base_tag} ({n})"
+        if not Tag.query.filter_by(name=candidate, kind='archive').first():
+            return candidate
+        n += 1
+
+
 def compute_next_archive_tag_name(artist: str, width: int = ARCHIVE_NUM_WIDTH) -> str:
    """
+    DEPRECATED: Use compute_archive_tag_name() instead.
+
    Return a unique incremental tag name like: 'archive:{artist}/0001', '0002', ...
    - Looks only at Tag(kind='archive') with names starting 'archive:{artist}/'
    - Finds max numeric suffix and returns next number (zero-padded).
@@ -83,6 +83,72 @@ def find_sidecar_json(image_path: str) -> Optional[str]:
    return None


+def find_archive_sidecar(archive_path: str) -> Optional[str]:
+    """
+    Find the Gallery-DL sidecar JSON file for an archive.
+
+    Gallery-DL names sidecars based on the archive's content filename,
+    which may differ from the archive filename due to prefixes added
+    during download (e.g., "02_HolyMeh July 2023.rar" -> "HolyMeh July 2023.json").
+
+    Args:
+        archive_path: Path to the archive file (e.g., "02_Filename.rar")
+
+    Returns:
+        Path to the sidecar JSON if found, None otherwise.
+    """
+    archive_path = Path(archive_path)
+    parent_dir = archive_path.parent
+    stem = archive_path.stem  # filename without extension
+
+    # Pattern 1: Exact stem match (most common case)
+    # e.g., "HolyMeh July 2023.rar" -> "HolyMeh July 2023.json"
+    stem_json = parent_dir / f"{stem}.json"
+    if stem_json.exists():
+        return str(stem_json)
+
+    # Pattern 2: Strip numeric prefix from archive name
+    # e.g., "02_HolyMeh July 2023.rar" -> "HolyMeh July 2023.json"
+    stem_no_prefix = re.sub(r'^\d+_', '', stem)
+    if stem_no_prefix != stem:
+        alt_json = parent_dir / f"{stem_no_prefix}.json"
+        if alt_json.exists():
+            return str(alt_json)
+
+    # Pattern 3: Look for any JSON in the directory that references this archive
+    # by checking the 'file_name' or 'filename' field in the JSON
+    for json_file in parent_dir.glob("*.json"):
+        try:
+            with open(json_file, "r", encoding="utf-8") as f:
+                data = json.load(f)
+
+            # Check if this JSON references our archive
+            # Gallery-DL uses 'file.file_name' or 'filename' + 'extension'
+            file_info = data.get("file", {})
+            json_filename = file_info.get("file_name", "")
+            if not json_filename:
+                # Try filename + extension pattern
+                fn = data.get("filename", "")
+                ext = data.get("extension", "")
+                if fn and ext:
+                    json_filename = f"{fn}.{ext}"
+
+            # Compare with archive filename (with or without prefix)
+            archive_filename = archive_path.name
+            archive_filename_no_prefix = re.sub(r'^\d+_', '', archive_filename)
+
+            if json_filename and (
+                json_filename == archive_filename or
+                json_filename == archive_filename_no_prefix
+            ):
+                return str(json_file)
+
+        except (json.JSONDecodeError, IOError):
+            continue
+
+    return None
+
+
 def load_sidecar_metadata(json_path: str) -> Optional[dict]:
    """Load and parse a Gallery-DL sidecar JSON file."""
    try:
@@ -54,10 +54,11 @@ services:
        condition: service_healthy
    restart: unless-stopped

-  # Celery worker for processing import tasks
-  celery-worker:
+  # Celery worker for processing import tasks (heavy processing)
+  # Handles: import, thumbnail, sidecar, default queues
+  worker:
    build: .
-    command: celery -A app.celery_app:celery worker --loglevel=info -Q scan,import,thumbnail,sidecar,default --concurrency=${CELERY_WORKER_CONCURRENCY:-2}
+    command: celery -A app.celery_app:celery worker --loglevel=info -Q import,thumbnail,sidecar,default --concurrency=${CELERY_WORKER_CONCURRENCY:-2}
    environment:
      - TZ=${TZ:-America/New_York}
      - DB_USER=${DB_USER:-imagerepo}
@@ -78,10 +79,12 @@ services:
        condition: service_healthy
    restart: unless-stopped

-  # Celery Beat scheduler for periodic tasks
-  celery-beat:
+  # Celery scheduler: Beat (periodic tasks) + Worker (maintenance/scan queues)
+  # Handles: maintenance, scan queues + periodic task scheduling
+  # This ensures maintenance tasks aren't blocked by large import queues
+  scheduler:
    build: .
-    command: celery -A app.celery_app:celery beat --loglevel=info
+    command: celery -A app.celery_app:celery worker --beat --loglevel=info -Q maintenance,scan --concurrency=1
    environment:
      - TZ=${TZ:-America/New_York}
      - DB_USER=${DB_USER:-imagerepo}
@@ -92,9 +95,14 @@ services:
      - CELERY_BROKER_URL=redis://redis:6379/0
      - CELERY_RESULT_BACKEND=redis://redis:6379/0
      - IMPORT_EVERY_SECONDS=${IMPORT_EVERY_SECONDS:-28800}
+    volumes:
+      - ./imagerepo/images:/images
+      - ./import:/import
    depends_on:
-      - redis
-      - celery-worker
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
    restart: unless-stopped

 volumes:
@@ -4,7 +4,7 @@ A Flask-based image repository with Celery task processing for importing, organi

 > **IMPORTANT**: This summary must be kept up to date with any code changes. Update the timestamp below when making modifications.
 >
-> **Last Updated**: 2026-01-31 (Scan optimization: quick/deep scan modes, archive tag merging)
+> **Last Updated**: 2026-02-01 (Settings dashboard: system stats, task filtering, error details)

 ---

@@ -290,12 +290,20 @@ Import tasks support two modes based on `context.deep_scan`:
 | Route | Purpose |
 |-------|---------|
 | `POST /api/import/trigger` | Start scan (params: `deep=true` for deep scan, `regenerate_thumbnails`, `reapply_sidecars`, `verify_hashes`) |
-| `GET /api/import/status` | Queue status and recent tasks |
+| `GET /api/import/status` | Queue status counts and active batch info |
+| `GET /api/import/tasks` | List tasks with filtering (params: `status`, `task_type`, `search`, `limit`, `offset`) |
 | `GET /api/import/task/<id>` | Task details |
 | `POST /api/import/retry-failed` | Retry failed tasks |
-| `POST /api/import/clear-completed` | Clear completed tasks |
+| `POST /api/import/clear-completed` | Clear tasks (params: `statuses`, `older_than_days` for age-based cleanup) |
+| `GET /api/import/task-stats` | Task counts with age breakdown (today, this week, older) |
 | `GET /api/import/batches` | List import batches |

+### System Stats API
+
+| Route | Purpose |
+|-------|---------|
+| `GET /api/system/stats` | System overview: total images, tags, storage used, pending imports |
+
 ### Settings API

 | Route | Line | Purpose |