Optimize scan process with quick/deep modes and archive tag merging
Quick scan (default): - Pre-load all ImportTask records for O(1) duplicate checks - Batch task creation commits (50 at a time) - Cache import settings (5-min TTL) - Skip pHash comparison for speed Deep scan (on-demand): - Full reprocessing of all files - pHash similarity detection - Optional thumbnail regeneration - Optional sidecar re-application Archive tag merging: - Add archive tags to existing images when duplicates found - Works for both hash and pHash duplicate detection Also fixes: - Add missing source/post tag icons to gallery templates
This commit is contained in:
Bryan Van Deusen
+326
-98
@@ -4,14 +4,21 @@ Directory scanning task for the import pipeline.
|
||||
|
||||
This task walks the import directory, detects new/changed files,
|
||||
creates ImportTask records, and queues appropriate processing tasks.
|
||||
|
||||
Two scan modes:
|
||||
- Quick scan (default): Pre-loads existing task/hash data for O(1) lookups.
|
||||
Only queues truly new files. Skips pHash comparison for speed.
|
||||
- Deep scan (on-demand): Full reprocessing of all files. Re-extracts metadata,
|
||||
recalculates hashes, regenerates thumbnails. For data integrity verification.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from app.celery_app import celery
|
||||
from app import db
|
||||
from app.models import ImportTask, ImportBatch
|
||||
from app.models import ImportTask, ImportBatch, ImageRecord
|
||||
|
||||
log = logging.getLogger('celery.tasks.scan')
|
||||
|
||||
@@ -26,18 +33,81 @@ ALLOWED_ARCHIVE_EXTS = (
|
||||
)
|
||||
|
||||
|
||||
def _build_task_lookup() -> dict:
|
||||
"""
|
||||
Pre-load all ImportTask records into a lookup dict for O(1) checks.
|
||||
Key: (source_path, task_type)
|
||||
Value: dict with 'status', 'file_size' for quick comparison
|
||||
|
||||
This replaces per-file database queries with a single upfront query.
|
||||
"""
|
||||
lookup = {}
|
||||
for task in ImportTask.query.with_entities(
|
||||
ImportTask.source_path,
|
||||
ImportTask.task_type,
|
||||
ImportTask.status,
|
||||
ImportTask.file_size
|
||||
).all():
|
||||
key = (task.source_path, task.task_type)
|
||||
# Keep the most relevant status for each path
|
||||
# Priority: active > complete > skipped > failed
|
||||
existing = lookup.get(key)
|
||||
if existing is None:
|
||||
lookup[key] = {'status': task.status, 'file_size': task.file_size}
|
||||
elif task.status in ('pending', 'queued', 'processing'):
|
||||
# Active task takes priority
|
||||
lookup[key] = {'status': task.status, 'file_size': task.file_size}
|
||||
elif existing['status'] not in ('pending', 'queued', 'processing'):
|
||||
# Replace if existing is not active and new is complete
|
||||
if task.status == 'complete':
|
||||
lookup[key] = {'status': task.status, 'file_size': task.file_size}
|
||||
return lookup
|
||||
|
||||
|
||||
def _should_skip_file(source_path: str, file_size: int, task_type: str,
|
||||
task_lookup: dict) -> Optional[str]:
|
||||
"""
|
||||
Check if file should be skipped using pre-loaded lookup.
|
||||
|
||||
Returns:
|
||||
Skip reason string if should skip, None if should process
|
||||
"""
|
||||
key = (source_path, task_type)
|
||||
existing = task_lookup.get(key)
|
||||
|
||||
if existing is None:
|
||||
return None # New file, process it
|
||||
|
||||
status = existing['status']
|
||||
|
||||
# Skip if task is currently active
|
||||
if status in ('pending', 'queued', 'processing'):
|
||||
return 'already_active'
|
||||
|
||||
# Skip if completed/skipped with same size
|
||||
if status in ('complete', 'skipped') and existing['file_size'] == file_size:
|
||||
return 'already_processed'
|
||||
|
||||
# File size changed - re-process
|
||||
return None
|
||||
|
||||
|
||||
@celery.task(bind=True, name='app.tasks.scan.scan_directory')
|
||||
def scan_directory(self, source_dir: str = '/import', dest_dir: str = '/images'):
|
||||
"""
|
||||
Scan source directory for new/changed files.
|
||||
Quick scan: Efficiently detect and queue new/changed files.
|
||||
|
||||
Creates ImportTask records for each file and queues them for processing.
|
||||
Optimizations over naive approach:
|
||||
- Pre-loads all existing task records in one query (O(1) lookup per file)
|
||||
- Batches task creation commits
|
||||
- Skips pHash comparison (relies on content hash for duplicates)
|
||||
|
||||
Strategy:
|
||||
1. Walk directory tree under each artist folder
|
||||
2. For each file, check if already in ImportTask with same path+size
|
||||
3. If new/changed, create ImportTask and queue appropriate task
|
||||
4. Handle archives specially (first volume only, queue as atomic unit)
|
||||
1. Load existing ImportTask lookup dict (single query)
|
||||
2. Walk directory tree under each artist folder
|
||||
3. For each file, check lookup dict (no DB queries per file)
|
||||
4. Queue new/changed files for processing
|
||||
5. Handle archives specially (first volume only)
|
||||
|
||||
Args:
|
||||
source_dir: Root directory to scan (default: /import)
|
||||
@@ -49,7 +119,12 @@ def scan_directory(self, source_dir: str = '/import', dest_dir: str = '/images')
|
||||
from app.tasks.import_file import import_media_file, import_archive
|
||||
from app.utils.image_importer import is_first_volume
|
||||
|
||||
log.info(f"Starting directory scan: {source_dir}")
|
||||
log.info(f"Starting quick scan: {source_dir}")
|
||||
|
||||
# Pre-load existing tasks for O(1) lookup (single query)
|
||||
log.debug("Loading existing task lookup...")
|
||||
task_lookup = _build_task_lookup()
|
||||
log.debug(f"Loaded {len(task_lookup)} existing task records")
|
||||
|
||||
# Create batch record
|
||||
batch = ImportBatch(source_directory=source_dir)
|
||||
@@ -65,6 +140,10 @@ def scan_directory(self, source_dir: str = '/import', dest_dir: str = '/images')
|
||||
'skipped_non_media': 0,
|
||||
}
|
||||
|
||||
# Collect tasks to create in batches
|
||||
pending_tasks = []
|
||||
BATCH_SIZE = 50 # Commit every 50 tasks
|
||||
|
||||
try:
|
||||
# Iterate through artist directories
|
||||
for artist_dir in os.listdir(source_dir):
|
||||
@@ -85,6 +164,12 @@ def scan_directory(self, source_dir: str = '/import', dest_dir: str = '/images')
|
||||
if lower.endswith('.json'):
|
||||
continue
|
||||
|
||||
# Get file size once (used for lookup)
|
||||
try:
|
||||
file_size = os.path.getsize(full_path)
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
# Check if archive
|
||||
if lower.endswith(ALLOWED_ARCHIVE_EXTS):
|
||||
# Only process first volume of multi-part archives
|
||||
@@ -92,23 +177,25 @@ def scan_directory(self, source_dir: str = '/import', dest_dir: str = '/images')
|
||||
stats['skipped_non_first_volume'] += 1
|
||||
continue
|
||||
|
||||
# Create and queue archive task
|
||||
task_record = _create_task_if_new(
|
||||
# Check lookup dict (no DB query)
|
||||
skip_reason = _should_skip_file(
|
||||
full_path, file_size, 'import_archive', task_lookup
|
||||
)
|
||||
if skip_reason:
|
||||
stats['skipped_existing'] += 1
|
||||
continue
|
||||
|
||||
# Queue archive task
|
||||
task = ImportTask(
|
||||
source_path=full_path,
|
||||
task_type='import_archive',
|
||||
file_size=file_size,
|
||||
batch_id=batch.id,
|
||||
context={'artist': artist_dir, 'dest_dir': dest_dir}
|
||||
context={'artist': artist_dir, 'dest_dir': dest_dir},
|
||||
status='pending'
|
||||
)
|
||||
|
||||
if task_record:
|
||||
result = import_archive.delay(task_record.id)
|
||||
task_record.celery_task_id = result.id
|
||||
task_record.status = 'queued'
|
||||
db.session.commit()
|
||||
stats['queued_archives'] += 1
|
||||
log.debug(f"Queued archive: {filename}")
|
||||
else:
|
||||
stats['skipped_existing'] += 1
|
||||
pending_tasks.append(('archive', task))
|
||||
stats['queued_archives'] += 1
|
||||
continue
|
||||
|
||||
# Check if media file
|
||||
@@ -116,29 +203,40 @@ def scan_directory(self, source_dir: str = '/import', dest_dir: str = '/images')
|
||||
stats['skipped_non_media'] += 1
|
||||
continue
|
||||
|
||||
# Create and queue media file task
|
||||
task_record = _create_task_if_new(
|
||||
# Check lookup dict (no DB query)
|
||||
skip_reason = _should_skip_file(
|
||||
full_path, file_size, 'import_image', task_lookup
|
||||
)
|
||||
if skip_reason:
|
||||
stats['skipped_existing'] += 1
|
||||
continue
|
||||
|
||||
# Queue media file task
|
||||
task = ImportTask(
|
||||
source_path=full_path,
|
||||
task_type='import_image',
|
||||
file_size=file_size,
|
||||
batch_id=batch.id,
|
||||
context={'artist': artist_dir, 'dest_dir': dest_dir}
|
||||
context={'artist': artist_dir, 'dest_dir': dest_dir},
|
||||
status='pending'
|
||||
)
|
||||
pending_tasks.append(('media', task))
|
||||
stats['queued_media'] += 1
|
||||
|
||||
if task_record:
|
||||
result = import_media_file.delay(task_record.id)
|
||||
task_record.celery_task_id = result.id
|
||||
task_record.status = 'queued'
|
||||
db.session.commit()
|
||||
stats['queued_media'] += 1
|
||||
log.debug(f"Queued media: {filename}")
|
||||
else:
|
||||
stats['skipped_existing'] += 1
|
||||
# Batch commit
|
||||
if len(pending_tasks) >= BATCH_SIZE:
|
||||
_flush_pending_tasks(pending_tasks)
|
||||
pending_tasks = []
|
||||
|
||||
# Flush remaining tasks
|
||||
if pending_tasks:
|
||||
_flush_pending_tasks(pending_tasks)
|
||||
|
||||
# Update batch with totals
|
||||
batch.total_files = stats['total_files']
|
||||
db.session.commit()
|
||||
|
||||
log.info(f"Scan complete. Queued {stats['queued_media']} media, "
|
||||
log.info(f"Quick scan complete. Queued {stats['queued_media']} media, "
|
||||
f"{stats['queued_archives']} archives. "
|
||||
f"Skipped {stats['skipped_existing']} existing.")
|
||||
|
||||
@@ -154,78 +252,208 @@ def scan_directory(self, source_dir: str = '/import', dest_dir: str = '/images')
|
||||
raise
|
||||
|
||||
|
||||
def _create_task_if_new(source_path: str, task_type: str, batch_id: int, context: dict) -> ImportTask:
|
||||
def _flush_pending_tasks(pending_tasks: list):
|
||||
"""
|
||||
Create ImportTask if no existing task for this file.
|
||||
|
||||
Uses file path and size for change detection.
|
||||
Hash is computed later during actual processing.
|
||||
|
||||
Args:
|
||||
source_path: Full path to source file
|
||||
task_type: Type of task ('import_image', 'import_archive', etc.)
|
||||
batch_id: ID of the current ImportBatch
|
||||
context: Context dict with artist, dest_dir, etc.
|
||||
|
||||
Returns:
|
||||
New ImportTask record if created, None if already exists
|
||||
Batch add tasks to session, commit, and queue Celery jobs.
|
||||
"""
|
||||
try:
|
||||
file_size = os.path.getsize(source_path)
|
||||
except OSError as e:
|
||||
log.warning(f"Cannot access file {source_path}: {e}")
|
||||
return None
|
||||
from app.tasks.import_file import import_media_file, import_archive
|
||||
|
||||
# Check for existing pending/queued/processing task for same file
|
||||
existing_active = ImportTask.query.filter_by(
|
||||
source_path=source_path,
|
||||
task_type=task_type
|
||||
).filter(
|
||||
ImportTask.status.in_(['pending', 'queued', 'processing'])
|
||||
).first()
|
||||
# Add all tasks to session
|
||||
for task_type, task in pending_tasks:
|
||||
db.session.add(task)
|
||||
|
||||
if existing_active:
|
||||
log.debug(f"Task already active for: {source_path}")
|
||||
return None
|
||||
# Flush to get IDs
|
||||
db.session.flush()
|
||||
|
||||
# Check for completed task with same file path and size
|
||||
# If size changed, we should re-import
|
||||
completed_same = ImportTask.query.filter_by(
|
||||
source_path=source_path,
|
||||
task_type=task_type,
|
||||
file_size=file_size,
|
||||
status='complete'
|
||||
).first()
|
||||
# Queue Celery jobs
|
||||
for task_type, task in pending_tasks:
|
||||
if task_type == 'archive':
|
||||
result = import_archive.delay(task.id)
|
||||
else:
|
||||
result = import_media_file.delay(task.id)
|
||||
task.celery_task_id = result.id
|
||||
task.status = 'queued'
|
||||
|
||||
if completed_same:
|
||||
log.debug(f"Already imported (same size): {source_path}")
|
||||
return None
|
||||
|
||||
# Also check 'skipped' status - don't re-queue skipped files unless size changed
|
||||
skipped_same = ImportTask.query.filter_by(
|
||||
source_path=source_path,
|
||||
task_type=task_type,
|
||||
file_size=file_size,
|
||||
status='skipped'
|
||||
).first()
|
||||
|
||||
if skipped_same:
|
||||
log.debug(f"Previously skipped (same size): {source_path}")
|
||||
return None
|
||||
|
||||
# Create new task
|
||||
task = ImportTask(
|
||||
source_path=source_path,
|
||||
task_type=task_type,
|
||||
file_size=file_size,
|
||||
batch_id=batch_id,
|
||||
context=context,
|
||||
status='pending'
|
||||
)
|
||||
db.session.add(task)
|
||||
# Commit batch
|
||||
db.session.commit()
|
||||
|
||||
return task
|
||||
|
||||
@celery.task(bind=True, name='app.tasks.scan.deep_scan_directory')
|
||||
def deep_scan_directory(self, source_dir: str = '/import', dest_dir: str = '/images',
|
||||
regenerate_thumbnails: bool = True,
|
||||
reapply_sidecars: bool = True,
|
||||
verify_hashes: bool = True):
|
||||
"""
|
||||
Deep scan: Full reprocessing of all files for data integrity verification.
|
||||
|
||||
Unlike quick scan, this:
|
||||
- Ignores existing task status (forces reprocessing)
|
||||
- Re-extracts all metadata
|
||||
- Recalculates content hashes (optional verification)
|
||||
- Regenerates thumbnails (optional)
|
||||
- Re-applies sidecar metadata (optional)
|
||||
- Runs full pHash comparison for similarity detection
|
||||
|
||||
Use cases:
|
||||
- Data integrity verification after storage issues
|
||||
- Picking up metadata from new/updated sidecar files
|
||||
- Regenerating thumbnails after algorithm changes
|
||||
- Recovery from partial imports
|
||||
|
||||
Args:
|
||||
source_dir: Root directory to scan (default: /import)
|
||||
dest_dir: Destination for imported files (default: /images)
|
||||
regenerate_thumbnails: Regenerate all thumbnails (default: True)
|
||||
reapply_sidecars: Re-apply metadata from sidecar files (default: True)
|
||||
verify_hashes: Verify content hashes match stored records (default: True)
|
||||
|
||||
Returns:
|
||||
dict with batch_id and statistics
|
||||
"""
|
||||
from app.tasks.import_file import import_media_file, import_archive
|
||||
from app.utils.image_importer import is_first_volume
|
||||
|
||||
log.info(f"Starting DEEP scan: {source_dir}")
|
||||
log.info(f"Options: thumbnails={regenerate_thumbnails}, sidecars={reapply_sidecars}, verify={verify_hashes}")
|
||||
|
||||
# Create batch record with deep scan flag
|
||||
batch = ImportBatch(source_directory=source_dir)
|
||||
db.session.add(batch)
|
||||
db.session.commit()
|
||||
|
||||
stats = {
|
||||
'total_files': 0,
|
||||
'queued_media': 0,
|
||||
'queued_archives': 0,
|
||||
'skipped_active': 0, # Only skip currently processing files
|
||||
'skipped_non_first_volume': 0,
|
||||
'skipped_non_media': 0,
|
||||
}
|
||||
|
||||
# For deep scan, only load active tasks to avoid double-processing
|
||||
active_tasks = set()
|
||||
for task in ImportTask.query.filter(
|
||||
ImportTask.status.in_(['pending', 'queued', 'processing'])
|
||||
).with_entities(ImportTask.source_path, ImportTask.task_type).all():
|
||||
active_tasks.add((task.source_path, task.task_type))
|
||||
|
||||
log.debug(f"Found {len(active_tasks)} currently active tasks to skip")
|
||||
|
||||
pending_tasks = []
|
||||
BATCH_SIZE = 50
|
||||
|
||||
try:
|
||||
# Iterate through artist directories
|
||||
for artist_dir in os.listdir(source_dir):
|
||||
artist_path = os.path.join(source_dir, artist_dir)
|
||||
if not os.path.isdir(artist_path):
|
||||
continue
|
||||
|
||||
log.debug(f"Deep scanning artist directory: {artist_dir}")
|
||||
|
||||
# Walk all files in artist directory (including subdirectories)
|
||||
for root, _, files in os.walk(artist_path):
|
||||
for filename in files:
|
||||
full_path = os.path.join(root, filename)
|
||||
lower = filename.lower()
|
||||
stats['total_files'] += 1
|
||||
|
||||
# Skip sidecar JSON files (handled during import)
|
||||
if lower.endswith('.json'):
|
||||
continue
|
||||
|
||||
# Get file size
|
||||
try:
|
||||
file_size = os.path.getsize(full_path)
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
# Check if archive
|
||||
if lower.endswith(ALLOWED_ARCHIVE_EXTS):
|
||||
if not is_first_volume(full_path):
|
||||
stats['skipped_non_first_volume'] += 1
|
||||
continue
|
||||
|
||||
# Only skip if currently active (deep scan reprocesses everything else)
|
||||
if (full_path, 'import_archive') in active_tasks:
|
||||
stats['skipped_active'] += 1
|
||||
continue
|
||||
|
||||
task = ImportTask(
|
||||
source_path=full_path,
|
||||
task_type='import_archive',
|
||||
file_size=file_size,
|
||||
batch_id=batch.id,
|
||||
context={
|
||||
'artist': artist_dir,
|
||||
'dest_dir': dest_dir,
|
||||
'deep_scan': True,
|
||||
'regenerate_thumbnails': regenerate_thumbnails,
|
||||
'reapply_sidecars': reapply_sidecars,
|
||||
'verify_hashes': verify_hashes
|
||||
},
|
||||
status='pending'
|
||||
)
|
||||
pending_tasks.append(('archive', task))
|
||||
stats['queued_archives'] += 1
|
||||
continue
|
||||
|
||||
# Check if media file
|
||||
if not lower.endswith(ALLOWED_MEDIA_EXTS):
|
||||
stats['skipped_non_media'] += 1
|
||||
continue
|
||||
|
||||
# Only skip if currently active
|
||||
if (full_path, 'import_image') in active_tasks:
|
||||
stats['skipped_active'] += 1
|
||||
continue
|
||||
|
||||
task = ImportTask(
|
||||
source_path=full_path,
|
||||
task_type='import_image',
|
||||
file_size=file_size,
|
||||
batch_id=batch.id,
|
||||
context={
|
||||
'artist': artist_dir,
|
||||
'dest_dir': dest_dir,
|
||||
'deep_scan': True,
|
||||
'regenerate_thumbnails': regenerate_thumbnails,
|
||||
'reapply_sidecars': reapply_sidecars,
|
||||
'verify_hashes': verify_hashes
|
||||
},
|
||||
status='pending'
|
||||
)
|
||||
pending_tasks.append(('media', task))
|
||||
stats['queued_media'] += 1
|
||||
|
||||
# Batch commit
|
||||
if len(pending_tasks) >= BATCH_SIZE:
|
||||
_flush_pending_tasks(pending_tasks)
|
||||
pending_tasks = []
|
||||
|
||||
# Flush remaining tasks
|
||||
if pending_tasks:
|
||||
_flush_pending_tasks(pending_tasks)
|
||||
|
||||
# Update batch with totals
|
||||
batch.total_files = stats['total_files']
|
||||
db.session.commit()
|
||||
|
||||
log.info(f"Deep scan complete. Queued {stats['queued_media']} media, "
|
||||
f"{stats['queued_archives']} archives. "
|
||||
f"Skipped {stats['skipped_active']} active.")
|
||||
|
||||
return {
|
||||
'batch_id': batch.id,
|
||||
'stats': stats,
|
||||
'mode': 'deep'
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
log.error(f"Deep scan failed: {e}", exc_info=True)
|
||||
batch.status = 'failed'
|
||||
db.session.commit()
|
||||
raise
|
||||
|
||||
|
||||
@celery.task(name='app.tasks.scan.recover_interrupted_tasks')
|
||||
|
||||
Reference in New Issue
Block a user