changes to mobile styling in modal view, complete reword of backend worker system
This commit is contained in:
Bryan Van Deusen
@@ -0,0 +1,10 @@
|
||||
# app/tasks/__init__.py
|
||||
"""
|
||||
Celery task modules for the ImageRepo import pipeline.
|
||||
|
||||
Task Types:
|
||||
- scan: Directory scanning and task queuing
|
||||
- import_file: Individual file import (images, videos, archives)
|
||||
- thumbnail: Thumbnail generation
|
||||
- sidecar: Gallery-DL metadata extraction and application
|
||||
"""
|
||||
@@ -0,0 +1,507 @@
|
||||
# app/tasks/import_file.py
|
||||
"""
|
||||
File import tasks for images, videos, and archives.
|
||||
|
||||
These tasks handle the actual import of individual files,
|
||||
including filtering, duplicate detection, and storage.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
import imagehash
|
||||
|
||||
from app.celery_app import celery
|
||||
from app import db
|
||||
from app.models import ImportTask, ImageRecord, Tag, ArchiveRecord
|
||||
|
||||
log = logging.getLogger('celery.tasks.import_file')
|
||||
|
||||
|
||||
@celery.task(bind=True, name='app.tasks.import_file.import_media_file',
|
||||
max_retries=3, default_retry_delay=60)
|
||||
def import_media_file(self, task_id: int):
|
||||
"""
|
||||
Import a single media file (image or video).
|
||||
|
||||
Steps:
|
||||
1. Load ImportTask record
|
||||
2. Apply filters (dimensions, transparency, single-color)
|
||||
3. Check for duplicates (hash and pHash)
|
||||
4. Copy to content-addressed storage
|
||||
5. Create ImageRecord
|
||||
6. Queue thumbnail task
|
||||
7. Queue sidecar metadata task
|
||||
8. Update ImportTask status
|
||||
|
||||
Args:
|
||||
task_id: ID of the ImportTask record
|
||||
|
||||
Returns:
|
||||
dict with status and result info
|
||||
"""
|
||||
from app.tasks.thumbnail import generate_thumbnail_task
|
||||
from app.tasks.sidecar import apply_sidecar_metadata
|
||||
from app.utils.image_importer import (
|
||||
extract_metadata, calculate_hash, calculate_perceptual_hash,
|
||||
is_mostly_transparent, is_mostly_single_color, find_similar_image,
|
||||
build_hashed_dest_path, load_import_settings, supersede_image,
|
||||
generate_thumbnail, generate_video_thumbnail_mirrored
|
||||
)
|
||||
from app.utils.metadata_enrichment import find_sidecar_json
|
||||
|
||||
task = ImportTask.query.get(task_id)
|
||||
if not task:
|
||||
log.error(f"Task {task_id} not found")
|
||||
return {'error': 'Task not found'}
|
||||
|
||||
task.status = 'processing'
|
||||
task.started_at = datetime.utcnow()
|
||||
db.session.commit()
|
||||
|
||||
try:
|
||||
src_path = task.source_path
|
||||
context = task.context or {}
|
||||
artist = context.get('artist', 'unknown')
|
||||
dest_dir = context.get('dest_dir', '/images')
|
||||
|
||||
log.info(f"Processing: {src_path}")
|
||||
|
||||
# Load settings
|
||||
settings = load_import_settings()
|
||||
|
||||
# Validate file exists
|
||||
if not os.path.exists(src_path):
|
||||
return _fail_task(task, 'Source file not found')
|
||||
|
||||
filename = os.path.basename(src_path)
|
||||
lower = filename.lower()
|
||||
|
||||
# Extract metadata first for filtering
|
||||
metadata = extract_metadata(src_path)
|
||||
if not metadata.get('width') or not metadata.get('height'):
|
||||
return _skip_task(task, 'Missing dimension data')
|
||||
|
||||
# Apply dimension filter
|
||||
min_width = settings.get('min_width', 0)
|
||||
min_height = settings.get('min_height', 0)
|
||||
if min_width > 0 and metadata['width'] < min_width:
|
||||
return _skip_task(task, f"Too small: {metadata['width']}x{metadata['height']}")
|
||||
if min_height > 0 and metadata['height'] < min_height:
|
||||
return _skip_task(task, f"Too small: {metadata['width']}x{metadata['height']}")
|
||||
|
||||
# Apply transparency filter
|
||||
if settings.get('skip_transparent') and lower.endswith(('.png', '.gif', '.webp')):
|
||||
threshold = settings.get('transparency_threshold', 0.9)
|
||||
if is_mostly_transparent(src_path, threshold):
|
||||
return _skip_task(task, 'Mostly transparent')
|
||||
|
||||
# Apply single-color filter
|
||||
if settings.get('skip_single_color', True) and lower.endswith(('.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp')):
|
||||
if is_mostly_single_color(src_path,
|
||||
settings.get('single_color_threshold', 0.95),
|
||||
settings.get('single_color_tolerance', 30)):
|
||||
return _skip_task(task, 'Mostly single color')
|
||||
|
||||
# Compute content hash
|
||||
file_hash = calculate_hash(src_path)
|
||||
task.file_hash = file_hash
|
||||
|
||||
# Check for exact duplicate by hash
|
||||
existing = ImageRecord.query.filter_by(hash=file_hash).first()
|
||||
if existing:
|
||||
# Still try to enrich with sidecar metadata
|
||||
sidecar_path = find_sidecar_json(src_path)
|
||||
if sidecar_path:
|
||||
apply_sidecar_metadata.delay(existing.id, sidecar_path)
|
||||
return _skip_task(task, 'Duplicate by hash', result_image_id=existing.id)
|
||||
|
||||
# Compute pHash for similarity check
|
||||
phash = calculate_perceptual_hash(src_path)
|
||||
phash_threshold = settings.get('phash_threshold', 10)
|
||||
|
||||
if phash and phash_threshold > 0:
|
||||
# Load existing pHashes for comparison
|
||||
existing_phashes = [
|
||||
(imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height, img.id)
|
||||
for img in ImageRecord.query.filter(ImageRecord.perceptual_hash.isnot(None)).all()
|
||||
]
|
||||
|
||||
relationship, similar_id = find_similar_image(
|
||||
phash, metadata['width'], metadata['height'],
|
||||
existing_phashes, threshold=phash_threshold
|
||||
)
|
||||
|
||||
if relationship == 'larger_exists':
|
||||
return _skip_task(task, 'Similar larger image exists')
|
||||
|
||||
# Handle supersede case - new image is larger than existing
|
||||
if relationship == 'smaller_exists' and settings.get('supersede_smaller', True) and similar_id:
|
||||
old_record = ImageRecord.query.get(similar_id)
|
||||
if old_record:
|
||||
return _supersede_existing(
|
||||
task, old_record, src_path, file_hash, phash,
|
||||
metadata, artist, dest_dir, filename
|
||||
)
|
||||
|
||||
# Normal import - copy to destination
|
||||
target_dir = os.path.join(dest_dir, artist)
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
dest_path = build_hashed_dest_path(target_dir, filename, file_hash)
|
||||
shutil.copy2(src_path, dest_path)
|
||||
|
||||
# Generate thumbnail
|
||||
try:
|
||||
if dest_path.lower().endswith(('.mp4', '.mov', '.webm', '.avi', '.mkv')):
|
||||
thumb_path = generate_video_thumbnail_mirrored(dest_path)
|
||||
else:
|
||||
thumb_path = generate_thumbnail(dest_path)
|
||||
except Exception as e:
|
||||
log.warning(f"Thumbnail generation failed for {filename}: {e}")
|
||||
thumb_path = None
|
||||
|
||||
# Create ImageRecord
|
||||
record = ImageRecord(
|
||||
filename=filename,
|
||||
filepath=dest_path,
|
||||
thumb_path=thumb_path,
|
||||
hash=file_hash,
|
||||
perceptual_hash=str(phash) if phash else None,
|
||||
file_size=metadata['file_size'],
|
||||
width=metadata['width'],
|
||||
height=metadata['height'],
|
||||
format=metadata['format'],
|
||||
camera_model=metadata.get('camera_model'),
|
||||
taken_at=metadata.get('taken_at'),
|
||||
imported_at=datetime.utcnow()
|
||||
)
|
||||
|
||||
# Add artist tag
|
||||
_add_artist_tag(record, artist)
|
||||
|
||||
db.session.add(record)
|
||||
db.session.commit()
|
||||
|
||||
# Complete task
|
||||
task.status = 'complete'
|
||||
task.result_image_id = record.id
|
||||
task.completed_at = datetime.utcnow()
|
||||
db.session.commit()
|
||||
|
||||
# Queue sidecar metadata enrichment
|
||||
sidecar_path = find_sidecar_json(src_path)
|
||||
if sidecar_path:
|
||||
apply_sidecar_metadata.delay(record.id, sidecar_path)
|
||||
|
||||
# Update batch stats
|
||||
if task.batch_id:
|
||||
from app.tasks.scan import update_batch_stats
|
||||
update_batch_stats.delay(task.batch_id)
|
||||
|
||||
log.info(f"Imported: {filename} -> {dest_path}")
|
||||
|
||||
return {
|
||||
'status': 'imported',
|
||||
'image_id': record.id,
|
||||
'dest_path': dest_path
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
log.error(f"Import failed for task {task_id}: {e}", exc_info=True)
|
||||
|
||||
task.retry_count += 1
|
||||
if task.retry_count < 3:
|
||||
task.status = 'queued'
|
||||
db.session.commit()
|
||||
raise self.retry(exc=e)
|
||||
else:
|
||||
return _fail_task(task, str(e))
|
||||
|
||||
|
||||
@celery.task(bind=True, name='app.tasks.import_file.import_archive',
|
||||
max_retries=2, default_retry_delay=120,
|
||||
soft_time_limit=1800, time_limit=3600) # 30min soft, 60min hard for large archives
|
||||
def import_archive(self, task_id: int):
|
||||
"""
|
||||
Import an archive file.
|
||||
|
||||
Steps:
|
||||
1. Check if archive already processed (by hash)
|
||||
2. Extract to temp directory
|
||||
3. Create ImportTask for each media file inside
|
||||
4. Queue import tasks for extracted files
|
||||
5. Create archive tag if files were found
|
||||
6. Clean up is deferred until child tasks complete
|
||||
|
||||
Args:
|
||||
task_id: ID of the ImportTask record
|
||||
|
||||
Returns:
|
||||
dict with status and counts
|
||||
"""
|
||||
from app.utils.image_importer import calculate_hash, compute_next_archive_tag_name
|
||||
from pyunpack import Archive
|
||||
|
||||
task = ImportTask.query.get(task_id)
|
||||
if not task:
|
||||
return {'error': 'Task not found'}
|
||||
|
||||
task.status = 'processing'
|
||||
task.started_at = datetime.utcnow()
|
||||
db.session.commit()
|
||||
|
||||
tmpdir = None
|
||||
try:
|
||||
archive_path = task.source_path
|
||||
context = task.context or {}
|
||||
artist = context.get('artist', 'unknown')
|
||||
dest_dir = context.get('dest_dir', '/images')
|
||||
|
||||
log.info(f"Processing archive: {archive_path}")
|
||||
|
||||
# Compute archive hash
|
||||
archive_hash = calculate_hash(archive_path)
|
||||
task.file_hash = archive_hash
|
||||
|
||||
# Check if already processed
|
||||
existing = ArchiveRecord.query.filter_by(hash=archive_hash).first()
|
||||
if existing:
|
||||
return _skip_task(task, 'Archive already imported')
|
||||
|
||||
# Get archive size
|
||||
archive_size = os.path.getsize(archive_path)
|
||||
|
||||
# Extract archive
|
||||
tmpdir = tempfile.mkdtemp(prefix='extract_')
|
||||
log.info(f"Extracting to: {tmpdir}")
|
||||
|
||||
try:
|
||||
Archive(archive_path).extractall(tmpdir)
|
||||
except Exception as e:
|
||||
log.error(f"Extraction failed: {e}")
|
||||
if tmpdir and os.path.exists(tmpdir):
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
return _fail_task(task, f'Extraction failed: {e}')
|
||||
|
||||
# Find media files in extracted content
|
||||
media_files = []
|
||||
for root, _, files in os.walk(tmpdir):
|
||||
for filename in files:
|
||||
lower = filename.lower()
|
||||
if lower.endswith(('.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp',
|
||||
'.mp4', '.mov', '.webm', '.avi', '.mkv')):
|
||||
media_files.append(os.path.join(root, filename))
|
||||
|
||||
if not media_files:
|
||||
log.info(f"No media files found in archive: {archive_path}")
|
||||
if tmpdir and os.path.exists(tmpdir):
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
return _skip_task(task, 'No media files in archive')
|
||||
|
||||
log.info(f"Found {len(media_files)} media files in archive")
|
||||
|
||||
# Create archive tag
|
||||
tag_name = compute_next_archive_tag_name(artist)
|
||||
archive_tag = Tag.query.filter_by(name=tag_name).first()
|
||||
if not archive_tag:
|
||||
archive_tag = Tag(name=tag_name, kind='archive')
|
||||
db.session.add(archive_tag)
|
||||
db.session.flush()
|
||||
|
||||
# Create ArchiveRecord
|
||||
archive_record = ArchiveRecord(
|
||||
filename=archive_path,
|
||||
file_size=archive_size,
|
||||
hash=archive_hash,
|
||||
artist=artist,
|
||||
tag_id=archive_tag.id
|
||||
)
|
||||
db.session.add(archive_record)
|
||||
db.session.commit()
|
||||
|
||||
# Queue tasks for each media file
|
||||
queued_count = 0
|
||||
for media_path in media_files:
|
||||
child_task = ImportTask(
|
||||
source_path=media_path,
|
||||
task_type='import_image',
|
||||
batch_id=task.batch_id,
|
||||
context={
|
||||
'artist': artist,
|
||||
'dest_dir': dest_dir,
|
||||
'archive_task_id': task.id,
|
||||
'archive_tag_id': archive_tag.id,
|
||||
'temp_dir': tmpdir
|
||||
},
|
||||
status='pending'
|
||||
)
|
||||
db.session.add(child_task)
|
||||
db.session.flush()
|
||||
|
||||
result = import_media_file.delay(child_task.id)
|
||||
child_task.celery_task_id = result.id
|
||||
child_task.status = 'queued'
|
||||
queued_count += 1
|
||||
|
||||
# Store child task info for cleanup tracking
|
||||
task.context = task.context or {}
|
||||
task.context['temp_dir'] = tmpdir
|
||||
task.context['archive_tag_id'] = archive_tag.id
|
||||
task.context['child_count'] = queued_count
|
||||
|
||||
task.status = 'complete'
|
||||
task.completed_at = datetime.utcnow()
|
||||
db.session.commit()
|
||||
|
||||
log.info(f"Archive processed: queued {queued_count} files from {archive_path}")
|
||||
|
||||
# Note: temp directory cleanup happens via a separate periodic task
|
||||
# or when all child tasks complete
|
||||
|
||||
return {
|
||||
'status': 'extracted',
|
||||
'queued_files': queued_count,
|
||||
'archive_tag': tag_name
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
log.error(f"Archive import failed: {e}", exc_info=True)
|
||||
if tmpdir and os.path.exists(tmpdir):
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
return _fail_task(task, str(e))
|
||||
|
||||
|
||||
def _fail_task(task: ImportTask, error: str, **kwargs) -> dict:
|
||||
"""Mark task as failed with error message."""
|
||||
task.status = 'failed'
|
||||
task.error_message = error
|
||||
task.completed_at = datetime.utcnow()
|
||||
for key, value in kwargs.items():
|
||||
setattr(task, key, value)
|
||||
db.session.commit()
|
||||
|
||||
# Update batch stats
|
||||
if task.batch_id:
|
||||
from app.tasks.scan import update_batch_stats
|
||||
update_batch_stats.delay(task.batch_id)
|
||||
|
||||
log.error(f"Task {task.id} failed: {error}")
|
||||
return {'status': 'failed', 'error': error}
|
||||
|
||||
|
||||
def _skip_task(task: ImportTask, reason: str, **kwargs) -> dict:
|
||||
"""Mark task as skipped with reason."""
|
||||
task.status = 'skipped'
|
||||
task.error_message = reason
|
||||
task.completed_at = datetime.utcnow()
|
||||
for key, value in kwargs.items():
|
||||
setattr(task, key, value)
|
||||
db.session.commit()
|
||||
|
||||
# Update batch stats
|
||||
if task.batch_id:
|
||||
from app.tasks.scan import update_batch_stats
|
||||
update_batch_stats.delay(task.batch_id)
|
||||
|
||||
log.info(f"Task {task.id} skipped: {reason}")
|
||||
return {'status': 'skipped', 'reason': reason}
|
||||
|
||||
|
||||
def _add_artist_tag(record: ImageRecord, artist: str):
|
||||
"""Add artist tag to image record."""
|
||||
if not artist:
|
||||
return
|
||||
|
||||
tag_name = f"artist:{artist}"
|
||||
tag = Tag.query.filter_by(name=tag_name).first()
|
||||
if not tag:
|
||||
tag = Tag(name=tag_name, kind='artist')
|
||||
db.session.add(tag)
|
||||
db.session.flush()
|
||||
|
||||
if tag not in record.tags:
|
||||
record.tags.append(tag)
|
||||
|
||||
|
||||
def _add_archive_tag(record: ImageRecord, archive_tag_id: int):
|
||||
"""Add archive tag to image record."""
|
||||
if not archive_tag_id:
|
||||
return
|
||||
|
||||
tag = Tag.query.get(archive_tag_id)
|
||||
if tag and tag not in record.tags:
|
||||
record.tags.append(tag)
|
||||
|
||||
|
||||
def _supersede_existing(task: ImportTask, old_record: ImageRecord, src_path: str,
|
||||
file_hash: str, phash, metadata: dict, artist: str,
|
||||
dest_dir: str, filename: str) -> dict:
|
||||
"""
|
||||
Replace an existing smaller image with a larger version.
|
||||
|
||||
Preserves tags and metadata from the old record.
|
||||
"""
|
||||
from app.utils.image_importer import (
|
||||
build_hashed_dest_path, generate_thumbnail, generate_video_thumbnail_mirrored,
|
||||
supersede_image
|
||||
)
|
||||
from app.utils.metadata_enrichment import find_sidecar_json, enrich_from_sidecar
|
||||
from app.tasks.sidecar import apply_sidecar_metadata
|
||||
|
||||
log.info(f"Superseding {old_record.filename} ({old_record.width}x{old_record.height}) "
|
||||
f"with {filename} ({metadata['width']}x{metadata['height']})")
|
||||
|
||||
# Enrich metadata from sidecar if available
|
||||
sidecar_path = find_sidecar_json(src_path)
|
||||
if sidecar_path:
|
||||
enriched = enrich_from_sidecar(sidecar_path, {'taken_at': metadata.get('taken_at')})
|
||||
if enriched.get('taken_at'):
|
||||
metadata['taken_at'] = enriched['taken_at']
|
||||
|
||||
# Copy new file to destination
|
||||
target_dir = os.path.join(dest_dir, artist)
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
dest_path = build_hashed_dest_path(target_dir, filename, file_hash)
|
||||
shutil.copy2(src_path, dest_path)
|
||||
|
||||
# Generate new thumbnail
|
||||
try:
|
||||
if dest_path.lower().endswith(('.mp4', '.mov', '.webm', '.avi', '.mkv')):
|
||||
thumb_path = generate_video_thumbnail_mirrored(dest_path)
|
||||
else:
|
||||
thumb_path = generate_thumbnail(dest_path)
|
||||
except Exception as e:
|
||||
log.warning(f"Thumbnail generation failed: {e}")
|
||||
thumb_path = None
|
||||
|
||||
# Supersede the old record
|
||||
record = supersede_image(
|
||||
old_record, dest_path, thumb_path, file_hash, phash, metadata, filename
|
||||
)
|
||||
|
||||
db.session.commit()
|
||||
|
||||
# Complete task
|
||||
task.status = 'complete'
|
||||
task.result_image_id = record.id
|
||||
task.completed_at = datetime.utcnow()
|
||||
db.session.commit()
|
||||
|
||||
# Apply sidecar metadata if available
|
||||
if sidecar_path:
|
||||
apply_sidecar_metadata.delay(record.id, sidecar_path)
|
||||
|
||||
# Update batch stats
|
||||
if task.batch_id:
|
||||
from app.tasks.scan import update_batch_stats
|
||||
update_batch_stats.delay(task.batch_id)
|
||||
|
||||
return {
|
||||
'status': 'superseded',
|
||||
'image_id': record.id,
|
||||
'dest_path': dest_path
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
# app/tasks/scan.py
|
||||
"""
|
||||
Directory scanning task for the import pipeline.
|
||||
|
||||
This task walks the import directory, detects new/changed files,
|
||||
creates ImportTask records, and queues appropriate processing tasks.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from app.celery_app import celery
|
||||
from app import db
|
||||
from app.models import ImportTask, ImportBatch
|
||||
|
||||
log = logging.getLogger('celery.tasks.scan')
|
||||
|
||||
# Supported file extensions (imported from image_importer for consistency)
|
||||
ALLOWED_MEDIA_EXTS = (
|
||||
'.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff', '.tif',
|
||||
'.mp4', '.mov', '.webm', '.avi', '.mkv'
|
||||
)
|
||||
ALLOWED_ARCHIVE_EXTS = (
|
||||
'.zip', '.rar', '.7z', '.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tbz2',
|
||||
'.cbr', '.cbz'
|
||||
)
|
||||
|
||||
|
||||
@celery.task(bind=True, name='app.tasks.scan.scan_directory')
|
||||
def scan_directory(self, source_dir: str = '/import', dest_dir: str = '/images'):
|
||||
"""
|
||||
Scan source directory for new/changed files.
|
||||
|
||||
Creates ImportTask records for each file and queues them for processing.
|
||||
|
||||
Strategy:
|
||||
1. Walk directory tree under each artist folder
|
||||
2. For each file, check if already in ImportTask with same path+size
|
||||
3. If new/changed, create ImportTask and queue appropriate task
|
||||
4. Handle archives specially (first volume only, queue as atomic unit)
|
||||
|
||||
Args:
|
||||
source_dir: Root directory to scan (default: /import)
|
||||
dest_dir: Destination for imported files (default: /images)
|
||||
|
||||
Returns:
|
||||
dict with batch_id and statistics
|
||||
"""
|
||||
from app.tasks.import_file import import_media_file, import_archive
|
||||
from app.utils.image_importer import is_first_volume
|
||||
|
||||
log.info(f"Starting directory scan: {source_dir}")
|
||||
|
||||
# Create batch record
|
||||
batch = ImportBatch(source_directory=source_dir)
|
||||
db.session.add(batch)
|
||||
db.session.commit()
|
||||
|
||||
stats = {
|
||||
'total_files': 0,
|
||||
'queued_media': 0,
|
||||
'queued_archives': 0,
|
||||
'skipped_existing': 0,
|
||||
'skipped_non_first_volume': 0,
|
||||
'skipped_non_media': 0,
|
||||
}
|
||||
|
||||
try:
|
||||
# Iterate through artist directories
|
||||
for artist_dir in os.listdir(source_dir):
|
||||
artist_path = os.path.join(source_dir, artist_dir)
|
||||
if not os.path.isdir(artist_path):
|
||||
continue
|
||||
|
||||
log.debug(f"Scanning artist directory: {artist_dir}")
|
||||
|
||||
# Walk all files in artist directory (including subdirectories)
|
||||
for root, _, files in os.walk(artist_path):
|
||||
for filename in files:
|
||||
full_path = os.path.join(root, filename)
|
||||
lower = filename.lower()
|
||||
stats['total_files'] += 1
|
||||
|
||||
# Skip sidecar JSON files (handled during import)
|
||||
if lower.endswith('.json'):
|
||||
continue
|
||||
|
||||
# Check if archive
|
||||
if lower.endswith(ALLOWED_ARCHIVE_EXTS):
|
||||
# Only process first volume of multi-part archives
|
||||
if not is_first_volume(full_path):
|
||||
stats['skipped_non_first_volume'] += 1
|
||||
continue
|
||||
|
||||
# Create and queue archive task
|
||||
task_record = _create_task_if_new(
|
||||
source_path=full_path,
|
||||
task_type='import_archive',
|
||||
batch_id=batch.id,
|
||||
context={'artist': artist_dir, 'dest_dir': dest_dir}
|
||||
)
|
||||
|
||||
if task_record:
|
||||
result = import_archive.delay(task_record.id)
|
||||
task_record.celery_task_id = result.id
|
||||
task_record.status = 'queued'
|
||||
db.session.commit()
|
||||
stats['queued_archives'] += 1
|
||||
log.debug(f"Queued archive: {filename}")
|
||||
else:
|
||||
stats['skipped_existing'] += 1
|
||||
continue
|
||||
|
||||
# Check if media file
|
||||
if not lower.endswith(ALLOWED_MEDIA_EXTS):
|
||||
stats['skipped_non_media'] += 1
|
||||
continue
|
||||
|
||||
# Create and queue media file task
|
||||
task_record = _create_task_if_new(
|
||||
source_path=full_path,
|
||||
task_type='import_image',
|
||||
batch_id=batch.id,
|
||||
context={'artist': artist_dir, 'dest_dir': dest_dir}
|
||||
)
|
||||
|
||||
if task_record:
|
||||
result = import_media_file.delay(task_record.id)
|
||||
task_record.celery_task_id = result.id
|
||||
task_record.status = 'queued'
|
||||
db.session.commit()
|
||||
stats['queued_media'] += 1
|
||||
log.debug(f"Queued media: {filename}")
|
||||
else:
|
||||
stats['skipped_existing'] += 1
|
||||
|
||||
# Update batch with totals
|
||||
batch.total_files = stats['total_files']
|
||||
db.session.commit()
|
||||
|
||||
log.info(f"Scan complete. Queued {stats['queued_media']} media, "
|
||||
f"{stats['queued_archives']} archives. "
|
||||
f"Skipped {stats['skipped_existing']} existing.")
|
||||
|
||||
return {
|
||||
'batch_id': batch.id,
|
||||
'stats': stats
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
log.error(f"Scan failed: {e}", exc_info=True)
|
||||
batch.status = 'failed'
|
||||
db.session.commit()
|
||||
raise
|
||||
|
||||
|
||||
def _create_task_if_new(source_path: str, task_type: str, batch_id: int, context: dict) -> ImportTask:
|
||||
"""
|
||||
Create ImportTask if no existing task for this file.
|
||||
|
||||
Uses file path and size for change detection.
|
||||
Hash is computed later during actual processing.
|
||||
|
||||
Args:
|
||||
source_path: Full path to source file
|
||||
task_type: Type of task ('import_image', 'import_archive', etc.)
|
||||
batch_id: ID of the current ImportBatch
|
||||
context: Context dict with artist, dest_dir, etc.
|
||||
|
||||
Returns:
|
||||
New ImportTask record if created, None if already exists
|
||||
"""
|
||||
try:
|
||||
file_size = os.path.getsize(source_path)
|
||||
except OSError as e:
|
||||
log.warning(f"Cannot access file {source_path}: {e}")
|
||||
return None
|
||||
|
||||
# Check for existing pending/queued/processing task for same file
|
||||
existing_active = ImportTask.query.filter_by(
|
||||
source_path=source_path,
|
||||
task_type=task_type
|
||||
).filter(
|
||||
ImportTask.status.in_(['pending', 'queued', 'processing'])
|
||||
).first()
|
||||
|
||||
if existing_active:
|
||||
log.debug(f"Task already active for: {source_path}")
|
||||
return None
|
||||
|
||||
# Check for completed task with same file path and size
|
||||
# If size changed, we should re-import
|
||||
completed_same = ImportTask.query.filter_by(
|
||||
source_path=source_path,
|
||||
task_type=task_type,
|
||||
file_size=file_size,
|
||||
status='complete'
|
||||
).first()
|
||||
|
||||
if completed_same:
|
||||
log.debug(f"Already imported (same size): {source_path}")
|
||||
return None
|
||||
|
||||
# Also check 'skipped' status - don't re-queue skipped files unless size changed
|
||||
skipped_same = ImportTask.query.filter_by(
|
||||
source_path=source_path,
|
||||
task_type=task_type,
|
||||
file_size=file_size,
|
||||
status='skipped'
|
||||
).first()
|
||||
|
||||
if skipped_same:
|
||||
log.debug(f"Previously skipped (same size): {source_path}")
|
||||
return None
|
||||
|
||||
# Create new task
|
||||
task = ImportTask(
|
||||
source_path=source_path,
|
||||
task_type=task_type,
|
||||
file_size=file_size,
|
||||
batch_id=batch_id,
|
||||
context=context,
|
||||
status='pending'
|
||||
)
|
||||
db.session.add(task)
|
||||
db.session.commit()
|
||||
|
||||
return task
|
||||
|
||||
|
||||
@celery.task(name='app.tasks.scan.recover_interrupted_tasks')
|
||||
def recover_interrupted_tasks():
|
||||
"""
|
||||
Find and re-queue tasks that were interrupted.
|
||||
|
||||
Called periodically by Celery Beat to recover from worker crashes.
|
||||
Tasks with status 'processing' or 'queued' that have no active
|
||||
Celery task are re-queued.
|
||||
|
||||
Returns:
|
||||
dict with count of recovered tasks
|
||||
"""
|
||||
from app.tasks.import_file import import_media_file, import_archive
|
||||
from app.tasks.thumbnail import generate_thumbnail_task
|
||||
|
||||
log.info("Checking for interrupted tasks...")
|
||||
|
||||
# Find tasks that are stuck in processing/queued state
|
||||
# We look for tasks that have been in this state for more than 10 minutes
|
||||
cutoff = datetime.utcnow()
|
||||
|
||||
interrupted = ImportTask.query.filter(
|
||||
ImportTask.status.in_(['processing', 'queued'])
|
||||
).all()
|
||||
|
||||
recovered = 0
|
||||
for task in interrupted:
|
||||
# Check if Celery task is still active
|
||||
if task.celery_task_id:
|
||||
from app.celery_app import celery as celery_app
|
||||
result = celery_app.AsyncResult(task.celery_task_id)
|
||||
# If task is still pending/started, don't re-queue
|
||||
if result.state in ('PENDING', 'STARTED', 'RETRY'):
|
||||
continue
|
||||
|
||||
log.info(f"Recovering task {task.id}: {task.source_path}")
|
||||
|
||||
# Reset task state
|
||||
task.status = 'pending'
|
||||
task.celery_task_id = None
|
||||
task.started_at = None
|
||||
task.retry_count += 1
|
||||
|
||||
# Re-queue based on type
|
||||
if task.task_type == 'import_image':
|
||||
result = import_media_file.delay(task.id)
|
||||
elif task.task_type == 'import_archive':
|
||||
result = import_archive.delay(task.id)
|
||||
elif task.task_type == 'thumbnail':
|
||||
# Thumbnail tasks store image_id in context
|
||||
image_id = task.context.get('image_id') if task.context else None
|
||||
if image_id:
|
||||
result = generate_thumbnail_task.delay(image_id)
|
||||
else:
|
||||
task.status = 'failed'
|
||||
task.error_message = 'Missing image_id in context'
|
||||
db.session.commit()
|
||||
continue
|
||||
else:
|
||||
log.warning(f"Unknown task type: {task.task_type}")
|
||||
continue
|
||||
|
||||
task.celery_task_id = result.id
|
||||
task.status = 'queued'
|
||||
recovered += 1
|
||||
|
||||
db.session.commit()
|
||||
|
||||
if recovered > 0:
|
||||
log.info(f"Recovered {recovered} interrupted tasks")
|
||||
|
||||
return {'recovered': recovered}
|
||||
|
||||
|
||||
@celery.task(name='app.tasks.scan.update_batch_stats')
|
||||
def update_batch_stats(batch_id: int):
|
||||
"""
|
||||
Update statistics for an ImportBatch based on its tasks.
|
||||
|
||||
Args:
|
||||
batch_id: ID of the batch to update
|
||||
"""
|
||||
batch = ImportBatch.query.get(batch_id)
|
||||
if not batch:
|
||||
return
|
||||
|
||||
from sqlalchemy import func
|
||||
|
||||
# Count tasks by status
|
||||
status_counts = dict(
|
||||
db.session.query(
|
||||
ImportTask.status,
|
||||
func.count(ImportTask.id)
|
||||
).filter(
|
||||
ImportTask.batch_id == batch_id
|
||||
).group_by(ImportTask.status).all()
|
||||
)
|
||||
|
||||
batch.processed_files = sum(
|
||||
status_counts.get(s, 0)
|
||||
for s in ['complete', 'skipped', 'failed']
|
||||
)
|
||||
batch.imported_count = status_counts.get('complete', 0)
|
||||
batch.skipped_count = status_counts.get('skipped', 0)
|
||||
batch.failed_count = status_counts.get('failed', 0)
|
||||
|
||||
# Check if batch is complete
|
||||
pending_or_active = sum(
|
||||
status_counts.get(s, 0)
|
||||
for s in ['pending', 'queued', 'processing']
|
||||
)
|
||||
|
||||
if pending_or_active == 0 and batch.status == 'running':
|
||||
batch.status = 'complete'
|
||||
batch.completed_at = datetime.utcnow()
|
||||
log.info(f"Batch {batch_id} complete: {batch.imported_count} imported, "
|
||||
f"{batch.skipped_count} skipped, {batch.failed_count} failed")
|
||||
|
||||
db.session.commit()
|
||||
@@ -0,0 +1,181 @@
|
||||
# app/tasks/sidecar.py
|
||||
"""
|
||||
Sidecar metadata extraction and application tasks.
|
||||
|
||||
Handles finding and applying metadata from Gallery-DL sidecar JSON files
|
||||
to imported images.
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from app.celery_app import celery
|
||||
from app import db
|
||||
from app.models import ImageRecord, Tag
|
||||
|
||||
log = logging.getLogger('celery.tasks.sidecar')
|
||||
|
||||
|
||||
@celery.task(bind=True, name='app.tasks.sidecar.apply_sidecar_metadata',
|
||||
max_retries=2, default_retry_delay=30)
|
||||
def apply_sidecar_metadata(self, image_id: int, sidecar_path: str):
|
||||
"""
|
||||
Apply metadata from a sidecar JSON file to an image record.
|
||||
|
||||
Extracts platform info, artist, post ID, dates, and generates
|
||||
appropriate tags (source:platform, post:platform:artist:id).
|
||||
|
||||
Args:
|
||||
image_id: ID of the ImageRecord to update
|
||||
sidecar_path: Path to the sidecar JSON file
|
||||
|
||||
Returns:
|
||||
dict with status and applied metadata info
|
||||
"""
|
||||
from app.utils.metadata_enrichment import (
|
||||
enrich_from_sidecar, resolve_date_conflict, load_sidecar_metadata
|
||||
)
|
||||
|
||||
record = ImageRecord.query.get(image_id)
|
||||
if not record:
|
||||
log.warning(f"Image {image_id} not found")
|
||||
return {'error': 'Image not found'}
|
||||
|
||||
try:
|
||||
# Load sidecar data
|
||||
sidecar_data = load_sidecar_metadata(sidecar_path)
|
||||
if not sidecar_data:
|
||||
return {'status': 'no_metadata', 'reason': 'Failed to load sidecar'}
|
||||
|
||||
# Get enriched metadata
|
||||
enriched = enrich_from_sidecar(sidecar_path, {'taken_at': record.taken_at})
|
||||
|
||||
if not enriched.get('raw_metadata'):
|
||||
return {'status': 'no_metadata', 'reason': 'No metadata extracted'}
|
||||
|
||||
tags_added = 0
|
||||
|
||||
# Update date if earlier
|
||||
new_date = resolve_date_conflict(record.taken_at, enriched.get('taken_at'))
|
||||
if new_date != record.taken_at:
|
||||
record.taken_at = new_date
|
||||
log.debug(f"Updated date for {image_id}: {new_date}")
|
||||
|
||||
# Apply tags from metadata
|
||||
for tag_info in enriched.get('tags', []):
|
||||
tag_name = tag_info.get('name')
|
||||
tag_kind = tag_info.get('kind')
|
||||
if not tag_name:
|
||||
continue
|
||||
|
||||
tag = Tag.query.filter_by(name=tag_name).first()
|
||||
if not tag:
|
||||
tag = Tag(name=tag_name, kind=tag_kind)
|
||||
db.session.add(tag)
|
||||
db.session.flush()
|
||||
log.debug(f"Created tag: {tag_name} ({tag_kind})")
|
||||
|
||||
if tag not in record.tags:
|
||||
record.tags.append(tag)
|
||||
tags_added += 1
|
||||
|
||||
db.session.commit()
|
||||
|
||||
log.info(f"Applied sidecar metadata to {image_id}: "
|
||||
f"platform={enriched.get('platform')}, tags_added={tags_added}")
|
||||
|
||||
return {
|
||||
'status': 'applied',
|
||||
'platform': enriched.get('platform'),
|
||||
'artist': enriched.get('artist'),
|
||||
'post_id': enriched.get('post_id'),
|
||||
'tags_added': tags_added
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
log.error(f"Sidecar application failed for {image_id}: {e}", exc_info=True)
|
||||
raise self.retry(exc=e)
|
||||
|
||||
|
||||
@celery.task(name='app.tasks.sidecar.scan_and_match_sidecars')
|
||||
def scan_and_match_sidecars(batch_id: int = None, limit: int = 1000):
|
||||
"""
|
||||
Scan for sidecar files and match them to imported images.
|
||||
|
||||
Looks at recently imported images and tries to find corresponding
|
||||
sidecar JSON files for metadata enrichment.
|
||||
|
||||
Args:
|
||||
batch_id: Optional batch ID to limit scope
|
||||
limit: Maximum number of images to process
|
||||
|
||||
Returns:
|
||||
dict with count of matched sidecars
|
||||
"""
|
||||
from app.utils.metadata_enrichment import find_sidecar_json
|
||||
|
||||
log.info(f"Scanning for unmatched sidecars (batch={batch_id}, limit={limit})")
|
||||
|
||||
# Build query for images that might need sidecar matching
|
||||
query = ImageRecord.query.filter(
|
||||
ImageRecord.filepath.isnot(None)
|
||||
).order_by(ImageRecord.imported_at.desc())
|
||||
|
||||
if limit:
|
||||
query = query.limit(limit)
|
||||
|
||||
matched = 0
|
||||
already_has_source = 0
|
||||
|
||||
for record in query.all():
|
||||
# Check if already has source tag (indicates sidecar was processed)
|
||||
has_source_tag = any(
|
||||
tag.kind == 'source' or tag.kind == 'post'
|
||||
for tag in record.tags
|
||||
)
|
||||
if has_source_tag:
|
||||
already_has_source += 1
|
||||
continue
|
||||
|
||||
# Try to find sidecar
|
||||
sidecar_path = find_sidecar_json(record.filepath)
|
||||
if sidecar_path:
|
||||
apply_sidecar_metadata.delay(record.id, sidecar_path)
|
||||
matched += 1
|
||||
|
||||
log.info(f"Sidecar scan complete: matched={matched}, already_tagged={already_has_source}")
|
||||
return {'matched': matched, 'already_tagged': already_has_source}
|
||||
|
||||
|
||||
@celery.task(name='app.tasks.sidecar.reprocess_sidecars_for_tag')
|
||||
def reprocess_sidecars_for_tag(tag_name: str):
|
||||
"""
|
||||
Reprocess sidecar metadata for all images with a specific tag.
|
||||
|
||||
Useful for re-running sidecar extraction after updates to
|
||||
the metadata extraction logic.
|
||||
|
||||
Args:
|
||||
tag_name: Name of the tag to filter by
|
||||
|
||||
Returns:
|
||||
dict with count of queued tasks
|
||||
"""
|
||||
from app.utils.metadata_enrichment import find_sidecar_json
|
||||
|
||||
tag = Tag.query.filter_by(name=tag_name).first()
|
||||
if not tag:
|
||||
return {'error': f'Tag not found: {tag_name}'}
|
||||
|
||||
queued = 0
|
||||
for record in tag.images:
|
||||
if not record.filepath:
|
||||
continue
|
||||
|
||||
sidecar_path = find_sidecar_json(record.filepath)
|
||||
if sidecar_path:
|
||||
apply_sidecar_metadata.delay(record.id, sidecar_path)
|
||||
queued += 1
|
||||
|
||||
log.info(f"Queued {queued} sidecar reprocessing tasks for tag: {tag_name}")
|
||||
return {'queued': queued, 'tag': tag_name}
|
||||
@@ -0,0 +1,153 @@
|
||||
# app/tasks/thumbnail.py
|
||||
"""
|
||||
Thumbnail generation tasks.
|
||||
|
||||
Handles generating thumbnails for images and videos,
|
||||
both individually and in bulk.
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from app.celery_app import celery
|
||||
from app import db
|
||||
from app.models import ImageRecord, ImportTask
|
||||
|
||||
log = logging.getLogger('celery.tasks.thumbnail')
|
||||
|
||||
|
||||
@celery.task(bind=True, name='app.tasks.thumbnail.generate_thumbnail_task',
|
||||
max_retries=2, default_retry_delay=30)
|
||||
def generate_thumbnail_task(self, image_id: int, overwrite: bool = False):
|
||||
"""
|
||||
Generate thumbnail for a single image/video.
|
||||
|
||||
Args:
|
||||
image_id: ID of the ImageRecord
|
||||
overwrite: Whether to overwrite existing thumbnail
|
||||
|
||||
Returns:
|
||||
dict with status and thumbnail path
|
||||
"""
|
||||
from app.utils.image_importer import generate_thumbnail, generate_video_thumbnail_mirrored
|
||||
|
||||
record = ImageRecord.query.get(image_id)
|
||||
if not record:
|
||||
log.warning(f"Image {image_id} not found")
|
||||
return {'error': 'Image not found'}
|
||||
|
||||
try:
|
||||
filepath = record.filepath
|
||||
if not filepath:
|
||||
return {'error': 'No filepath'}
|
||||
|
||||
# Check if thumbnail already exists and we're not overwriting
|
||||
if record.thumb_path and not overwrite:
|
||||
import os
|
||||
if os.path.exists(record.thumb_path):
|
||||
return {'status': 'exists', 'thumb_path': record.thumb_path}
|
||||
|
||||
log.debug(f"Generating thumbnail for: {filepath}")
|
||||
|
||||
is_video = filepath.lower().endswith(('.mp4', '.mov', '.webm', '.avi', '.mkv'))
|
||||
|
||||
if is_video:
|
||||
thumb_path = generate_video_thumbnail_mirrored(filepath)
|
||||
else:
|
||||
thumb_path = generate_thumbnail(filepath, overwrite=overwrite)
|
||||
|
||||
if thumb_path:
|
||||
record.thumb_path = thumb_path
|
||||
db.session.commit()
|
||||
log.info(f"Generated thumbnail: {thumb_path}")
|
||||
return {'status': 'success', 'thumb_path': thumb_path}
|
||||
else:
|
||||
return {'status': 'failed', 'error': 'Thumbnail generation returned None'}
|
||||
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
log.error(f"Thumbnail generation failed for {image_id}: {e}", exc_info=True)
|
||||
raise self.retry(exc=e)
|
||||
|
||||
|
||||
@celery.task(name='app.tasks.thumbnail.regenerate_all_thumbnails')
|
||||
def regenerate_all_thumbnails(batch_size: int = 100, overwrite: bool = True):
|
||||
"""
|
||||
Queue thumbnail regeneration for all images.
|
||||
|
||||
Processes images in batches to avoid memory issues.
|
||||
|
||||
Args:
|
||||
batch_size: Number of images to process per batch
|
||||
overwrite: Whether to overwrite existing thumbnails
|
||||
|
||||
Returns:
|
||||
dict with count of queued tasks
|
||||
"""
|
||||
log.info("Starting bulk thumbnail regeneration")
|
||||
|
||||
count = 0
|
||||
last_id = 0
|
||||
|
||||
while True:
|
||||
records = ImageRecord.query.filter(
|
||||
ImageRecord.id > last_id,
|
||||
ImageRecord.filepath.isnot(None)
|
||||
).order_by(ImageRecord.id.asc()).limit(batch_size).all()
|
||||
|
||||
if not records:
|
||||
break
|
||||
|
||||
for record in records:
|
||||
generate_thumbnail_task.delay(record.id, overwrite=overwrite)
|
||||
count += 1
|
||||
|
||||
last_id = records[-1].id
|
||||
log.info(f"Queued {count} thumbnail tasks so far...")
|
||||
|
||||
log.info(f"Thumbnail regeneration complete: queued {count} tasks")
|
||||
return {'queued': count}
|
||||
|
||||
|
||||
@celery.task(name='app.tasks.thumbnail.regenerate_missing_thumbnails')
|
||||
def regenerate_missing_thumbnails(batch_size: int = 100):
|
||||
"""
|
||||
Generate thumbnails only for images that don't have them.
|
||||
|
||||
Args:
|
||||
batch_size: Number of images to process per batch
|
||||
|
||||
Returns:
|
||||
dict with count of queued tasks
|
||||
"""
|
||||
import os
|
||||
|
||||
log.info("Checking for missing thumbnails")
|
||||
|
||||
count = 0
|
||||
last_id = 0
|
||||
|
||||
while True:
|
||||
records = ImageRecord.query.filter(
|
||||
ImageRecord.id > last_id,
|
||||
ImageRecord.filepath.isnot(None)
|
||||
).order_by(ImageRecord.id.asc()).limit(batch_size).all()
|
||||
|
||||
if not records:
|
||||
break
|
||||
|
||||
for record in records:
|
||||
# Check if thumbnail is missing
|
||||
needs_thumb = False
|
||||
if not record.thumb_path:
|
||||
needs_thumb = True
|
||||
elif not os.path.exists(record.thumb_path):
|
||||
needs_thumb = True
|
||||
|
||||
if needs_thumb:
|
||||
generate_thumbnail_task.delay(record.id, overwrite=False)
|
||||
count += 1
|
||||
|
||||
last_id = records[-1].id
|
||||
|
||||
log.info(f"Missing thumbnail check complete: queued {count} tasks")
|
||||
return {'queued': count}
|
||||
Reference in New Issue
Block a user