# app/tasks/sidecar.py """ Sidecar metadata extraction and application tasks. Handles finding and applying metadata from Gallery-DL sidecar JSON files to imported images. """ import logging from datetime import datetime from app.celery_app import celery from app import db from app.models import ImageRecord, Tag, PostMetadata log = logging.getLogger('celery.tasks.sidecar') @celery.task(bind=True, name='app.tasks.sidecar.apply_sidecar_metadata', max_retries=2, default_retry_delay=30) def apply_sidecar_metadata(self, image_id: int, sidecar_path: str): """ Apply metadata from a sidecar JSON file to an image record. Extracts platform info, artist, post ID, dates, and generates appropriate tags (source:platform, post:platform:artist:id). Args: image_id: ID of the ImageRecord to update sidecar_path: Path to the sidecar JSON file Returns: dict with status and applied metadata info """ from app.utils.metadata_enrichment import ( enrich_from_sidecar, resolve_date_conflict, load_sidecar_metadata ) record = ImageRecord.query.get(image_id) if not record: log.warning(f"Image {image_id} not found") return {'error': 'Image not found'} try: # Load sidecar data sidecar_data = load_sidecar_metadata(sidecar_path) if not sidecar_data: return {'status': 'no_metadata', 'reason': 'Failed to load sidecar'} # Get enriched metadata enriched = enrich_from_sidecar(sidecar_path, {'taken_at': record.taken_at}) if not enriched.get('raw_metadata'): return {'status': 'no_metadata', 'reason': 'No metadata extracted'} tags_added = 0 post_tag = None # Track the post tag for PostMetadata # Update date if earlier new_date = resolve_date_conflict(record.taken_at, enriched.get('taken_at')) if new_date != record.taken_at: record.taken_at = new_date log.debug(f"Updated date for {image_id}: {new_date}") # Apply tags from metadata for tag_info in enriched.get('tags', []): tag_name = tag_info.get('name') tag_kind = tag_info.get('kind') if not tag_name: continue tag = Tag.query.filter_by(name=tag_name).first() if not tag: tag = Tag(name=tag_name, kind=tag_kind) db.session.add(tag) db.session.flush() log.debug(f"Created tag: {tag_name} ({tag_kind})") if tag not in record.tags: record.tags.append(tag) tags_added += 1 # Track the post tag for PostMetadata creation if tag_kind == 'post': post_tag = tag # Create or update PostMetadata for post tags post_metadata_created = False if post_tag and enriched.get('platform') and enriched.get('post_id'): existing_pm = PostMetadata.query.filter_by(tag_id=post_tag.id).first() if not existing_pm: # Create new PostMetadata record pm = PostMetadata( tag_id=post_tag.id, platform=enriched.get('platform'), post_id=enriched.get('post_id'), artist=enriched.get('artist'), title=enriched.get('post_title'), description=enriched.get('description'), source_url=enriched.get('source_url'), attachment_count=enriched.get('attachment_count', 0), published_at=enriched.get('taken_at'), ) db.session.add(pm) post_metadata_created = True log.debug(f"Created PostMetadata for tag {post_tag.name}") else: # Update existing PostMetadata if it has missing fields updated = False if not existing_pm.title and enriched.get('post_title'): existing_pm.title = enriched.get('post_title') updated = True if not existing_pm.description and enriched.get('description'): existing_pm.description = enriched.get('description') updated = True if not existing_pm.source_url and enriched.get('source_url'): existing_pm.source_url = enriched.get('source_url') updated = True # Always update attachment count to get accurate count if enriched.get('attachment_count', 0) > existing_pm.attachment_count: existing_pm.attachment_count = enriched.get('attachment_count', 0) updated = True if updated: log.debug(f"Updated PostMetadata for tag {post_tag.name}") db.session.commit() log.info(f"Applied sidecar metadata to {image_id}: " f"platform={enriched.get('platform')}, tags_added={tags_added}, " f"post_metadata_created={post_metadata_created}") return { 'status': 'applied', 'platform': enriched.get('platform'), 'artist': enriched.get('artist'), 'post_id': enriched.get('post_id'), 'tags_added': tags_added, 'post_metadata_created': post_metadata_created } except Exception as e: db.session.rollback() log.error(f"Sidecar application failed for {image_id}: {e}", exc_info=True) raise self.retry(exc=e) @celery.task(name='app.tasks.sidecar.scan_and_match_sidecars') def scan_and_match_sidecars(batch_id: int = None, limit: int = 1000): """ Scan for sidecar files and match them to imported images. Looks at recently imported images and tries to find corresponding sidecar JSON files for metadata enrichment. Args: batch_id: Optional batch ID to limit scope limit: Maximum number of images to process Returns: dict with count of matched sidecars """ from app.utils.metadata_enrichment import find_sidecar_json log.info(f"Scanning for unmatched sidecars (batch={batch_id}, limit={limit})") # Build query for images that might need sidecar matching query = ImageRecord.query.filter( ImageRecord.filepath.isnot(None) ).order_by(ImageRecord.imported_at.desc()) if limit: query = query.limit(limit) matched = 0 already_has_source = 0 for record in query.all(): # Check if already has source tag (indicates sidecar was processed) has_source_tag = any( tag.kind == 'source' or tag.kind == 'post' for tag in record.tags ) if has_source_tag: already_has_source += 1 continue # Try to find sidecar sidecar_path = find_sidecar_json(record.filepath) if sidecar_path: apply_sidecar_metadata.delay(record.id, sidecar_path) matched += 1 log.info(f"Sidecar scan complete: matched={matched}, already_tagged={already_has_source}") return {'matched': matched, 'already_tagged': already_has_source} @celery.task(name='app.tasks.sidecar.reprocess_sidecars_for_tag') def reprocess_sidecars_for_tag(tag_name: str): """ Reprocess sidecar metadata for all images with a specific tag. Useful for re-running sidecar extraction after updates to the metadata extraction logic. Args: tag_name: Name of the tag to filter by Returns: dict with count of queued tasks """ from app.utils.metadata_enrichment import find_sidecar_json tag = Tag.query.filter_by(name=tag_name).first() if not tag: return {'error': f'Tag not found: {tag_name}'} queued = 0 for record in tag.images: if not record.filepath: continue sidecar_path = find_sidecar_json(record.filepath) if sidecar_path: apply_sidecar_metadata.delay(record.id, sidecar_path) queued += 1 log.info(f"Queued {queued} sidecar reprocessing tasks for tag: {tag_name}") return {'queued': queued, 'tag': tag_name}