Bryan Van Deusen
228 lines
8.1 KiB
Python
228 lines
8.1 KiB
Python
# app/tasks/sidecar.py
|
|
"""
|
|
Sidecar metadata extraction and application tasks.
|
|
|
|
Handles finding and applying metadata from Gallery-DL sidecar JSON files
|
|
to imported images.
|
|
"""
|
|
import logging
|
|
from datetime import datetime
|
|
|
|
from app.celery_app import celery
|
|
from app import db
|
|
from app.models import ImageRecord, Tag, PostMetadata
|
|
|
|
log = logging.getLogger('celery.tasks.sidecar')
|
|
|
|
|
|
@celery.task(bind=True, name='app.tasks.sidecar.apply_sidecar_metadata',
|
|
max_retries=2, default_retry_delay=30)
|
|
def apply_sidecar_metadata(self, image_id: int, sidecar_path: str):
|
|
"""
|
|
Apply metadata from a sidecar JSON file to an image record.
|
|
|
|
Extracts platform info, artist, post ID, dates, and generates
|
|
appropriate tags (source:platform, post:platform:artist:id).
|
|
|
|
Args:
|
|
image_id: ID of the ImageRecord to update
|
|
sidecar_path: Path to the sidecar JSON file
|
|
|
|
Returns:
|
|
dict with status and applied metadata info
|
|
"""
|
|
from app.utils.metadata_enrichment import (
|
|
enrich_from_sidecar, resolve_date_conflict, load_sidecar_metadata
|
|
)
|
|
|
|
record = ImageRecord.query.get(image_id)
|
|
if not record:
|
|
log.warning(f"Image {image_id} not found")
|
|
return {'error': 'Image not found'}
|
|
|
|
try:
|
|
# Load sidecar data
|
|
sidecar_data = load_sidecar_metadata(sidecar_path)
|
|
if not sidecar_data:
|
|
return {'status': 'no_metadata', 'reason': 'Failed to load sidecar'}
|
|
|
|
# Get enriched metadata
|
|
enriched = enrich_from_sidecar(sidecar_path, {'taken_at': record.taken_at})
|
|
|
|
if not enriched.get('raw_metadata'):
|
|
return {'status': 'no_metadata', 'reason': 'No metadata extracted'}
|
|
|
|
tags_added = 0
|
|
post_tag = None # Track the post tag for PostMetadata
|
|
|
|
# Update date if earlier
|
|
new_date = resolve_date_conflict(record.taken_at, enriched.get('taken_at'))
|
|
if new_date != record.taken_at:
|
|
record.taken_at = new_date
|
|
log.debug(f"Updated date for {image_id}: {new_date}")
|
|
|
|
# Apply tags from metadata
|
|
for tag_info in enriched.get('tags', []):
|
|
tag_name = tag_info.get('name')
|
|
tag_kind = tag_info.get('kind')
|
|
if not tag_name:
|
|
continue
|
|
|
|
tag = Tag.query.filter_by(name=tag_name).first()
|
|
if not tag:
|
|
tag = Tag(name=tag_name, kind=tag_kind)
|
|
db.session.add(tag)
|
|
db.session.flush()
|
|
log.debug(f"Created tag: {tag_name} ({tag_kind})")
|
|
|
|
if tag not in record.tags:
|
|
record.tags.append(tag)
|
|
tags_added += 1
|
|
|
|
# Track the post tag for PostMetadata creation
|
|
if tag_kind == 'post':
|
|
post_tag = tag
|
|
|
|
# Create or update PostMetadata for post tags
|
|
post_metadata_created = False
|
|
if post_tag and enriched.get('platform') and enriched.get('post_id'):
|
|
existing_pm = PostMetadata.query.filter_by(tag_id=post_tag.id).first()
|
|
if not existing_pm:
|
|
# Create new PostMetadata record
|
|
pm = PostMetadata(
|
|
tag_id=post_tag.id,
|
|
platform=enriched.get('platform'),
|
|
post_id=enriched.get('post_id'),
|
|
artist=enriched.get('artist'),
|
|
title=enriched.get('post_title'),
|
|
description=enriched.get('description'),
|
|
source_url=enriched.get('source_url'),
|
|
attachment_count=enriched.get('attachment_count', 0),
|
|
published_at=enriched.get('taken_at'),
|
|
)
|
|
db.session.add(pm)
|
|
post_metadata_created = True
|
|
log.debug(f"Created PostMetadata for tag {post_tag.name}")
|
|
else:
|
|
# Update existing PostMetadata if it has missing fields
|
|
updated = False
|
|
if not existing_pm.title and enriched.get('post_title'):
|
|
existing_pm.title = enriched.get('post_title')
|
|
updated = True
|
|
if not existing_pm.description and enriched.get('description'):
|
|
existing_pm.description = enriched.get('description')
|
|
updated = True
|
|
if not existing_pm.source_url and enriched.get('source_url'):
|
|
existing_pm.source_url = enriched.get('source_url')
|
|
updated = True
|
|
# Always update attachment count to get accurate count
|
|
if enriched.get('attachment_count', 0) > existing_pm.attachment_count:
|
|
existing_pm.attachment_count = enriched.get('attachment_count', 0)
|
|
updated = True
|
|
if updated:
|
|
log.debug(f"Updated PostMetadata for tag {post_tag.name}")
|
|
|
|
db.session.commit()
|
|
|
|
log.info(f"Applied sidecar metadata to {image_id}: "
|
|
f"platform={enriched.get('platform')}, tags_added={tags_added}, "
|
|
f"post_metadata_created={post_metadata_created}")
|
|
|
|
return {
|
|
'status': 'applied',
|
|
'platform': enriched.get('platform'),
|
|
'artist': enriched.get('artist'),
|
|
'post_id': enriched.get('post_id'),
|
|
'tags_added': tags_added,
|
|
'post_metadata_created': post_metadata_created
|
|
}
|
|
|
|
except Exception as e:
|
|
db.session.rollback()
|
|
log.error(f"Sidecar application failed for {image_id}: {e}", exc_info=True)
|
|
raise self.retry(exc=e)
|
|
|
|
|
|
@celery.task(name='app.tasks.sidecar.scan_and_match_sidecars')
|
|
def scan_and_match_sidecars(batch_id: int = None, limit: int = 1000):
|
|
"""
|
|
Scan for sidecar files and match them to imported images.
|
|
|
|
Looks at recently imported images and tries to find corresponding
|
|
sidecar JSON files for metadata enrichment.
|
|
|
|
Args:
|
|
batch_id: Optional batch ID to limit scope
|
|
limit: Maximum number of images to process
|
|
|
|
Returns:
|
|
dict with count of matched sidecars
|
|
"""
|
|
from app.utils.metadata_enrichment import find_sidecar_json
|
|
|
|
log.info(f"Scanning for unmatched sidecars (batch={batch_id}, limit={limit})")
|
|
|
|
# Build query for images that might need sidecar matching
|
|
query = ImageRecord.query.filter(
|
|
ImageRecord.filepath.isnot(None)
|
|
).order_by(ImageRecord.imported_at.desc())
|
|
|
|
if limit:
|
|
query = query.limit(limit)
|
|
|
|
matched = 0
|
|
already_has_source = 0
|
|
|
|
for record in query.all():
|
|
# Check if already has source tag (indicates sidecar was processed)
|
|
has_source_tag = any(
|
|
tag.kind == 'source' or tag.kind == 'post'
|
|
for tag in record.tags
|
|
)
|
|
if has_source_tag:
|
|
already_has_source += 1
|
|
continue
|
|
|
|
# Try to find sidecar
|
|
sidecar_path = find_sidecar_json(record.filepath)
|
|
if sidecar_path:
|
|
apply_sidecar_metadata.delay(record.id, sidecar_path)
|
|
matched += 1
|
|
|
|
log.info(f"Sidecar scan complete: matched={matched}, already_tagged={already_has_source}")
|
|
return {'matched': matched, 'already_tagged': already_has_source}
|
|
|
|
|
|
@celery.task(name='app.tasks.sidecar.reprocess_sidecars_for_tag')
|
|
def reprocess_sidecars_for_tag(tag_name: str):
|
|
"""
|
|
Reprocess sidecar metadata for all images with a specific tag.
|
|
|
|
Useful for re-running sidecar extraction after updates to
|
|
the metadata extraction logic.
|
|
|
|
Args:
|
|
tag_name: Name of the tag to filter by
|
|
|
|
Returns:
|
|
dict with count of queued tasks
|
|
"""
|
|
from app.utils.metadata_enrichment import find_sidecar_json
|
|
|
|
tag = Tag.query.filter_by(name=tag_name).first()
|
|
if not tag:
|
|
return {'error': f'Tag not found: {tag_name}'}
|
|
|
|
queued = 0
|
|
for record in tag.images:
|
|
if not record.filepath:
|
|
continue
|
|
|
|
sidecar_path = find_sidecar_json(record.filepath)
|
|
if sidecar_path:
|
|
apply_sidecar_metadata.delay(record.id, sidecar_path)
|
|
queued += 1
|
|
|
|
log.info(f"Queued {queued} sidecar reprocessing tasks for tag: {tag_name}")
|
|
return {'queued': queued, 'tag': tag_name}
|