bvandeusen/imagerepo
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize scan process with quick/deep modes and archive tag merging

Browse Source 
Quick scan (default):
- Pre-load all ImportTask records for O(1) duplicate checks
- Batch task creation commits (50 at a time)
- Cache import settings (5-min TTL)
- Skip pHash comparison for speed

Deep scan (on-demand):
- Full reprocessing of all files
- pHash similarity detection
- Optional thumbnail regeneration
- Optional sidecar re-application

Archive tag merging:
- Add archive tags to existing images when duplicates found
- Works for both hash and pHash duplicate detection

Also fixes:
- Add missing source/post tag icons to gallery templates
This commit is contained in:
Bryan Van Deusen
2026-01-31 15:18:43 -05:00
parent d0fcde38e8
commit 042a69f9c3
7 changed files with 595 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/tasks/import_file.py
+144 -35
View File
@@ -4,12 +4,19 @@ File import tasks for images, videos, and archives.
These tasks handle the actual import of individual files,
including filtering, duplicate detection, and storage.
Supports two modes:
- Quick import (default): Fast duplicate detection by content hash only.
Skips pHash comparison for speed.
- Deep import (via deep_scan): Full reprocessing with pHash comparison,
optional thumbnail regeneration, and sidecar re-application.
"""
import os
import shutil
import tempfile
import logging
from datetime import datetime
from functools import lru_cache
import imagehash
@@ -19,6 +26,30 @@ from app.models import ImportTask, ImageRecord, Tag, ArchiveRecord
log = logging.getLogger('celery.tasks.import_file')
# Cache import settings to avoid repeated DB/file reads
_settings_cache = None
_settings_cache_time = None
SETTINGS_CACHE_TTL = 300 # 5 minutes
def _get_cached_settings() -> dict:
"""
Get import settings with caching.
Caches for 5 minutes to avoid repeated DB/file reads during batch imports.
"""
global _settings_cache, _settings_cache_time
from app.utils.image_importer import load_import_settings
now = datetime.utcnow()
if _settings_cache is None or _settings_cache_time is None:
_settings_cache = load_import_settings()
_settings_cache_time = now
elif (now - _settings_cache_time).total_seconds() > SETTINGS_CACHE_TTL:
_settings_cache = load_import_settings()
_settings_cache_time = now
return _settings_cache
@celery.task(bind=True, name='app.tasks.import_file.import_media_file',
max_retries=3, default_retry_delay=60)
@@ -26,13 +57,19 @@ def import_media_file(self, task_id: int):
"""
Import a single media file (image or video).
Supports two modes based on context.deep_scan flag:
- Quick mode (default): Fast duplicate detection by content hash only.
Skips pHash comparison for speed.
- Deep mode: Full reprocessing with pHash comparison, optional
thumbnail regeneration, and sidecar re-application.
Steps:
1. Load ImportTask record
2. Apply filters (dimensions, transparency, single-color)
3. Check for duplicates (hash and pHash)
3. Check for duplicates (hash, and pHash in deep mode)
4. Copy to content-addressed storage
5. Create ImageRecord
6. Queue thumbnail task
6. Generate thumbnail
7. Queue sidecar metadata task
8. Update ImportTask status
@@ -47,7 +84,7 @@ def import_media_file(self, task_id: int):
from app.utils.image_importer import (
extract_metadata, calculate_hash, calculate_perceptual_hash,
is_mostly_transparent, is_mostly_single_color, find_similar_image,
build_hashed_dest_path, load_import_settings, supersede_image,
build_hashed_dest_path, supersede_image,
generate_thumbnail, generate_video_thumbnail_mirrored,
transcode_video_to_mp4, needs_transcode
)
@@ -68,10 +105,16 @@ def import_media_file(self, task_id: int):
artist = context.get('artist', 'unknown')
dest_dir = context.get('dest_dir', '/images')
log.info(f"Processing: {src_path}")
# Deep scan options
is_deep_scan = context.get('deep_scan', False)
regenerate_thumbnails = context.get('regenerate_thumbnails', False)
reapply_sidecars = context.get('reapply_sidecars', False)
verify_hashes = context.get('verify_hashes', False)
# Load settings
settings = load_import_settings()
log.info(f"Processing: {src_path} (deep_scan={is_deep_scan})")
# Load settings with caching (avoids repeated DB reads)
settings = _get_cached_settings()
# Validate file exists
if not os.path.exists(src_path):
@@ -113,43 +156,75 @@ def import_media_file(self, task_id: int):
# Check for exact duplicate by hash
existing = ImageRecord.query.filter_by(hash=file_hash).first()
if existing:
# Still try to enrich with sidecar metadata
# Merge archive tag if this file came from an archive
# This ensures the existing image gets tagged with all archives it appears in
archive_tag_id = context.get('archive_tag_id')
if archive_tag_id:
_add_archive_tag(existing, archive_tag_id)
db.session.commit()
log.info(f"Added archive tag {archive_tag_id} to existing image {existing.id}")
# In deep scan mode with reapply_sidecars, always try to enrich
# In quick mode, also try if sidecar exists
sidecar_path = find_sidecar_json(src_path)
if sidecar_path:
if sidecar_path and (is_deep_scan and reapply_sidecars or not is_deep_scan):
apply_sidecar_metadata.delay(existing.id, sidecar_path)
# In deep scan with regenerate_thumbnails, regenerate even for duplicates
if is_deep_scan and regenerate_thumbnails:
_regenerate_thumbnail(existing)
return _skip_task(task, 'Duplicate by hash', result_image_id=existing.id)
# Compute pHash for similarity check
phash = calculate_perceptual_hash(src_path)
phash_threshold = settings.get('phash_threshold', 10)
# pHash similarity check
# Quick mode: SKIP pHash comparison (relies on content hash for duplicates)
# Deep mode: Full pHash comparison for similarity detection
phash = None
if is_deep_scan:
phash = calculate_perceptual_hash(src_path)
phash_threshold = settings.get('phash_threshold', 10)
if phash and phash_threshold > 0:
# Load existing pHashes for comparison
existing_phashes = [
(imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height, img.id)
for img in ImageRecord.query.filter(ImageRecord.perceptual_hash.isnot(None)).all()
]
if phash and phash_threshold > 0:
# Load existing pHashes for comparison
# Note: In production with millions of images, consider using
# a spatial index or pre-computed pHash buckets
existing_phashes = [
(imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height, img.id)
for img in ImageRecord.query.filter(ImageRecord.perceptual_hash.isnot(None)).all()
]
relationship, similar_id = find_similar_image(
phash, metadata['width'], metadata['height'],
existing_phashes, threshold=phash_threshold
)
relationship, similar_id = find_similar_image(
phash, metadata['width'], metadata['height'],
existing_phashes, threshold=phash_threshold
)
if relationship == 'larger_exists':
# Still try to enrich the existing larger image with sidecar metadata
sidecar_path = find_sidecar_json(src_path)
if sidecar_path and similar_id:
apply_sidecar_metadata.delay(similar_id, sidecar_path)
return _skip_task(task, 'Similar larger image exists', result_image_id=similar_id)
if relationship == 'larger_exists':
# Merge archive tag if this file came from an archive
archive_tag_id = context.get('archive_tag_id')
if archive_tag_id and similar_id:
similar_record = ImageRecord.query.get(similar_id)
if similar_record:
_add_archive_tag(similar_record, archive_tag_id)
db.session.commit()
log.info(f"Added archive tag {archive_tag_id} to similar image {similar_id}")
# Handle supersede case - new image is larger than existing
if relationship == 'smaller_exists' and settings.get('supersede_smaller', True) and similar_id:
old_record = ImageRecord.query.get(similar_id)
if old_record:
return _supersede_existing(
task, old_record, src_path, file_hash, phash,
metadata, artist, dest_dir, filename
)
# Still try to enrich the existing larger image with sidecar metadata
sidecar_path = find_sidecar_json(src_path)
if sidecar_path and similar_id:
apply_sidecar_metadata.delay(similar_id, sidecar_path)
return _skip_task(task, 'Similar larger image exists', result_image_id=similar_id)
# Handle supersede case - new image is larger than existing
if relationship == 'smaller_exists' and settings.get('supersede_smaller', True) and similar_id:
old_record = ImageRecord.query.get(similar_id)
if old_record:
return _supersede_existing(
task, old_record, src_path, file_hash, phash,
metadata, artist, dest_dir, filename
)
else:
# Quick mode: only compute pHash for storage, not comparison
phash = calculate_perceptual_hash(src_path)
# Normal import - copy to destination
target_dir = os.path.join(dest_dir, artist)
@@ -474,6 +549,40 @@ def _add_archive_tag(record: ImageRecord, archive_tag_id: int):
record.tags.append(tag)
def _regenerate_thumbnail(record: ImageRecord) -> bool:
"""
Regenerate thumbnail for an existing image record.
Used by deep scan to refresh thumbnails.
Returns:
True if thumbnail was regenerated, False on error
"""
from app.utils.image_importer import generate_thumbnail, generate_video_thumbnail_mirrored
try:
filepath = record.filepath
if not filepath or not os.path.exists(filepath):
log.warning(f"Cannot regenerate thumbnail: file not found {filepath}")
return False
lower = filepath.lower()
if lower.endswith(('.mp4', '.mov', '.webm', '.avi', '.mkv')):
new_thumb = generate_video_thumbnail_mirrored(filepath)
else:
new_thumb = generate_thumbnail(filepath, overwrite=True)
if new_thumb:
record.thumb_path = new_thumb
db.session.commit()
log.info(f"Regenerated thumbnail for image {record.id}")
return True
return False
except Exception as e:
log.warning(f"Failed to regenerate thumbnail for image {record.id}: {e}")
return False
def _supersede_existing(task: ImportTask, old_record: ImageRecord, src_path: str,
file_hash: str, phash, metadata: dict, artist: str,
dest_dir: str, filename: str) -> dict: