Optimize scan process with quick/deep modes and archive tag merging

Quick scan (default):
- Pre-load all ImportTask records for O(1) duplicate checks
- Batch task creation commits (50 at a time)
- Cache import settings (5-min TTL)
- Skip pHash comparison for speed

Deep scan (on-demand):
- Full reprocessing of all files
- pHash similarity detection
- Optional thumbnail regeneration
- Optional sidecar re-application

Archive tag merging:
- Add archive tags to existing images when duplicates found
- Works for both hash and pHash duplicate detection

Also fixes:
- Add missing source/post tag icons to gallery templates
This commit is contained in:
Bryan Van Deusen
2026-01-31 15:18:43 -05:00
parent d0fcde38e8
commit 042a69f9c3
7 changed files with 595 additions and 171 deletions
+35 -4
View File
@@ -1114,20 +1114,51 @@ def delete_filtered_images():
def trigger_import_celery():
"""
Trigger a directory scan and import via Celery.
Replaces the old flag-file mechanism.
Supports two modes:
- Quick scan (default): Fast O(1) duplicate detection using pre-loaded lookups.
Only queues truly new files. Skips pHash comparison for speed.
- Deep scan: Full reprocessing of all files. Re-extracts metadata,
runs pHash comparison, optionally regenerates thumbnails and re-applies sidecars.
POST params:
source_dir: Source directory (default: /import)
dest_dir: Destination directory (default: /images)
deep: Set to 'true' for deep scan mode
regenerate_thumbnails: For deep scan, regenerate all thumbnails (default: true)
reapply_sidecars: For deep scan, re-apply sidecar metadata (default: true)
verify_hashes: For deep scan, verify content hashes (default: true)
"""
try:
from app.tasks.scan import scan_directory
from app.tasks.scan import scan_directory, deep_scan_directory
source_dir = request.form.get('source_dir', '/import')
dest_dir = request.form.get('dest_dir', '/images')
is_deep = request.form.get('deep', 'false').lower() == 'true'
result = scan_directory.delay(source_dir, dest_dir)
if is_deep:
# Deep scan with full reprocessing
regenerate_thumbnails = request.form.get('regenerate_thumbnails', 'true').lower() == 'true'
reapply_sidecars = request.form.get('reapply_sidecars', 'true').lower() == 'true'
verify_hashes = request.form.get('verify_hashes', 'true').lower() == 'true'
result = deep_scan_directory.delay(
source_dir, dest_dir,
regenerate_thumbnails=regenerate_thumbnails,
reapply_sidecars=reapply_sidecars,
verify_hashes=verify_hashes
)
message = 'Deep import scan started'
else:
# Quick scan with optimized lookups
result = scan_directory.delay(source_dir, dest_dir)
message = 'Quick import scan started'
return jsonify({
'ok': True,
'task_id': result.id,
'message': 'Import scan started'
'message': message,
'mode': 'deep' if is_deep else 'quick'
})
except Exception as e:
return jsonify({'ok': False, 'error': str(e)}), 500