bvandeusen/imagerepo
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed metadata time import error and moved quest status widget to top of settings.

Browse Source
This commit is contained in:
Bryan Van Deusen
2026-01-21 11:30:45 -05:00
parent dc45a09681
commit 041a3dbfb4
7 changed files with 108 additions and 760 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/main.py
-71
View File
@@ -490,55 +490,6 @@ def settings():
return render_template('settings.html')
@main.route('/trigger-thumbnail-generation', methods=['POST'])
def trigger_thumbnail_generation():
try:
with open('/import/thumbnail.flag', 'w') as f:
f.write("trigger")
flash("Thumbnail regeneration has been triggered.", "success")
except Exception as e:
flash(f"Failed to trigger thumbnail regeneration: {e}", "danger")
return redirect(url_for('main.settings'))
@main.route('/import-images')
def trigger_image_import():
with open('/import/trigger.flag', 'w') as f:
f.write('start')
flash("Image import triggered.", "success")
return redirect(url_for('main.settings'))
@main.route('/reset-db', methods=['POST'])
def reset_db():
"""
Safe reset that works with Postgres (TRUNCATE ... CASCADE).
Falls back to ordered deletes if TRUNCATE isn't supported (e.g., SQLite).
"""
try:
# Postgres fast path
db.session.execute(text("""
TRUNCATE TABLE image_tags, archive_record, image_record, tag
RESTART IDENTITY CASCADE
"""))
db.session.commit()
except Exception:
# Fallback: manual delete order
db.session.execute(image_tags.delete())
ArchiveRecord.query.delete()
ImageRecord.query.delete()
Tag.query.delete()
db.session.commit()
# Optional: clear thumbs on disk to regenerate fresh
shutil.rmtree("/images/thumbs", ignore_errors=True)
os.makedirs("/images/thumbs", exist_ok=True)
flash("Database has been reset. All records removed.", "info")
return redirect(url_for('main.settings'))
# ----------------------------
# Tag add/remove endpoints
# ----------------------------
@@ -870,28 +821,6 @@ def save_import_settings():
return jsonify(ok=False, error=str(e)), 500
@main.get("/api/import-stats")
def get_import_stats():
"""Get import statistics including filtered image counts."""
import json
stats_path = "/import/stats.json"
try:
if os.path.exists(stats_path):
with open(stats_path, "r") as f:
stats = json.load(f)
else:
stats = {
"total_processed": 0,
"total_imported": 0,
"filtered_by_dimension": 0,
"filtered_by_transparency": 0,
"filtered_by_duplicate": 0,
}
return jsonify(ok=True, stats=stats)
except Exception as e:
return jsonify(ok=False, error=str(e)), 500
# ----------------------------
# Image filter/cleanup endpoints
# ----------------------------
app/static/style.css
+8 -15
View File
@@ -953,37 +953,30 @@ header {
------------------------------------------------------------------------------*/
.settings-grid {
display: grid;
grid-template-columns: repeat(3, 1fr);
grid-template-columns: repeat(2, 1fr);
gap: 1.5rem;
max-width: 1600px;
margin: 1.5rem auto;
padding: 0 1.5rem;
align-items: start;
}
.settings-grid-full {
max-width: 1600px;
margin: 1.5rem auto;
padding: 0 1.5rem;
}
.settings-column {
display: flex;
flex-direction: column;
gap: 1.5rem;
}
@media (max-width: 1200px) {
.settings-grid {
grid-template-columns: repeat(2, 1fr);
}
.settings-column:first-child {
grid-column: 1 / -1;
flex-direction: row;
}
.settings-column:first-child > .settings-container {
flex: 1;
}
}
@media (max-width: 768px) {
.settings-grid {
grid-template-columns: 1fr;
padding: 0 1rem;
}
.settings-column:first-child {
flex-direction: column;
.settings-grid-full {
padding: 0 1rem;
}
}
.settings-container {
app/tasks/import_file.py
+6 -5
View File
@@ -273,8 +273,9 @@ def import_archive(self, task_id: int):
Returns:
dict with status and counts
"""
from app.utils.image_importer import calculate_hash, compute_next_archive_tag_name
from pyunpack import Archive
from app.utils.image_importer import (
calculate_hash, compute_next_archive_tag_name, extract_archive_resilient, ExtractError
)
task = ImportTask.query.get(task_id)
if not task:
@@ -305,13 +306,13 @@ def import_archive(self, task_id: int):
# Get archive size
archive_size = os.path.getsize(archive_path)
# Extract archive
# Extract archive using resilient extractor (tries unar then 7z for RAR)
tmpdir = tempfile.mkdtemp(prefix='extract_')
log.info(f"Extracting to: {tmpdir}")
try:
Archive(archive_path).extractall(tmpdir)
except Exception as e:
extract_archive_resilient(archive_path, tmpdir)
except ExtractError as e:
log.error(f"Extraction failed: {e}")
if tmpdir and os.path.exists(tmpdir):
shutil.rmtree(tmpdir, ignore_errors=True)
app/templates/settings.html
+81 -143
View File
@@ -4,38 +4,93 @@
{% block content %}
<h1 style="text-align:center; margin-top: 1rem;">Settings</h1>
<div class="settings-grid">
<!-- Column 1: Info & Admin Actions -->
<div class="settings-column">
<div class="settings-container">
<div class="settings-info">
<h2>About These Tools</h2>
<p>Use these tools to manage your image library. You can trigger imports, configure filters, and clean up unwanted images.</p>
<p style="margin-top: 0.75rem;"><strong>Note:</strong> Resetting the database does not delete the actual image files from disk.</p>
<!-- Import Queue Status (Full Width at Top) -->
<div class="settings-grid-full">
<div class="settings-container">
<div class="settings-section">
<h2>Import Queue Status</h2>
<p class="settings-description">Monitor and control the Celery-based import queue system.</p>
<div id="queueStatusSection">
<div class="stats-grid" style="grid-template-columns: repeat(6, 1fr);">
<div class="stat-item">
<span class="stat-value" id="queuePending">-</span>
<span class="stat-label">Pending</span>
</div>
<div class="stat-item">
<span class="stat-value" id="queueQueued">-</span>
<span class="stat-label">Queued</span>
</div>
<div class="stat-item">
<span class="stat-value" id="queueProcessing">-</span>
<span class="stat-label">Processing</span>
</div>
<div class="stat-item">
<span class="stat-value" id="queueComplete">-</span>
<span class="stat-label">Complete</span>
</div>
<div class="stat-item">
<span class="stat-value" id="queueSkipped">-</span>
<span class="stat-label">Skipped</span>
</div>
<div class="stat-item">
<span class="stat-value" id="queueFailed">-</span>
<span class="stat-label">Failed</span>
</div>
</div>
<div id="activeBatchInfo" class="import-stats" style="margin-top: 1rem; display: none;">
<h3>Active Batch</h3>
<p>Processing <strong id="batchDir">-</strong></p>
<div class="batch-progress">
<div class="progress-bar">
<div class="progress-fill" id="batchProgressFill" style="width: 0%"></div>
</div>
<span id="batchProgressText">0 / 0</span>
</div>
</div>
<div class="queue-actions" style="margin-top: 1rem; display: flex; gap: 0.75rem; flex-wrap: wrap;">
<button id="triggerImportBtn" class="btn primary-btn">Trigger Import Scan</button>
<button id="retryFailedBtn" class="btn warning-btn">Retry Failed Tasks</button>
<button id="clearCompletedBtn" class="btn secondary-btn">Clear Completed</button>
<button id="regenMissingThumbsBtn" class="btn secondary-btn">Regenerate Missing Thumbnails</button>
</div>
<div id="celeryWorkerInfo" style="margin-top: 1rem;">
<p style="color: var(--text-muted); font-size: 0.85rem;">
Workers: <strong id="workerCount">-</strong> |
Active Tasks: <strong id="activeTasks">-</strong>
</p>
</div>
</div>
</div>
<div class="settings-container">
<div class="settings-section">
<h2>Admin Actions</h2>
<div class="settings-actions">
<form action="{{ url_for('main.trigger_image_import') }}" method="get">
<button type="submit" class="btn primary-btn">Trigger Image Import</button>
</form>
<form action="{{ url_for('main.trigger_thumbnail_generation') }}" method="post" onsubmit="return confirm('Are you sure you want to regenerate all thumbnails?');">
<button type="submit" class="btn warning-btn">Regenerate Thumbnails</button>
</form>
<form action="{{ url_for('main.reset_db') }}" method="post" onsubmit="return confirm('Are you sure you want to delete all image records? This action cannot be undone.');">
<button type="submit" class="btn danger-btn">Reset Image Database</button>
</form>
<!-- Recent Tasks Table -->
<div id="recentTasksSection" style="margin-top: 1.5rem;">
<h3>Recent Tasks</h3>
<div class="tasks-table-wrapper" style="max-height: 300px; overflow-y: auto;">
<table class="tasks-table">
<thead>
<tr>
<th>File</th>
<th>Type</th>
<th>Status</th>
<th>Error</th>
<th>Time</th>
</tr>
</thead>
<tbody id="recentTasksBody">
<tr><td colspan="5" style="text-align: center; color: var(--text-muted);">Loading...</td></tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
<!-- Column 2: Import Filters -->
<div class="settings-grid">
<!-- Column 1: Import Filters -->
<div class="settings-column">
<div class="settings-container">
<div class="settings-section">
@@ -81,33 +136,11 @@
<button type="submit" class="btn primary-btn" style="width: 100%;">Save Filter Settings</button>
</form>
<div id="importStats" class="import-stats">
<h3>Last Import Statistics</h3>
<div class="stats-grid">
<div class="stat-item">
<span class="stat-value" id="statProcessed">-</span>
<span class="stat-label">Processed</span>
</div>
<div class="stat-item">
<span class="stat-value" id="statImported">-</span>
<span class="stat-label">Imported</span>
</div>
<div class="stat-item">
<span class="stat-value" id="statDimension">-</span>
<span class="stat-label">Filtered (size)</span>
</div>
<div class="stat-item">
<span class="stat-value" id="statTransparency">-</span>
<span class="stat-label">Filtered (transparent)</span>
</div>
</div>
</div>
</div>
</div>
</div>
<!-- Column 3: Deletion Tools -->
<!-- Column 2: Deletion Tools -->
<div class="settings-column">
<div class="settings-container">
<div class="settings-section">
@@ -246,89 +279,6 @@
</div>
</div>
<!-- Import Queue Status (Full Width) -->
<div class="settings-container" style="max-width: 1200px; margin: 2rem auto;">
<div class="settings-section">
<h2>Import Queue Status</h2>
<p class="settings-description">Monitor and control the Celery-based import queue system.</p>
<div id="queueStatusSection">
<div class="stats-grid" style="grid-template-columns: repeat(6, 1fr);">
<div class="stat-item">
<span class="stat-value" id="queuePending">-</span>
<span class="stat-label">Pending</span>
</div>
<div class="stat-item">
<span class="stat-value" id="queueQueued">-</span>
<span class="stat-label">Queued</span>
</div>
<div class="stat-item">
<span class="stat-value" id="queueProcessing">-</span>
<span class="stat-label">Processing</span>
</div>
<div class="stat-item">
<span class="stat-value" id="queueComplete">-</span>
<span class="stat-label">Complete</span>
</div>
<div class="stat-item">
<span class="stat-value" id="queueSkipped">-</span>
<span class="stat-label">Skipped</span>
</div>
<div class="stat-item">
<span class="stat-value" id="queueFailed">-</span>
<span class="stat-label">Failed</span>
</div>
</div>
<div id="activeBatchInfo" class="import-stats" style="margin-top: 1rem; display: none;">
<h3>Active Batch</h3>
<p>Processing <strong id="batchDir">-</strong></p>
<div class="batch-progress">
<div class="progress-bar">
<div class="progress-fill" id="batchProgressFill" style="width: 0%"></div>
</div>
<span id="batchProgressText">0 / 0</span>
</div>
</div>
<div class="queue-actions" style="margin-top: 1rem; display: flex; gap: 0.75rem; flex-wrap: wrap;">
<button id="triggerImportBtn" class="btn primary-btn">Trigger Import Scan</button>
<button id="retryFailedBtn" class="btn warning-btn">Retry Failed Tasks</button>
<button id="clearCompletedBtn" class="btn secondary-btn">Clear Completed</button>
<button id="regenMissingThumbsBtn" class="btn secondary-btn">Regenerate Missing Thumbnails</button>
</div>
<div id="celeryWorkerInfo" style="margin-top: 1rem;">
<p style="color: var(--text-muted); font-size: 0.85rem;">
Workers: <strong id="workerCount">-</strong> |
Active Tasks: <strong id="activeTasks">-</strong>
</p>
</div>
</div>
<!-- Recent Tasks Table -->
<div id="recentTasksSection" style="margin-top: 1.5rem;">
<h3>Recent Tasks</h3>
<div class="tasks-table-wrapper" style="max-height: 300px; overflow-y: auto;">
<table class="tasks-table">
<thead>
<tr>
<th>File</th>
<th>Type</th>
<th>Status</th>
<th>Error</th>
<th>Time</th>
</tr>
</thead>
<tbody id="recentTasksBody">
<tr><td colspan="5" style="text-align: center; color: var(--text-muted);">Loading...</td></tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
<style>
.tasks-table {
width: 100%;
@@ -797,18 +747,6 @@ document.addEventListener('DOMContentLoaded', () => {
}
});
// Load import stats
fetch('/api/import-stats')
.then(r => r.json())
.then(data => {
if (data.ok) {
document.getElementById('statProcessed').textContent = data.stats.total_processed || 0;
document.getElementById('statImported').textContent = data.stats.total_imported || 0;
document.getElementById('statDimension').textContent = data.stats.filtered_by_dimension || 0;
document.getElementById('statTransparency').textContent = data.stats.filtered_by_transparency || 0;
}
});
// Load tags for deletion dropdown
loadTagsForDeletion();
app/utils/image_importer.py
+1 -524
View File
@@ -12,15 +12,12 @@ import re
import shutil
import subprocess
import tempfile
import time
import uuid
from PIL import Image, ImageOps
import exifread
import imagehash
from pyunpack import Archive
from app import db
from app.models import ImageRecord, Tag, ArchiveRecord
@@ -48,7 +45,6 @@ ALLOWED_ARCHIVE_EXTS = (
# Import filter settings (loaded from file or env)
IMPORT_SETTINGS_PATH = "/import/settings.json"
IMPORT_STATS_PATH = "/import/stats.json"
def load_import_settings() -> dict:
@@ -92,15 +88,6 @@ def load_import_settings() -> dict:
return defaults
def save_import_stats(stats: dict) -> None:
"""Save import statistics to file."""
try:
with open(IMPORT_STATS_PATH, "w") as f:
json.dump(stats, f, indent=2)
except Exception as e:
print(f"[WARN] Failed to save import stats: {e}")
def is_mostly_transparent(file_path: str, threshold: float = 0.9) -> bool:
"""
Check if an image is mostly transparent (> threshold % of pixels are transparent).
@@ -398,298 +385,7 @@ def extract_archive_resilient(archive_path: str, out_dir: str) -> None:
# =============================================================================
# Core: Import task & archive processing
# =============================================================================
def import_images_task(source_dir: str, dest_dir: str) -> str:
"""
Walk the import tree, process archives first (only first volumes),
then media files. Batches DB commits for efficiency.
"""
# Cleanup any stale extraction dirs from previous runs/crashes
cleanup_stale_tempdirs()
# Load import filter settings
settings = load_import_settings()
min_width = settings.get("min_width", 0)
min_height = settings.get("min_height", 0)
skip_transparent = settings.get("skip_transparent", False)
transparency_threshold = settings.get("transparency_threshold", 0.9)
skip_single_color = settings.get("skip_single_color", True)
single_color_threshold = settings.get("single_color_threshold", 0.95)
single_color_tolerance = settings.get("single_color_tolerance", 30)
phash_threshold = settings.get("phash_threshold", 2)
supersede_smaller = settings.get("supersede_smaller", True)
print(f"[INFO] Import settings: min_width={min_width}, min_height={min_height}, "
f"skip_transparent={skip_transparent}, transparency_threshold={transparency_threshold}, "
f"skip_single_color={skip_single_color}, single_color_threshold={single_color_threshold}, "
f"phash_threshold={phash_threshold}, supersede_smaller={supersede_smaller}")
# Statistics tracking
stats = {
"total_processed": 0,
"total_imported": 0,
"total_superseded": 0,
"filtered_by_dimension": 0,
"filtered_by_transparency": 0,
"filtered_by_single_color": 0,
"filtered_by_duplicate": 0,
}
imported: list[str] = []
batch_size = 10
batch_counter = 0
# Preload existing pHashes to avoid N+1 lookups
# Format: (phash, width, height, image_id) for supersede functionality
existing_phashes = [
(imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height, img.id)
for img in ImageRecord.query.filter(ImageRecord.perceptual_hash != None).all()
]
# Ensure thumbs base exists
(Path(dest_dir) / "thumbs").mkdir(parents=True, exist_ok=True)
# Optional free-space guard
min_free = float(os.environ.get(ENV_MINFREE, "0") or "0")
try:
ensure_free_space_or_raise(get_tmp_base(), min_free)
except Exception as e:
print(f"[WARN] Free-space check failed: {e}")
for artist_dir in os.listdir(source_dir):
artist_path = os.path.join(source_dir, artist_dir)
if not os.path.isdir(artist_path):
continue
for root, _, files in os.walk(artist_path):
for file in files:
full_path = os.path.join(root, file)
lower = file.lower()
# Archives first
if lower.endswith(ALLOWED_ARCHIVE_EXTS):
if not is_first_volume(full_path):
print(f"[SKIP] Not first archive volume: {file}")
continue
try:
count = process_archive(
archive_path=full_path,
dest_dir=dest_dir,
artist=artist_dir,
existing_phashes=existing_phashes,
settings=settings,
stats=stats
)
imported.append(f"[archive]{file}:{count}")
except Exception as e:
print(f"[WARN] Failed processing archive {file}: {e}")
continue
# Non-archive media
if not lower.endswith(ALLOWED_MEDIA_EXTS):
continue
stats["total_processed"] += 1
# Apply dimension filter
if min_width > 0 or min_height > 0:
md = extract_metadata(full_path)
if md.get("width") and md.get("height"):
if md["width"] < min_width or md["height"] < min_height:
print(f"[SKIP] Too small ({md['width']}x{md['height']}): {file}")
stats["filtered_by_dimension"] += 1
continue
# Apply transparency filter
if skip_transparent and lower.endswith((".png", ".gif", ".webp")):
if is_mostly_transparent(full_path, transparency_threshold):
print(f"[SKIP] Mostly transparent: {file}")
stats["filtered_by_transparency"] += 1
continue
# Apply single-color filter (skip images that are mostly one solid color)
if skip_single_color and lower.endswith((".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")):
if is_mostly_single_color(full_path, single_color_threshold, single_color_tolerance):
print(f"[SKIP] Mostly single color: {file}")
stats["filtered_by_single_color"] += 1
continue
added, phash, _rec, superseded = import_single_file(
src_path=full_path,
dest_dir=dest_dir,
artist=artist_dir,
existing_phashes=existing_phashes,
phash_threshold=phash_threshold,
supersede_smaller=supersede_smaller
)
if added:
imported.append(file)
if superseded:
stats["total_superseded"] += 1
else:
stats["total_imported"] += 1
elif _rec is None and phash is None:
# Was a duplicate
stats["filtered_by_duplicate"] += 1
# Add new image's phash to the tracking list (for fresh imports only, not supersedes)
if phash and _rec and not superseded:
existing_phashes.append((phash, _rec.width, _rec.height, _rec.id))
batch_counter += 1
if batch_counter >= batch_size:
db.session.commit()
print(f"[INFO] Committed batch of {batch_size} items.")
batch_counter = 0
if "sqlite" in db.engine.url.drivername:
time.sleep(1)
if batch_counter > 0:
db.session.commit()
print(f"[INFO] Committed final batch of {batch_counter} items.")
# Save stats
save_import_stats(stats)
summary = (f"Imported {stats['total_imported']} items, superseded {stats['total_superseded']} smaller versions. "
f"Filtered: {stats['filtered_by_dimension']} by size, {stats['filtered_by_transparency']} by transparency, "
f"{stats['filtered_by_single_color']} by single color, {stats['filtered_by_duplicate']} duplicates.")
print(f"[INFO] {summary}")
return summary
def process_archive(archive_path, dest_dir, artist, existing_phashes, settings=None, stats=None):
"""
Extracts archive to temp dir, imports/tag existing images, and records ArchiveRecord.
Creates an archive:* tag ONLY if at least one image was imported or tagged.
Returns number of items imported or tagged.
"""
if settings is None:
settings = load_import_settings()
if stats is None:
stats = {"total_processed": 0, "total_imported": 0, "total_superseded": 0,
"filtered_by_dimension": 0, "filtered_by_transparency": 0,
"filtered_by_single_color": 0, "filtered_by_duplicate": 0}
min_width = settings.get("min_width", 0)
min_height = settings.get("min_height", 0)
skip_transparent = settings.get("skip_transparent", False)
transparency_threshold = settings.get("transparency_threshold", 0.9)
skip_single_color = settings.get("skip_single_color", True)
single_color_threshold = settings.get("single_color_threshold", 0.95)
single_color_tolerance = settings.get("single_color_tolerance", 30)
phash_threshold = settings.get("phash_threshold", 2)
supersede_smaller = settings.get("supersede_smaller", True)
archive_size = os.path.getsize(archive_path)
archive_hash = calculate_hash(archive_path)
# Skip if this exact archive file was already processed (hash-based)
existing_archive = ArchiveRecord.query.filter_by(hash=archive_hash).first()
if existing_archive:
print(f"[SKIP] Archive already imported: {archive_path}")
return 0
tmpdir = tempfile.mkdtemp(prefix="extract_")
touched_records = [] # ImageRecord objects we imported or matched
imported_or_tagged = 0
created_tag = None
try:
# Extract using pyunpack
Archive(archive_path).extractall(tmpdir)
# Import / collect all media
for root, _, files in os.walk(tmpdir):
for file in files:
lower = file.lower()
if not lower.endswith(ALLOWED_MEDIA_EXTS):
continue
src_path = os.path.join(root, file)
stats["total_processed"] += 1
# Apply dimension filter
if min_width > 0 or min_height > 0:
md = extract_metadata(src_path)
if md.get("width") and md.get("height"):
if md["width"] < min_width or md["height"] < min_height:
print(f"[SKIP] Too small ({md['width']}x{md['height']}): {file}")
stats["filtered_by_dimension"] += 1
continue
# Apply transparency filter
if skip_transparent and lower.endswith((".png", ".gif", ".webp")):
if is_mostly_transparent(src_path, transparency_threshold):
print(f"[SKIP] Mostly transparent: {file}")
stats["filtered_by_transparency"] += 1
continue
# Apply single-color filter
if skip_single_color and lower.endswith((".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")):
if is_mostly_single_color(src_path, single_color_threshold, single_color_tolerance):
print(f"[SKIP] Mostly single color: {file}")
stats["filtered_by_single_color"] += 1
continue
added, _phash, rec, superseded = import_single_file(
src_path=src_path,
dest_dir=dest_dir,
artist=artist, # still used for auto artist:<name> tag
existing_phashes=existing_phashes,
commit=False,
phash_threshold=phash_threshold,
supersede_smaller=supersede_smaller
)
if rec is not None:
touched_records.append(rec)
if added:
imported_or_tagged += 1
if superseded:
stats["total_superseded"] += 1
else:
stats["total_imported"] += 1
elif rec is None and _phash is None:
stats["filtered_by_duplicate"] += 1
# Only now: create/attach the archive tag if we actually touched images
if touched_records:
# Incremental name like 'archive:{artist}/0001'
tag_name = compute_next_archive_tag_name(artist)
created_tag = Tag.query.filter_by(name=tag_name).first()
if not created_tag:
created_tag = Tag(name=tag_name, kind='archive')
db.session.add(created_tag)
db.session.flush()
newly_tagged = 0
for rec in touched_records:
if created_tag not in rec.tags:
rec.tags.append(created_tag)
newly_tagged += 1
imported_or_tagged += newly_tagged
# Record this archive as processed (tag may be None if no media)
arch = ArchiveRecord(
filename=archive_path,
file_size=archive_size,
hash=archive_hash,
artist=artist,
tag=created_tag
)
db.session.add(arch)
db.session.commit()
print(f"[INFO] Archive processed: {archive_path} items={imported_or_tagged}")
return imported_or_tagged
finally:
shutil.rmtree(tmpdir, ignore_errors=True)
# =============================================================================
# Core: Single-file import path (content-addressed storage)
# Content-addressed storage helpers
# =============================================================================
def build_hashed_dest_path(target_dir: str, original_name: str, content_hash: str) -> str:
@@ -712,225 +408,6 @@ def build_hashed_dest_path(target_dir: str, original_name: str, content_hash: st
i += 1
def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phashes,
commit: bool = True, phash_threshold: int = 2, supersede_smaller: bool = True):
"""
Import a single media file. If an equivalent file already exists, return that
record so callers can tag it (no duplicate import).
If a smaller visually-similar image exists and supersede_smaller is True,
the smaller image will be replaced with the larger one (preserving tags).
Also enriches metadata from Gallery-DL sidecar JSON files if present.
Returns (added: bool, phash_or_None, record_or_None, superseded: bool).
The superseded flag indicates if an existing smaller image was replaced.
"""
original_name = os.path.basename(src_path)
print(f"[INFO] Processing: {src_path}")
file_size = os.path.getsize(src_path)
# Compute content hash first (authoritative de-dup + used in filepath)
file_hash = calculate_hash(src_path)
existing = ImageRecord.query.filter_by(hash=file_hash).first()
if existing:
print(f"[SKIP] Duplicate by hash: {original_name}")
# Even for duplicates, try to enrich with sidecar metadata
_enrich_existing_record(existing, src_path)
return (False, None, existing, False)
# Optional info: same name+size (but different content) → we still import with hashed name
existing_ns = ImageRecord.query.filter_by(filename=original_name, file_size=file_size).first()
if existing_ns:
print(f"[INFO] Same name+size exists but different hash; storing with hashed name: {original_name}")
# Metadata & pHash similarity
metadata = extract_metadata(src_path)
if not metadata["width"] or not metadata["height"]:
print(f"[SKIP] Missing dimension data for {original_name}")
return (False, None, None, False)
phash = calculate_perceptual_hash(src_path)
# Check for similar images
superseded = False
if phash and phash_threshold > 0:
relationship, similar_id = find_similar_image(
phash, metadata["width"], metadata["height"], existing_phashes, threshold=phash_threshold
)
if relationship == "larger_exists":
print(f"[SKIP] {original_name} is visually similar to an existing larger image (threshold={phash_threshold}).")
return (False, phash, None, False)
elif relationship == "smaller_exists" and supersede_smaller and similar_id:
# Found a smaller version - supersede it
old_record = ImageRecord.query.get(similar_id)
if old_record:
# Enrich metadata from Gallery-DL sidecar JSON if present
enriched = _get_enriched_metadata(src_path, metadata)
metadata["taken_at"] = enriched.get("taken_at") or metadata["taken_at"]
# Copy new file to destination
target_dir = os.path.join(dest_dir, artist)
os.makedirs(target_dir, exist_ok=True)
dest_path = build_hashed_dest_path(target_dir, original_name, file_hash)
shutil.copy2(src_path, dest_path)
# Generate new thumbnail
try:
if dest_path.lower().endswith((".mp4", ".mov")):
thumb_path = generate_video_thumbnail_mirrored(dest_path)
else:
thumb_path = generate_thumbnail(dest_path)
except Exception as e:
print(f"[WARN] Failed to generate thumbnail for {original_name}: {e}")
thumb_path = None
# Supersede the old record (preserves tags)
record = supersede_image(
old_record, dest_path, thumb_path, file_hash, phash, metadata, original_name
)
# Apply any new tags from sidecar
_apply_enriched_tags(record, enriched)
if commit:
db.session.commit()
# Update existing_phashes list with new dimensions
for i, (ph, w, h, img_id) in enumerate(existing_phashes):
if img_id == similar_id:
existing_phashes[i] = (phash, metadata["width"], metadata["height"], img_id)
break
return (True, phash, record, True) # superseded=True
# No similar image found (or supersede disabled) - normal import
# Enrich metadata from Gallery-DL sidecar JSON if present
enriched = _get_enriched_metadata(src_path, metadata)
# Copy into /images/<artist>/<base>__<hash[:10]><ext>
target_dir = os.path.join(dest_dir, artist)
os.makedirs(target_dir, exist_ok=True)
dest_path = build_hashed_dest_path(target_dir, original_name, file_hash)
shutil.copy2(src_path, dest_path)
# Thumbnails (mirrored for both images and videos)
try:
if dest_path.lower().endswith((".mp4", ".mov")):
thumb_path = generate_video_thumbnail_mirrored(dest_path)
else:
thumb_path = generate_thumbnail(dest_path)
except Exception as e:
print(f"[WARN] Failed to generate thumbnail for {original_name}: {e}")
thumb_path = None
# Create DB record (keep display name as original; filepath is content-addressed)
# Use enriched taken_at (earliest date wins)
record = ImageRecord(
filename=original_name,
filepath=dest_path,
thumb_path=thumb_path,
hash=file_hash,
perceptual_hash=str(phash) if phash else None,
file_size=metadata["file_size"],
width=metadata["width"],
height=metadata["height"],
format=metadata["format"],
camera_model=metadata["camera_model"],
taken_at=enriched.get("taken_at") or metadata["taken_at"],
imported_at=datetime.utcnow()
)
# Auto-tag with artist:<name> (treat artist as a tag only)
if artist:
artist_tag_name = f"artist:{artist}"
artist_tag = Tag.query.filter_by(name=artist_tag_name).first()
if not artist_tag:
artist_tag = Tag(name=artist_tag_name, kind="artist")
db.session.add(artist_tag)
db.session.flush()
if artist_tag not in record.tags:
record.tags.append(artist_tag)
# Add tags from Gallery-DL metadata (source:platform, post:platform:artist:id)
_apply_enriched_tags(record, enriched)
db.session.add(record)
if commit:
db.session.commit()
return (True, phash, record, False) # superseded=False (new import)
def _get_enriched_metadata(src_path: str, existing_metadata: dict) -> dict:
"""
Get enriched metadata from Gallery-DL sidecar JSON.
Returns empty dict if no sidecar found.
"""
try:
from app.utils.metadata_enrichment import enrich_from_sidecar
return enrich_from_sidecar(src_path, existing_metadata)
except ImportError:
return {}
except Exception as e:
print(f"[WARN] Failed to enrich metadata for {src_path}: {e}")
return {}
def _apply_enriched_tags(record: ImageRecord, enriched: dict) -> None:
"""
Apply tags from enriched metadata to an ImageRecord.
Creates tags if they don't exist.
"""
if not enriched or "tags" not in enriched:
return
for tag_info in enriched.get("tags", []):
tag_name = tag_info.get("name")
tag_kind = tag_info.get("kind")
if not tag_name:
continue
tag = Tag.query.filter_by(name=tag_name).first()
if not tag:
tag = Tag(name=tag_name, kind=tag_kind)
db.session.add(tag)
db.session.flush()
if tag not in record.tags:
record.tags.append(tag)
print(f"[INFO] Added tag: {tag_name}")
def _enrich_existing_record(record: ImageRecord, src_path: str) -> None:
"""
Enrich an existing record with metadata from sidecar JSON.
Used when a duplicate is found - we still want to merge metadata.
- Updates taken_at if sidecar date is earlier
- Adds source/post tags if not already present
"""
try:
from app.utils.metadata_enrichment import enrich_from_sidecar, resolve_date_conflict
enriched = enrich_from_sidecar(src_path, {"taken_at": record.taken_at})
if not enriched.get("raw_metadata"):
return # No sidecar found
# Update date if sidecar has an earlier date
new_date = resolve_date_conflict(record.taken_at, enriched.get("taken_at"))
if new_date != record.taken_at:
print(f"[INFO] Updating taken_at for {record.filename}: {record.taken_at} -> {new_date}")
record.taken_at = new_date
# Add any new tags
_apply_enriched_tags(record, enriched)
except Exception as e:
print(f"[WARN] Failed to enrich existing record: {e}")
# =============================================================================
# Helpers: Thumbnails, metadata, hashing, similarity
# =============================================================================
app/utils/metadata_enrichment.py
+12
View File
@@ -301,6 +301,18 @@ def resolve_date_conflict(existing_date: Optional[datetime], new_date: Optional[
if new_date is None:
return existing_date
# Normalize timezone awareness before comparison
# If one is aware and one is naive, strip timezone from the aware one
existing_aware = existing_date.tzinfo is not None
new_aware = new_date.tzinfo is not None
if existing_aware and not new_aware:
# Strip timezone from existing to compare
existing_date = existing_date.replace(tzinfo=None)
elif new_aware and not existing_aware:
# Strip timezone from new to compare
new_date = new_date.replace(tzinfo=None)
# Return the earlier date
return min(existing_date, new_date)
requirements.txt
-2
View File
@@ -5,8 +5,6 @@ psycopg2-binary
pillow
exifread
imagehash
pyunpack
patool
gunicorn
# Celery task queue