bvandeusen/imagerepo
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed metadata time import error and moved quest status widget to top of settings.

Browse Source
This commit is contained in:
Bryan Van Deusen
2026-01-21 11:30:45 -05:00
parent dc45a09681
commit 041a3dbfb4
7 changed files with 108 additions and 760 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/utils/image_importer.py
+1 -524
View File
@@ -12,15 +12,12 @@ import re
import shutil
import subprocess
import tempfile
import time
import uuid
from PIL import Image, ImageOps
import exifread
import imagehash
from pyunpack import Archive
from app import db
from app.models import ImageRecord, Tag, ArchiveRecord
@@ -48,7 +45,6 @@ ALLOWED_ARCHIVE_EXTS = (
# Import filter settings (loaded from file or env)
IMPORT_SETTINGS_PATH = "/import/settings.json"
IMPORT_STATS_PATH = "/import/stats.json"
def load_import_settings() -> dict:
@@ -92,15 +88,6 @@ def load_import_settings() -> dict:
return defaults
def save_import_stats(stats: dict) -> None:
"""Save import statistics to file."""
try:
with open(IMPORT_STATS_PATH, "w") as f:
json.dump(stats, f, indent=2)
except Exception as e:
print(f"[WARN] Failed to save import stats: {e}")
def is_mostly_transparent(file_path: str, threshold: float = 0.9) -> bool:
"""
Check if an image is mostly transparent (> threshold % of pixels are transparent).
@@ -398,298 +385,7 @@ def extract_archive_resilient(archive_path: str, out_dir: str) -> None:
# =============================================================================
# Core: Import task & archive processing
# =============================================================================
def import_images_task(source_dir: str, dest_dir: str) -> str:
"""
Walk the import tree, process archives first (only first volumes),
then media files. Batches DB commits for efficiency.
"""
# Cleanup any stale extraction dirs from previous runs/crashes
cleanup_stale_tempdirs()
# Load import filter settings
settings = load_import_settings()
min_width = settings.get("min_width", 0)
min_height = settings.get("min_height", 0)
skip_transparent = settings.get("skip_transparent", False)
transparency_threshold = settings.get("transparency_threshold", 0.9)
skip_single_color = settings.get("skip_single_color", True)
single_color_threshold = settings.get("single_color_threshold", 0.95)
single_color_tolerance = settings.get("single_color_tolerance", 30)
phash_threshold = settings.get("phash_threshold", 2)
supersede_smaller = settings.get("supersede_smaller", True)
print(f"[INFO] Import settings: min_width={min_width}, min_height={min_height}, "
f"skip_transparent={skip_transparent}, transparency_threshold={transparency_threshold}, "
f"skip_single_color={skip_single_color}, single_color_threshold={single_color_threshold}, "
f"phash_threshold={phash_threshold}, supersede_smaller={supersede_smaller}")
# Statistics tracking
stats = {
"total_processed": 0,
"total_imported": 0,
"total_superseded": 0,
"filtered_by_dimension": 0,
"filtered_by_transparency": 0,
"filtered_by_single_color": 0,
"filtered_by_duplicate": 0,
}
imported: list[str] = []
batch_size = 10
batch_counter = 0
# Preload existing pHashes to avoid N+1 lookups
# Format: (phash, width, height, image_id) for supersede functionality
existing_phashes = [
(imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height, img.id)
for img in ImageRecord.query.filter(ImageRecord.perceptual_hash != None).all()
]
# Ensure thumbs base exists
(Path(dest_dir) / "thumbs").mkdir(parents=True, exist_ok=True)
# Optional free-space guard
min_free = float(os.environ.get(ENV_MINFREE, "0") or "0")
try:
ensure_free_space_or_raise(get_tmp_base(), min_free)
except Exception as e:
print(f"[WARN] Free-space check failed: {e}")
for artist_dir in os.listdir(source_dir):
artist_path = os.path.join(source_dir, artist_dir)
if not os.path.isdir(artist_path):
continue
for root, _, files in os.walk(artist_path):
for file in files:
full_path = os.path.join(root, file)
lower = file.lower()
# Archives first
if lower.endswith(ALLOWED_ARCHIVE_EXTS):
if not is_first_volume(full_path):
print(f"[SKIP] Not first archive volume: {file}")
continue
try:
count = process_archive(
archive_path=full_path,
dest_dir=dest_dir,
artist=artist_dir,
existing_phashes=existing_phashes,
settings=settings,
stats=stats
)
imported.append(f"[archive]{file}:{count}")
except Exception as e:
print(f"[WARN] Failed processing archive {file}: {e}")
continue
# Non-archive media
if not lower.endswith(ALLOWED_MEDIA_EXTS):
continue
stats["total_processed"] += 1
# Apply dimension filter
if min_width > 0 or min_height > 0:
md = extract_metadata(full_path)
if md.get("width") and md.get("height"):
if md["width"] < min_width or md["height"] < min_height:
print(f"[SKIP] Too small ({md['width']}x{md['height']}): {file}")
stats["filtered_by_dimension"] += 1
continue
# Apply transparency filter
if skip_transparent and lower.endswith((".png", ".gif", ".webp")):
if is_mostly_transparent(full_path, transparency_threshold):
print(f"[SKIP] Mostly transparent: {file}")
stats["filtered_by_transparency"] += 1
continue
# Apply single-color filter (skip images that are mostly one solid color)
if skip_single_color and lower.endswith((".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")):
if is_mostly_single_color(full_path, single_color_threshold, single_color_tolerance):
print(f"[SKIP] Mostly single color: {file}")
stats["filtered_by_single_color"] += 1
continue
added, phash, _rec, superseded = import_single_file(
src_path=full_path,
dest_dir=dest_dir,
artist=artist_dir,
existing_phashes=existing_phashes,
phash_threshold=phash_threshold,
supersede_smaller=supersede_smaller
)
if added:
imported.append(file)
if superseded:
stats["total_superseded"] += 1
else:
stats["total_imported"] += 1
elif _rec is None and phash is None:
# Was a duplicate
stats["filtered_by_duplicate"] += 1
# Add new image's phash to the tracking list (for fresh imports only, not supersedes)
if phash and _rec and not superseded:
existing_phashes.append((phash, _rec.width, _rec.height, _rec.id))
batch_counter += 1
if batch_counter >= batch_size:
db.session.commit()
print(f"[INFO] Committed batch of {batch_size} items.")
batch_counter = 0
if "sqlite" in db.engine.url.drivername:
time.sleep(1)
if batch_counter > 0:
db.session.commit()
print(f"[INFO] Committed final batch of {batch_counter} items.")
# Save stats
save_import_stats(stats)
summary = (f"Imported {stats['total_imported']} items, superseded {stats['total_superseded']} smaller versions. "
f"Filtered: {stats['filtered_by_dimension']} by size, {stats['filtered_by_transparency']} by transparency, "
f"{stats['filtered_by_single_color']} by single color, {stats['filtered_by_duplicate']} duplicates.")
print(f"[INFO] {summary}")
return summary
def process_archive(archive_path, dest_dir, artist, existing_phashes, settings=None, stats=None):
"""
Extracts archive to temp dir, imports/tag existing images, and records ArchiveRecord.
Creates an archive:* tag ONLY if at least one image was imported or tagged.
Returns number of items imported or tagged.
"""
if settings is None:
settings = load_import_settings()
if stats is None:
stats = {"total_processed": 0, "total_imported": 0, "total_superseded": 0,
"filtered_by_dimension": 0, "filtered_by_transparency": 0,
"filtered_by_single_color": 0, "filtered_by_duplicate": 0}
min_width = settings.get("min_width", 0)
min_height = settings.get("min_height", 0)
skip_transparent = settings.get("skip_transparent", False)
transparency_threshold = settings.get("transparency_threshold", 0.9)
skip_single_color = settings.get("skip_single_color", True)
single_color_threshold = settings.get("single_color_threshold", 0.95)
single_color_tolerance = settings.get("single_color_tolerance", 30)
phash_threshold = settings.get("phash_threshold", 2)
supersede_smaller = settings.get("supersede_smaller", True)
archive_size = os.path.getsize(archive_path)
archive_hash = calculate_hash(archive_path)
# Skip if this exact archive file was already processed (hash-based)
existing_archive = ArchiveRecord.query.filter_by(hash=archive_hash).first()
if existing_archive:
print(f"[SKIP] Archive already imported: {archive_path}")
return 0
tmpdir = tempfile.mkdtemp(prefix="extract_")
touched_records = [] # ImageRecord objects we imported or matched
imported_or_tagged = 0
created_tag = None
try:
# Extract using pyunpack
Archive(archive_path).extractall(tmpdir)
# Import / collect all media
for root, _, files in os.walk(tmpdir):
for file in files:
lower = file.lower()
if not lower.endswith(ALLOWED_MEDIA_EXTS):
continue
src_path = os.path.join(root, file)
stats["total_processed"] += 1
# Apply dimension filter
if min_width > 0 or min_height > 0:
md = extract_metadata(src_path)
if md.get("width") and md.get("height"):
if md["width"] < min_width or md["height"] < min_height:
print(f"[SKIP] Too small ({md['width']}x{md['height']}): {file}")
stats["filtered_by_dimension"] += 1
continue
# Apply transparency filter
if skip_transparent and lower.endswith((".png", ".gif", ".webp")):
if is_mostly_transparent(src_path, transparency_threshold):
print(f"[SKIP] Mostly transparent: {file}")
stats["filtered_by_transparency"] += 1
continue
# Apply single-color filter
if skip_single_color and lower.endswith((".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")):
if is_mostly_single_color(src_path, single_color_threshold, single_color_tolerance):
print(f"[SKIP] Mostly single color: {file}")
stats["filtered_by_single_color"] += 1
continue
added, _phash, rec, superseded = import_single_file(
src_path=src_path,
dest_dir=dest_dir,
artist=artist, # still used for auto artist:<name> tag
existing_phashes=existing_phashes,
commit=False,
phash_threshold=phash_threshold,
supersede_smaller=supersede_smaller
)
if rec is not None:
touched_records.append(rec)
if added:
imported_or_tagged += 1
if superseded:
stats["total_superseded"] += 1
else:
stats["total_imported"] += 1
elif rec is None and _phash is None:
stats["filtered_by_duplicate"] += 1
# Only now: create/attach the archive tag if we actually touched images
if touched_records:
# Incremental name like 'archive:{artist}/0001'
tag_name = compute_next_archive_tag_name(artist)
created_tag = Tag.query.filter_by(name=tag_name).first()
if not created_tag:
created_tag = Tag(name=tag_name, kind='archive')
db.session.add(created_tag)
db.session.flush()
newly_tagged = 0
for rec in touched_records:
if created_tag not in rec.tags:
rec.tags.append(created_tag)
newly_tagged += 1
imported_or_tagged += newly_tagged
# Record this archive as processed (tag may be None if no media)
arch = ArchiveRecord(
filename=archive_path,
file_size=archive_size,
hash=archive_hash,
artist=artist,
tag=created_tag
)
db.session.add(arch)
db.session.commit()
print(f"[INFO] Archive processed: {archive_path} items={imported_or_tagged}")
return imported_or_tagged
finally:
shutil.rmtree(tmpdir, ignore_errors=True)
# =============================================================================
# Core: Single-file import path (content-addressed storage)
# Content-addressed storage helpers
# =============================================================================
def build_hashed_dest_path(target_dir: str, original_name: str, content_hash: str) -> str:
@@ -712,225 +408,6 @@ def build_hashed_dest_path(target_dir: str, original_name: str, content_hash: st
i += 1
def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phashes,
commit: bool = True, phash_threshold: int = 2, supersede_smaller: bool = True):
"""
Import a single media file. If an equivalent file already exists, return that
record so callers can tag it (no duplicate import).
If a smaller visually-similar image exists and supersede_smaller is True,
the smaller image will be replaced with the larger one (preserving tags).
Also enriches metadata from Gallery-DL sidecar JSON files if present.
Returns (added: bool, phash_or_None, record_or_None, superseded: bool).
The superseded flag indicates if an existing smaller image was replaced.
"""
original_name = os.path.basename(src_path)
print(f"[INFO] Processing: {src_path}")
file_size = os.path.getsize(src_path)
# Compute content hash first (authoritative de-dup + used in filepath)
file_hash = calculate_hash(src_path)
existing = ImageRecord.query.filter_by(hash=file_hash).first()
if existing:
print(f"[SKIP] Duplicate by hash: {original_name}")
# Even for duplicates, try to enrich with sidecar metadata
_enrich_existing_record(existing, src_path)
return (False, None, existing, False)
# Optional info: same name+size (but different content) → we still import with hashed name
existing_ns = ImageRecord.query.filter_by(filename=original_name, file_size=file_size).first()
if existing_ns:
print(f"[INFO] Same name+size exists but different hash; storing with hashed name: {original_name}")
# Metadata & pHash similarity
metadata = extract_metadata(src_path)
if not metadata["width"] or not metadata["height"]:
print(f"[SKIP] Missing dimension data for {original_name}")
return (False, None, None, False)
phash = calculate_perceptual_hash(src_path)
# Check for similar images
superseded = False
if phash and phash_threshold > 0:
relationship, similar_id = find_similar_image(
phash, metadata["width"], metadata["height"], existing_phashes, threshold=phash_threshold
)
if relationship == "larger_exists":
print(f"[SKIP] {original_name} is visually similar to an existing larger image (threshold={phash_threshold}).")
return (False, phash, None, False)
elif relationship == "smaller_exists" and supersede_smaller and similar_id:
# Found a smaller version - supersede it
old_record = ImageRecord.query.get(similar_id)
if old_record:
# Enrich metadata from Gallery-DL sidecar JSON if present
enriched = _get_enriched_metadata(src_path, metadata)
metadata["taken_at"] = enriched.get("taken_at") or metadata["taken_at"]
# Copy new file to destination
target_dir = os.path.join(dest_dir, artist)
os.makedirs(target_dir, exist_ok=True)
dest_path = build_hashed_dest_path(target_dir, original_name, file_hash)
shutil.copy2(src_path, dest_path)
# Generate new thumbnail
try:
if dest_path.lower().endswith((".mp4", ".mov")):
thumb_path = generate_video_thumbnail_mirrored(dest_path)
else:
thumb_path = generate_thumbnail(dest_path)
except Exception as e:
print(f"[WARN] Failed to generate thumbnail for {original_name}: {e}")
thumb_path = None
# Supersede the old record (preserves tags)
record = supersede_image(
old_record, dest_path, thumb_path, file_hash, phash, metadata, original_name
)
# Apply any new tags from sidecar
_apply_enriched_tags(record, enriched)
if commit:
db.session.commit()
# Update existing_phashes list with new dimensions
for i, (ph, w, h, img_id) in enumerate(existing_phashes):
if img_id == similar_id:
existing_phashes[i] = (phash, metadata["width"], metadata["height"], img_id)
break
return (True, phash, record, True) # superseded=True
# No similar image found (or supersede disabled) - normal import
# Enrich metadata from Gallery-DL sidecar JSON if present
enriched = _get_enriched_metadata(src_path, metadata)
# Copy into /images/<artist>/<base>__<hash[:10]><ext>
target_dir = os.path.join(dest_dir, artist)
os.makedirs(target_dir, exist_ok=True)
dest_path = build_hashed_dest_path(target_dir, original_name, file_hash)
shutil.copy2(src_path, dest_path)
# Thumbnails (mirrored for both images and videos)
try:
if dest_path.lower().endswith((".mp4", ".mov")):
thumb_path = generate_video_thumbnail_mirrored(dest_path)
else:
thumb_path = generate_thumbnail(dest_path)
except Exception as e:
print(f"[WARN] Failed to generate thumbnail for {original_name}: {e}")
thumb_path = None
# Create DB record (keep display name as original; filepath is content-addressed)
# Use enriched taken_at (earliest date wins)
record = ImageRecord(
filename=original_name,
filepath=dest_path,
thumb_path=thumb_path,
hash=file_hash,
perceptual_hash=str(phash) if phash else None,
file_size=metadata["file_size"],
width=metadata["width"],
height=metadata["height"],
format=metadata["format"],
camera_model=metadata["camera_model"],
taken_at=enriched.get("taken_at") or metadata["taken_at"],
imported_at=datetime.utcnow()
)
# Auto-tag with artist:<name> (treat artist as a tag only)
if artist:
artist_tag_name = f"artist:{artist}"
artist_tag = Tag.query.filter_by(name=artist_tag_name).first()
if not artist_tag:
artist_tag = Tag(name=artist_tag_name, kind="artist")
db.session.add(artist_tag)
db.session.flush()
if artist_tag not in record.tags:
record.tags.append(artist_tag)
# Add tags from Gallery-DL metadata (source:platform, post:platform:artist:id)
_apply_enriched_tags(record, enriched)
db.session.add(record)
if commit:
db.session.commit()
return (True, phash, record, False) # superseded=False (new import)
def _get_enriched_metadata(src_path: str, existing_metadata: dict) -> dict:
"""
Get enriched metadata from Gallery-DL sidecar JSON.
Returns empty dict if no sidecar found.
"""
try:
from app.utils.metadata_enrichment import enrich_from_sidecar
return enrich_from_sidecar(src_path, existing_metadata)
except ImportError:
return {}
except Exception as e:
print(f"[WARN] Failed to enrich metadata for {src_path}: {e}")
return {}
def _apply_enriched_tags(record: ImageRecord, enriched: dict) -> None:
"""
Apply tags from enriched metadata to an ImageRecord.
Creates tags if they don't exist.
"""
if not enriched or "tags" not in enriched:
return
for tag_info in enriched.get("tags", []):
tag_name = tag_info.get("name")
tag_kind = tag_info.get("kind")
if not tag_name:
continue
tag = Tag.query.filter_by(name=tag_name).first()
if not tag:
tag = Tag(name=tag_name, kind=tag_kind)
db.session.add(tag)
db.session.flush()
if tag not in record.tags:
record.tags.append(tag)
print(f"[INFO] Added tag: {tag_name}")
def _enrich_existing_record(record: ImageRecord, src_path: str) -> None:
"""
Enrich an existing record with metadata from sidecar JSON.
Used when a duplicate is found - we still want to merge metadata.
- Updates taken_at if sidecar date is earlier
- Adds source/post tags if not already present
"""
try:
from app.utils.metadata_enrichment import enrich_from_sidecar, resolve_date_conflict
enriched = enrich_from_sidecar(src_path, {"taken_at": record.taken_at})
if not enriched.get("raw_metadata"):
return # No sidecar found
# Update date if sidecar has an earlier date
new_date = resolve_date_conflict(record.taken_at, enriched.get("taken_at"))
if new_date != record.taken_at:
print(f"[INFO] Updating taken_at for {record.filename}: {record.taken_at} -> {new_date}")
record.taken_at = new_date
# Add any new tags
_apply_enriched_tags(record, enriched)
except Exception as e:
print(f"[WARN] Failed to enrich existing record: {e}")
# =============================================================================
# Helpers: Thumbnails, metadata, hashing, similarity
# =============================================================================
app/utils/metadata_enrichment.py
+12
View File
@@ -301,6 +301,18 @@ def resolve_date_conflict(existing_date: Optional[datetime], new_date: Optional[
if new_date is None:
return existing_date
# Normalize timezone awareness before comparison
# If one is aware and one is naive, strip timezone from the aware one
existing_aware = existing_date.tzinfo is not None
new_aware = new_date.tzinfo is not None
if existing_aware and not new_aware:
# Strip timezone from existing to compare
existing_date = existing_date.replace(tzinfo=None)
elif new_aware and not existing_aware:
# Strip timezone from new to compare
new_date = new_date.replace(tzinfo=None)
# Return the earlier date
return min(existing_date, new_date)