moving styling and views to be more consistent and for gallery view to be less cumbersome.
This commit is contained in:
Bryan Van Deusen
+429
-37
@@ -50,21 +50,43 @@ IMPORT_STATS_PATH = "/import/stats.json"
|
||||
|
||||
|
||||
def load_import_settings() -> dict:
|
||||
"""Load import filter settings from file or environment."""
|
||||
"""
|
||||
Load import filter settings from database, with file and environment fallbacks.
|
||||
Priority: DB settings > file settings > environment variables
|
||||
"""
|
||||
# Start with environment defaults
|
||||
defaults = {
|
||||
"min_width": int(os.environ.get("IMPORT_MIN_WIDTH", "0")),
|
||||
"min_height": int(os.environ.get("IMPORT_MIN_HEIGHT", "0")),
|
||||
"skip_transparent": os.environ.get("IMPORT_SKIP_TRANSPARENT", "false").lower() == "true",
|
||||
"transparency_threshold": float(os.environ.get("IMPORT_TRANSPARENCY_THRESHOLD", "0.9")),
|
||||
"phash_threshold": int(os.environ.get("IMPORT_PHASH_THRESHOLD", "2")),
|
||||
"skip_single_color": os.environ.get("IMPORT_SKIP_SINGLE_COLOR", "true").lower() == "true",
|
||||
"single_color_threshold": float(os.environ.get("IMPORT_SINGLE_COLOR_THRESHOLD", "0.95")),
|
||||
"single_color_tolerance": int(os.environ.get("IMPORT_SINGLE_COLOR_TOLERANCE", "30")),
|
||||
"phash_threshold": int(os.environ.get("IMPORT_PHASH_THRESHOLD", "10")), # Default for 256-bit
|
||||
"supersede_smaller": os.environ.get("IMPORT_SUPERSEDE_SMALLER", "true").lower() == "true",
|
||||
}
|
||||
|
||||
# Try to load from file (legacy support)
|
||||
try:
|
||||
if os.path.exists(IMPORT_SETTINGS_PATH):
|
||||
with open(IMPORT_SETTINGS_PATH, "r") as f:
|
||||
file_settings = json.load(f)
|
||||
defaults.update(file_settings)
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to load import settings: {e}")
|
||||
print(f"[WARN] Failed to load import settings from file: {e}")
|
||||
|
||||
# Try to load from database (preferred)
|
||||
try:
|
||||
from app.utils.version_migration import get_import_settings
|
||||
db_settings = get_import_settings()
|
||||
# Only override if DB has non-default values
|
||||
for key, value in db_settings.items():
|
||||
if value is not None:
|
||||
defaults[key] = value
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to load import settings from DB: {e}")
|
||||
|
||||
return defaults
|
||||
|
||||
|
||||
@@ -106,6 +128,136 @@ def is_mostly_transparent(file_path: str, threshold: float = 0.9) -> bool:
|
||||
print(f"[WARN] Failed to check transparency for {file_path}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def is_mostly_single_color(file_path: str, threshold: float = 0.95, tolerance: int = 30) -> bool:
|
||||
"""
|
||||
Check if an image is mostly a single color (> threshold % of pixels are similar).
|
||||
Uses color distance tolerance to group similar colors.
|
||||
|
||||
Args:
|
||||
file_path: Path to the image file
|
||||
threshold: Percentage of pixels that must be similar (0.0-1.0, default 0.95 = 95%)
|
||||
tolerance: Maximum color distance to be considered "same" color (0-255 per channel, default 30)
|
||||
|
||||
Returns:
|
||||
True if the image is mostly a single solid color
|
||||
"""
|
||||
try:
|
||||
with Image.open(file_path) as img:
|
||||
# Convert to RGB for consistent color comparison
|
||||
if img.mode in ("RGBA", "LA"):
|
||||
# For images with alpha, only consider non-transparent pixels
|
||||
img_rgba = img.convert("RGBA") if img.mode != "RGBA" else img
|
||||
pixels = list(img_rgba.getdata())
|
||||
# Filter out mostly transparent pixels (alpha < 128)
|
||||
opaque_pixels = [(r, g, b) for r, g, b, a in pixels if a >= 128]
|
||||
if len(opaque_pixels) == 0:
|
||||
return True # Fully transparent = effectively single color
|
||||
pixels = opaque_pixels
|
||||
elif img.mode == "P":
|
||||
img = img.convert("RGB")
|
||||
pixels = list(img.getdata())
|
||||
elif img.mode == "L":
|
||||
# Grayscale - convert to RGB tuples
|
||||
pixels = [(p, p, p) for p in img.getdata()]
|
||||
else:
|
||||
if img.mode != "RGB":
|
||||
img = img.convert("RGB")
|
||||
pixels = list(img.getdata())
|
||||
|
||||
if len(pixels) == 0:
|
||||
return True
|
||||
|
||||
# Sample pixels for performance on large images
|
||||
total_pixels = len(pixels)
|
||||
if total_pixels > 10000:
|
||||
# Sample evenly distributed pixels
|
||||
step = total_pixels // 10000
|
||||
pixels = pixels[::step]
|
||||
|
||||
# Find the dominant color (most common pixel or average of sampled area)
|
||||
# Use the pixel at center as reference
|
||||
center_idx = len(pixels) // 2
|
||||
ref_color = pixels[center_idx]
|
||||
|
||||
# Count pixels within tolerance of reference color
|
||||
similar_count = 0
|
||||
for pixel in pixels:
|
||||
if isinstance(pixel, int):
|
||||
pixel = (pixel, pixel, pixel)
|
||||
# Calculate color distance (simple RGB distance)
|
||||
dist = abs(pixel[0] - ref_color[0]) + abs(pixel[1] - ref_color[1]) + abs(pixel[2] - ref_color[2])
|
||||
if dist <= tolerance * 3: # tolerance per channel * 3 channels
|
||||
similar_count += 1
|
||||
|
||||
similarity_ratio = similar_count / len(pixels)
|
||||
return similarity_ratio > threshold
|
||||
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to check single color for {file_path}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def supersede_image(old_record: "ImageRecord", new_filepath: str, new_thumb_path: str,
|
||||
new_hash: str, new_phash, new_metadata: dict, new_filename: str) -> "ImageRecord":
|
||||
"""
|
||||
Replace a smaller image with a larger version, preserving tags and metadata.
|
||||
|
||||
Args:
|
||||
old_record: The existing ImageRecord to supersede
|
||||
new_filepath: Path to the new larger image file
|
||||
new_thumb_path: Path to the new thumbnail
|
||||
new_hash: SHA256 hash of the new file
|
||||
new_phash: Perceptual hash of the new image
|
||||
new_metadata: Metadata dict for the new image
|
||||
new_filename: Original filename of the new image
|
||||
|
||||
Returns:
|
||||
The updated ImageRecord
|
||||
"""
|
||||
# Store old file paths for cleanup
|
||||
old_filepath = old_record.filepath
|
||||
old_thumb_path = old_record.thumb_path
|
||||
|
||||
print(f"[SUPERSEDE] Replacing {old_record.filename} ({old_record.width}x{old_record.height}) "
|
||||
f"with {new_filename} ({new_metadata['width']}x{new_metadata['height']})")
|
||||
|
||||
# Update record with new file info
|
||||
old_record.filename = new_filename
|
||||
old_record.filepath = new_filepath
|
||||
old_record.thumb_path = new_thumb_path
|
||||
old_record.hash = new_hash
|
||||
old_record.perceptual_hash = str(new_phash) if new_phash else None
|
||||
old_record.file_size = new_metadata["file_size"]
|
||||
old_record.width = new_metadata["width"]
|
||||
old_record.height = new_metadata["height"]
|
||||
old_record.format = new_metadata["format"]
|
||||
# Keep the earlier taken_at date (prefer original date)
|
||||
if new_metadata.get("taken_at") and old_record.taken_at:
|
||||
if new_metadata["taken_at"] < old_record.taken_at:
|
||||
old_record.taken_at = new_metadata["taken_at"]
|
||||
elif new_metadata.get("taken_at") and not old_record.taken_at:
|
||||
old_record.taken_at = new_metadata["taken_at"]
|
||||
# Don't update imported_at - keep original import time
|
||||
|
||||
# Delete old files
|
||||
try:
|
||||
if old_filepath and os.path.exists(old_filepath):
|
||||
os.remove(old_filepath)
|
||||
print(f"[SUPERSEDE] Deleted old file: {old_filepath}")
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to delete old file {old_filepath}: {e}")
|
||||
|
||||
try:
|
||||
if old_thumb_path and os.path.exists(old_thumb_path):
|
||||
os.remove(old_thumb_path)
|
||||
print(f"[SUPERSEDE] Deleted old thumbnail: {old_thumb_path}")
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to delete old thumbnail {old_thumb_path}: {e}")
|
||||
|
||||
return old_record
|
||||
|
||||
|
||||
# Regex for multi-part detection
|
||||
RAR_PART_RE = re.compile(r"\.part(\d+)\.rar$", re.IGNORECASE)
|
||||
SEVENZ_PART_RE = re.compile(r"\.7z\.(\d{3})$", re.IGNORECASE)
|
||||
@@ -233,19 +385,25 @@ def import_images_task(source_dir: str, dest_dir: str) -> str:
|
||||
min_height = settings.get("min_height", 0)
|
||||
skip_transparent = settings.get("skip_transparent", False)
|
||||
transparency_threshold = settings.get("transparency_threshold", 0.9)
|
||||
|
||||
skip_single_color = settings.get("skip_single_color", True)
|
||||
single_color_threshold = settings.get("single_color_threshold", 0.95)
|
||||
single_color_tolerance = settings.get("single_color_tolerance", 30)
|
||||
phash_threshold = settings.get("phash_threshold", 2)
|
||||
supersede_smaller = settings.get("supersede_smaller", True)
|
||||
|
||||
print(f"[INFO] Import settings: min_width={min_width}, min_height={min_height}, "
|
||||
f"skip_transparent={skip_transparent}, transparency_threshold={transparency_threshold}, "
|
||||
f"phash_threshold={phash_threshold}")
|
||||
f"skip_single_color={skip_single_color}, single_color_threshold={single_color_threshold}, "
|
||||
f"phash_threshold={phash_threshold}, supersede_smaller={supersede_smaller}")
|
||||
|
||||
# Statistics tracking
|
||||
stats = {
|
||||
"total_processed": 0,
|
||||
"total_imported": 0,
|
||||
"total_superseded": 0,
|
||||
"filtered_by_dimension": 0,
|
||||
"filtered_by_transparency": 0,
|
||||
"filtered_by_single_color": 0,
|
||||
"filtered_by_duplicate": 0,
|
||||
}
|
||||
|
||||
@@ -254,8 +412,9 @@ def import_images_task(source_dir: str, dest_dir: str) -> str:
|
||||
batch_counter = 0
|
||||
|
||||
# Preload existing pHashes to avoid N+1 lookups
|
||||
# Format: (phash, width, height, image_id) for supersede functionality
|
||||
existing_phashes = [
|
||||
(imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height)
|
||||
(imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height, img.id)
|
||||
for img in ImageRecord.query.filter(ImageRecord.perceptual_hash != None).all()
|
||||
]
|
||||
|
||||
@@ -320,24 +479,34 @@ def import_images_task(source_dir: str, dest_dir: str) -> str:
|
||||
stats["filtered_by_transparency"] += 1
|
||||
continue
|
||||
|
||||
added, phash, _rec = import_single_file(
|
||||
# Apply single-color filter (skip images that are mostly one solid color)
|
||||
if skip_single_color and lower.endswith((".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")):
|
||||
if is_mostly_single_color(full_path, single_color_threshold, single_color_tolerance):
|
||||
print(f"[SKIP] Mostly single color: {file}")
|
||||
stats["filtered_by_single_color"] += 1
|
||||
continue
|
||||
|
||||
added, phash, _rec, superseded = import_single_file(
|
||||
src_path=full_path,
|
||||
dest_dir=dest_dir,
|
||||
artist=artist_dir,
|
||||
existing_phashes=existing_phashes,
|
||||
phash_threshold=phash_threshold
|
||||
phash_threshold=phash_threshold,
|
||||
supersede_smaller=supersede_smaller
|
||||
)
|
||||
if added:
|
||||
imported.append(file)
|
||||
stats["total_imported"] += 1
|
||||
if superseded:
|
||||
stats["total_superseded"] += 1
|
||||
else:
|
||||
stats["total_imported"] += 1
|
||||
elif _rec is None and phash is None:
|
||||
# Was a duplicate
|
||||
stats["filtered_by_duplicate"] += 1
|
||||
|
||||
if phash:
|
||||
md = extract_metadata(full_path)
|
||||
if md.get("width") and md.get("height"):
|
||||
existing_phashes.append((phash, md["width"], md["height"]))
|
||||
# Add new image's phash to the tracking list (for fresh imports only, not supersedes)
|
||||
if phash and _rec and not superseded:
|
||||
existing_phashes.append((phash, _rec.width, _rec.height, _rec.id))
|
||||
|
||||
batch_counter += 1
|
||||
if batch_counter >= batch_size:
|
||||
@@ -354,7 +523,9 @@ def import_images_task(source_dir: str, dest_dir: str) -> str:
|
||||
# Save stats
|
||||
save_import_stats(stats)
|
||||
|
||||
summary = f"Imported {len(imported)} items. Filtered: {stats['filtered_by_dimension']} by size, {stats['filtered_by_transparency']} by transparency."
|
||||
summary = (f"Imported {stats['total_imported']} items, superseded {stats['total_superseded']} smaller versions. "
|
||||
f"Filtered: {stats['filtered_by_dimension']} by size, {stats['filtered_by_transparency']} by transparency, "
|
||||
f"{stats['filtered_by_single_color']} by single color, {stats['filtered_by_duplicate']} duplicates.")
|
||||
print(f"[INFO] {summary}")
|
||||
return summary
|
||||
|
||||
@@ -368,14 +539,19 @@ def process_archive(archive_path, dest_dir, artist, existing_phashes, settings=N
|
||||
if settings is None:
|
||||
settings = load_import_settings()
|
||||
if stats is None:
|
||||
stats = {"total_processed": 0, "total_imported": 0, "filtered_by_dimension": 0,
|
||||
"filtered_by_transparency": 0, "filtered_by_duplicate": 0}
|
||||
stats = {"total_processed": 0, "total_imported": 0, "total_superseded": 0,
|
||||
"filtered_by_dimension": 0, "filtered_by_transparency": 0,
|
||||
"filtered_by_single_color": 0, "filtered_by_duplicate": 0}
|
||||
|
||||
min_width = settings.get("min_width", 0)
|
||||
min_height = settings.get("min_height", 0)
|
||||
skip_transparent = settings.get("skip_transparent", False)
|
||||
transparency_threshold = settings.get("transparency_threshold", 0.9)
|
||||
skip_single_color = settings.get("skip_single_color", True)
|
||||
single_color_threshold = settings.get("single_color_threshold", 0.95)
|
||||
single_color_tolerance = settings.get("single_color_tolerance", 30)
|
||||
phash_threshold = settings.get("phash_threshold", 2)
|
||||
supersede_smaller = settings.get("supersede_smaller", True)
|
||||
|
||||
archive_size = os.path.getsize(archive_path)
|
||||
archive_hash = calculate_hash(archive_path)
|
||||
@@ -420,19 +596,30 @@ def process_archive(archive_path, dest_dir, artist, existing_phashes, settings=N
|
||||
stats["filtered_by_transparency"] += 1
|
||||
continue
|
||||
|
||||
added, _phash, rec = import_single_file(
|
||||
# Apply single-color filter
|
||||
if skip_single_color and lower.endswith((".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")):
|
||||
if is_mostly_single_color(src_path, single_color_threshold, single_color_tolerance):
|
||||
print(f"[SKIP] Mostly single color: {file}")
|
||||
stats["filtered_by_single_color"] += 1
|
||||
continue
|
||||
|
||||
added, _phash, rec, superseded = import_single_file(
|
||||
src_path=src_path,
|
||||
dest_dir=dest_dir,
|
||||
artist=artist, # still used for auto artist:<name> tag
|
||||
existing_phashes=existing_phashes,
|
||||
commit=False,
|
||||
phash_threshold=phash_threshold
|
||||
phash_threshold=phash_threshold,
|
||||
supersede_smaller=supersede_smaller
|
||||
)
|
||||
if rec is not None:
|
||||
touched_records.append(rec)
|
||||
if added:
|
||||
imported_or_tagged += 1
|
||||
stats["total_imported"] += 1
|
||||
if superseded:
|
||||
stats["total_superseded"] += 1
|
||||
else:
|
||||
stats["total_imported"] += 1
|
||||
elif rec is None and _phash is None:
|
||||
stats["filtered_by_duplicate"] += 1
|
||||
|
||||
@@ -495,11 +682,19 @@ def build_hashed_dest_path(target_dir: str, original_name: str, content_hash: st
|
||||
i += 1
|
||||
|
||||
|
||||
def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phashes, commit: bool = True, phash_threshold: int = 2):
|
||||
def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phashes,
|
||||
commit: bool = True, phash_threshold: int = 2, supersede_smaller: bool = True):
|
||||
"""
|
||||
Import a single media file. If an equivalent file already exists, return that
|
||||
record so callers can tag it (no duplicate import).
|
||||
Returns (added: bool, phash_or_None, record_or_None).
|
||||
|
||||
If a smaller visually-similar image exists and supersede_smaller is True,
|
||||
the smaller image will be replaced with the larger one (preserving tags).
|
||||
|
||||
Also enriches metadata from Gallery-DL sidecar JSON files if present.
|
||||
|
||||
Returns (added: bool, phash_or_None, record_or_None, superseded: bool).
|
||||
The superseded flag indicates if an existing smaller image was replaced.
|
||||
"""
|
||||
original_name = os.path.basename(src_path)
|
||||
print(f"[INFO] Processing: {src_path}")
|
||||
@@ -510,7 +705,9 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash
|
||||
existing = ImageRecord.query.filter_by(hash=file_hash).first()
|
||||
if existing:
|
||||
print(f"[SKIP] Duplicate by hash: {original_name}")
|
||||
return (False, None, existing)
|
||||
# Even for duplicates, try to enrich with sidecar metadata
|
||||
_enrich_existing_record(existing, src_path)
|
||||
return (False, None, existing, False)
|
||||
|
||||
# Optional info: same name+size (but different content) → we still import with hashed name
|
||||
existing_ns = ImageRecord.query.filter_by(filename=original_name, file_size=file_size).first()
|
||||
@@ -521,12 +718,67 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash
|
||||
metadata = extract_metadata(src_path)
|
||||
if not metadata["width"] or not metadata["height"]:
|
||||
print(f"[SKIP] Missing dimension data for {original_name}")
|
||||
return (False, None, None)
|
||||
return (False, None, None, False)
|
||||
|
||||
phash = calculate_perceptual_hash(src_path)
|
||||
if phash and phash_threshold > 0 and is_similar_image(phash, metadata["width"], metadata["height"], existing_phashes, threshold=phash_threshold):
|
||||
print(f"[SKIP] {original_name} is visually similar to an existing larger image (threshold={phash_threshold}).")
|
||||
return (False, phash, None)
|
||||
|
||||
# Check for similar images
|
||||
superseded = False
|
||||
if phash and phash_threshold > 0:
|
||||
relationship, similar_id = find_similar_image(
|
||||
phash, metadata["width"], metadata["height"], existing_phashes, threshold=phash_threshold
|
||||
)
|
||||
|
||||
if relationship == "larger_exists":
|
||||
print(f"[SKIP] {original_name} is visually similar to an existing larger image (threshold={phash_threshold}).")
|
||||
return (False, phash, None, False)
|
||||
|
||||
elif relationship == "smaller_exists" and supersede_smaller and similar_id:
|
||||
# Found a smaller version - supersede it
|
||||
old_record = ImageRecord.query.get(similar_id)
|
||||
if old_record:
|
||||
# Enrich metadata from Gallery-DL sidecar JSON if present
|
||||
enriched = _get_enriched_metadata(src_path, metadata)
|
||||
metadata["taken_at"] = enriched.get("taken_at") or metadata["taken_at"]
|
||||
|
||||
# Copy new file to destination
|
||||
target_dir = os.path.join(dest_dir, artist)
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
dest_path = build_hashed_dest_path(target_dir, original_name, file_hash)
|
||||
shutil.copy2(src_path, dest_path)
|
||||
|
||||
# Generate new thumbnail
|
||||
try:
|
||||
if dest_path.lower().endswith((".mp4", ".mov")):
|
||||
thumb_path = generate_video_thumbnail_mirrored(dest_path)
|
||||
else:
|
||||
thumb_path = generate_thumbnail(dest_path)
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to generate thumbnail for {original_name}: {e}")
|
||||
thumb_path = None
|
||||
|
||||
# Supersede the old record (preserves tags)
|
||||
record = supersede_image(
|
||||
old_record, dest_path, thumb_path, file_hash, phash, metadata, original_name
|
||||
)
|
||||
|
||||
# Apply any new tags from sidecar
|
||||
_apply_enriched_tags(record, enriched)
|
||||
|
||||
if commit:
|
||||
db.session.commit()
|
||||
|
||||
# Update existing_phashes list with new dimensions
|
||||
for i, (ph, w, h, img_id) in enumerate(existing_phashes):
|
||||
if img_id == similar_id:
|
||||
existing_phashes[i] = (phash, metadata["width"], metadata["height"], img_id)
|
||||
break
|
||||
|
||||
return (True, phash, record, True) # superseded=True
|
||||
|
||||
# No similar image found (or supersede disabled) - normal import
|
||||
# Enrich metadata from Gallery-DL sidecar JSON if present
|
||||
enriched = _get_enriched_metadata(src_path, metadata)
|
||||
|
||||
# Copy into /images/<artist>/<base>__<hash[:10]><ext>
|
||||
target_dir = os.path.join(dest_dir, artist)
|
||||
@@ -545,6 +797,7 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash
|
||||
thumb_path = None
|
||||
|
||||
# Create DB record (keep display name as original; filepath is content-addressed)
|
||||
# Use enriched taken_at (earliest date wins)
|
||||
record = ImageRecord(
|
||||
filename=original_name,
|
||||
filepath=dest_path,
|
||||
@@ -556,7 +809,7 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash
|
||||
height=metadata["height"],
|
||||
format=metadata["format"],
|
||||
camera_model=metadata["camera_model"],
|
||||
taken_at=metadata["taken_at"],
|
||||
taken_at=enriched.get("taken_at") or metadata["taken_at"],
|
||||
imported_at=datetime.utcnow()
|
||||
)
|
||||
|
||||
@@ -571,11 +824,81 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash
|
||||
if artist_tag not in record.tags:
|
||||
record.tags.append(artist_tag)
|
||||
|
||||
# Add tags from Gallery-DL metadata (source:platform, post:platform:artist:id)
|
||||
_apply_enriched_tags(record, enriched)
|
||||
|
||||
db.session.add(record)
|
||||
if commit:
|
||||
db.session.commit()
|
||||
return (True, phash, record)
|
||||
return (True, phash, record, False) # superseded=False (new import)
|
||||
|
||||
|
||||
def _get_enriched_metadata(src_path: str, existing_metadata: dict) -> dict:
|
||||
"""
|
||||
Get enriched metadata from Gallery-DL sidecar JSON.
|
||||
Returns empty dict if no sidecar found.
|
||||
"""
|
||||
try:
|
||||
from app.utils.metadata_enrichment import enrich_from_sidecar
|
||||
return enrich_from_sidecar(src_path, existing_metadata)
|
||||
except ImportError:
|
||||
return {}
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to enrich metadata for {src_path}: {e}")
|
||||
return {}
|
||||
|
||||
|
||||
def _apply_enriched_tags(record: ImageRecord, enriched: dict) -> None:
|
||||
"""
|
||||
Apply tags from enriched metadata to an ImageRecord.
|
||||
Creates tags if they don't exist.
|
||||
"""
|
||||
if not enriched or "tags" not in enriched:
|
||||
return
|
||||
|
||||
for tag_info in enriched.get("tags", []):
|
||||
tag_name = tag_info.get("name")
|
||||
tag_kind = tag_info.get("kind")
|
||||
if not tag_name:
|
||||
continue
|
||||
|
||||
tag = Tag.query.filter_by(name=tag_name).first()
|
||||
if not tag:
|
||||
tag = Tag(name=tag_name, kind=tag_kind)
|
||||
db.session.add(tag)
|
||||
db.session.flush()
|
||||
|
||||
if tag not in record.tags:
|
||||
record.tags.append(tag)
|
||||
print(f"[INFO] Added tag: {tag_name}")
|
||||
|
||||
|
||||
def _enrich_existing_record(record: ImageRecord, src_path: str) -> None:
|
||||
"""
|
||||
Enrich an existing record with metadata from sidecar JSON.
|
||||
Used when a duplicate is found - we still want to merge metadata.
|
||||
|
||||
- Updates taken_at if sidecar date is earlier
|
||||
- Adds source/post tags if not already present
|
||||
"""
|
||||
try:
|
||||
from app.utils.metadata_enrichment import enrich_from_sidecar, resolve_date_conflict
|
||||
|
||||
enriched = enrich_from_sidecar(src_path, {"taken_at": record.taken_at})
|
||||
if not enriched.get("raw_metadata"):
|
||||
return # No sidecar found
|
||||
|
||||
# Update date if sidecar has an earlier date
|
||||
new_date = resolve_date_conflict(record.taken_at, enriched.get("taken_at"))
|
||||
if new_date != record.taken_at:
|
||||
print(f"[INFO] Updating taken_at for {record.filename}: {record.taken_at} -> {new_date}")
|
||||
record.taken_at = new_date
|
||||
|
||||
# Add any new tags
|
||||
_apply_enriched_tags(record, enriched)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to enrich existing record: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -593,6 +916,10 @@ def generate_thumbnail(
|
||||
Save thumbnails mirrored under /images/thumbs/<...>.
|
||||
If prefer_jpeg is False (default), transparent images save as PNG, others as JPEG.
|
||||
If prefer_jpeg is True, all thumbs are JPEG with transparency flattened to `jpeg_bg`.
|
||||
|
||||
For images with extreme aspect ratios (very tall like comic strips, or very wide),
|
||||
crops to the center region before thumbnailing to produce better quality previews.
|
||||
|
||||
Returns the absolute path to the thumbnail.
|
||||
"""
|
||||
images_root = Path("/images").resolve()
|
||||
@@ -609,7 +936,27 @@ def generate_thumbnail(
|
||||
# Decide output extension/format
|
||||
with Image.open(image_path) as img:
|
||||
img = ImageOps.exif_transpose(img) # honor camera rotation
|
||||
img.thumbnail(size)
|
||||
|
||||
# Handle extreme aspect ratios by cropping to a more balanced region
|
||||
# This prevents very tall/wide images from becoming tiny slivers
|
||||
w, h = img.size
|
||||
aspect_ratio = w / h if h > 0 else 1
|
||||
|
||||
# If aspect ratio is extreme (< 0.5 means very tall, > 2.0 means very wide)
|
||||
# crop to a more balanced region before thumbnailing
|
||||
if aspect_ratio < 0.5:
|
||||
# Very tall image (like comic strip) - crop from top portion
|
||||
# Take a region that's closer to square, from the top
|
||||
crop_height = min(h, w * 2) # At most 2:1 tall
|
||||
img = img.crop((0, 0, w, crop_height))
|
||||
elif aspect_ratio > 2.0:
|
||||
# Very wide image - crop from center
|
||||
crop_width = min(w, h * 2) # At most 2:1 wide
|
||||
left = (w - crop_width) // 2
|
||||
img = img.crop((left, 0, left + crop_width, h))
|
||||
|
||||
# Use LANCZOS resampling for high-quality downscaling
|
||||
img.thumbnail(size, Image.Resampling.LANCZOS)
|
||||
|
||||
has_alpha = _has_alpha(img)
|
||||
|
||||
@@ -687,24 +1034,69 @@ def calculate_hash(file_path: str) -> str:
|
||||
return hash_func.hexdigest()
|
||||
|
||||
|
||||
def calculate_perceptual_hash(file_path: str):
|
||||
def calculate_perceptual_hash(file_path: str, hash_size: int = 16):
|
||||
"""
|
||||
Calculate a perceptual hash for an image.
|
||||
|
||||
Args:
|
||||
file_path: Path to the image file
|
||||
hash_size: Size of the hash grid (default 16 for 256-bit hash)
|
||||
hash_size=8 gives 64-bit, hash_size=16 gives 256-bit
|
||||
|
||||
Returns:
|
||||
ImageHash object or None if hashing fails (e.g., for videos)
|
||||
"""
|
||||
try:
|
||||
with Image.open(file_path) as img:
|
||||
return imagehash.phash(img)
|
||||
return imagehash.phash(img, hash_size=hash_size)
|
||||
except Exception:
|
||||
# videos or unreadable images land here (phash not applicable)
|
||||
return None
|
||||
|
||||
|
||||
def is_similar_image(phash, width: int, height: int, existing_phashes, threshold: int = 2) -> bool:
|
||||
for existing, ew, eh in existing_phashes:
|
||||
def find_similar_image(phash, width: int, height: int, existing_phashes, threshold: int = 10):
|
||||
"""
|
||||
Find visually similar images and determine relationship.
|
||||
|
||||
Args:
|
||||
phash: Perceptual hash of the new image
|
||||
width: Width of the new image
|
||||
height: Height of the new image
|
||||
existing_phashes: List of (phash, width, height, image_id) tuples for existing images
|
||||
threshold: Maximum Hamming distance to consider images similar
|
||||
Default is 10 for 256-bit hashes (was 2 for 64-bit)
|
||||
|
||||
Returns:
|
||||
tuple: (relationship, image_id) where relationship is one of:
|
||||
- None: No similar image found
|
||||
- "larger_exists": A larger or equal similar image exists (skip new image)
|
||||
- "smaller_exists": A smaller similar image exists (supersede it)
|
||||
image_id is the ID of the matching image, or None
|
||||
"""
|
||||
for existing_phash, ew, eh, image_id in existing_phashes:
|
||||
try:
|
||||
distance = phash - existing
|
||||
if distance <= threshold and ew >= width and eh >= height:
|
||||
return True
|
||||
distance = phash - existing_phash
|
||||
if distance <= threshold:
|
||||
# Similar image found - check size relationship
|
||||
if ew >= width and eh >= height:
|
||||
# Existing is larger or equal - skip new image
|
||||
return ("larger_exists", image_id)
|
||||
elif width > ew or height > eh:
|
||||
# New image is larger - supersede the existing one
|
||||
return ("smaller_exists", image_id)
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed pHash comparison: {e}")
|
||||
return False
|
||||
return (None, None)
|
||||
|
||||
|
||||
def is_similar_image(phash, width: int, height: int, existing_phashes, threshold: int = 10) -> bool:
|
||||
"""
|
||||
Check if an image is visually similar to any existing larger image.
|
||||
|
||||
Legacy wrapper for find_similar_image - returns True only if a larger version exists.
|
||||
"""
|
||||
relationship, _ = find_similar_image(phash, width, height, existing_phashes, threshold)
|
||||
return relationship == "larger_exists"
|
||||
|
||||
|
||||
def extract_metadata(file_path: str) -> dict:
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
# app/utils/metadata_enrichment.py
|
||||
"""
|
||||
Metadata enrichment from Gallery-DL sidecar JSON files.
|
||||
|
||||
This module handles:
|
||||
- Detection and parsing of Gallery-DL sidecar .json files
|
||||
- Extraction of metadata (dates, tags, post info) from various platforms
|
||||
- Tag generation for source tracking (source:platform, post:platform:artist:id)
|
||||
- Date conflict resolution (earliest date wins)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
log = logging.getLogger("metadata-enrichment")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Sidecar File Detection
|
||||
# =============================================================================
|
||||
|
||||
def find_sidecar_json(image_path: str) -> Optional[str]:
|
||||
"""
|
||||
Find the Gallery-DL sidecar JSON file for an image.
|
||||
|
||||
Gallery-DL creates JSON files with the UUID portion of the filename.
|
||||
For example:
|
||||
Image: 01_BBEFD45F-8775-4DF9-8BCA-99E6063E3616.jpg
|
||||
JSON: BBEFD45F-8775-4DF9-8BCA-99E6063E3616.json
|
||||
|
||||
Also checks for direct {filename}.json pattern.
|
||||
|
||||
Returns the path to the sidecar JSON if found, None otherwise.
|
||||
"""
|
||||
image_path = Path(image_path)
|
||||
parent_dir = image_path.parent
|
||||
stem = image_path.stem # filename without extension
|
||||
|
||||
# Pattern 1: Direct {filename}.json (e.g., image.jpg -> image.jpg.json)
|
||||
direct_json = image_path.with_suffix(image_path.suffix + ".json")
|
||||
if direct_json.exists():
|
||||
return str(direct_json)
|
||||
|
||||
# Pattern 2: Extract UUID from filename and look for {uuid}.json
|
||||
# Common patterns: "01_UUID", "UUID", "{num}_{UUID}"
|
||||
uuid_pattern = re.compile(r'([A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{12})', re.IGNORECASE)
|
||||
match = uuid_pattern.search(stem)
|
||||
if match:
|
||||
uuid = match.group(1)
|
||||
uuid_json = parent_dir / f"{uuid}.json"
|
||||
if uuid_json.exists():
|
||||
return str(uuid_json)
|
||||
|
||||
# Pattern 3: Just the stem without prefix numbers
|
||||
# e.g., "01_filename" -> look for "filename.json"
|
||||
stem_no_prefix = re.sub(r'^\d+_', '', stem)
|
||||
if stem_no_prefix != stem:
|
||||
alt_json = parent_dir / f"{stem_no_prefix}.json"
|
||||
if alt_json.exists():
|
||||
return str(alt_json)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def load_sidecar_metadata(json_path: str) -> Optional[dict]:
|
||||
"""Load and parse a Gallery-DL sidecar JSON file."""
|
||||
try:
|
||||
with open(json_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, IOError) as e:
|
||||
log.warning(f"Failed to load sidecar JSON {json_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Metadata Extraction (Platform-specific)
|
||||
# =============================================================================
|
||||
|
||||
def extract_platform(metadata: dict) -> Optional[str]:
|
||||
"""Extract the platform name from metadata."""
|
||||
# Check common fields
|
||||
if "category" in metadata:
|
||||
return metadata["category"].lower()
|
||||
|
||||
# Infer from URL patterns
|
||||
url = metadata.get("url", "") or metadata.get("post_url", "")
|
||||
if "patreon.com" in url:
|
||||
return "patreon"
|
||||
if "hentai-foundry" in url or "hentaifoundry" in url.lower():
|
||||
return "hentaifoundry"
|
||||
if "subscribestar" in url:
|
||||
return "subscribestar"
|
||||
if "discord.com" in url:
|
||||
return "discord"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extract_artist(metadata: dict) -> Optional[str]:
|
||||
"""Extract the artist/creator name from metadata."""
|
||||
# Try various common fields
|
||||
if "creator" in metadata and isinstance(metadata["creator"], dict):
|
||||
return metadata["creator"].get("vanity") or metadata["creator"].get("full_name")
|
||||
|
||||
if "author_name" in metadata:
|
||||
return metadata["author_name"]
|
||||
|
||||
if "artist" in metadata:
|
||||
return metadata["artist"]
|
||||
|
||||
# For hentaifoundry
|
||||
if "user" in metadata:
|
||||
return metadata["user"]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extract_post_id(metadata: dict) -> Optional[str]:
|
||||
"""Extract the post ID from metadata."""
|
||||
if "id" in metadata:
|
||||
return str(metadata["id"])
|
||||
|
||||
if "post_id" in metadata:
|
||||
return str(metadata["post_id"])
|
||||
|
||||
if "message_id" in metadata: # Discord
|
||||
return str(metadata["message_id"])
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extract_post_title(metadata: dict) -> Optional[str]:
|
||||
"""Extract the post title from metadata."""
|
||||
return metadata.get("title")
|
||||
|
||||
|
||||
def extract_date(metadata: dict) -> Optional[datetime]:
|
||||
"""
|
||||
Extract the publication/post date from metadata.
|
||||
Tries multiple common date fields and formats.
|
||||
"""
|
||||
date_fields = ["published_at", "date", "created_at", "timestamp"]
|
||||
|
||||
for field in date_fields:
|
||||
if field not in metadata:
|
||||
continue
|
||||
|
||||
value = metadata[field]
|
||||
if not value:
|
||||
continue
|
||||
|
||||
# Handle various formats
|
||||
parsed = _parse_date(value)
|
||||
if parsed:
|
||||
return parsed
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _parse_date(value) -> Optional[datetime]:
|
||||
"""Parse a date value that could be string, int, or already datetime."""
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
|
||||
if isinstance(value, (int, float)):
|
||||
# Unix timestamp
|
||||
try:
|
||||
return datetime.fromtimestamp(value)
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
|
||||
if isinstance(value, str):
|
||||
# Try common formats
|
||||
formats = [
|
||||
"%Y-%m-%dT%H:%M:%S.%f%z", # ISO with microseconds and timezone
|
||||
"%Y-%m-%dT%H:%M:%S%z", # ISO with timezone
|
||||
"%Y-%m-%dT%H:%M:%S.%f", # ISO with microseconds
|
||||
"%Y-%m-%dT%H:%M:%S", # ISO basic
|
||||
"%Y-%m-%d %H:%M:%S", # Space-separated
|
||||
"%Y-%m-%d", # Date only
|
||||
]
|
||||
|
||||
# Handle timezone offset with colon (Python < 3.11 compat)
|
||||
# e.g., "2023-08-01T04:20:02.000+00:00" -> "2023-08-01T04:20:02.000+0000"
|
||||
normalized = re.sub(r'([+-]\d{2}):(\d{2})$', r'\1\2', value)
|
||||
|
||||
for fmt in formats:
|
||||
try:
|
||||
return datetime.strptime(normalized, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
# Try without timezone info
|
||||
# Strip timezone suffix for naive parsing
|
||||
stripped = re.sub(r'[+-]\d{2}:?\d{2}$', '', value)
|
||||
stripped = re.sub(r'Z$', '', stripped)
|
||||
for fmt in formats:
|
||||
try:
|
||||
return datetime.strptime(stripped, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extract_content_tags(metadata: dict) -> list[str]:
|
||||
"""
|
||||
Extract content tags from metadata (not to be confused with DB tags).
|
||||
These are tags the creator applied to the post.
|
||||
"""
|
||||
tags = []
|
||||
|
||||
# Direct tags field
|
||||
if "tags" in metadata and isinstance(metadata["tags"], list):
|
||||
tags.extend(metadata["tags"])
|
||||
|
||||
return [str(t) for t in tags if t]
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Tag Generation
|
||||
# =============================================================================
|
||||
|
||||
def generate_source_tags(metadata: dict, artist_override: Optional[str] = None) -> list[dict]:
|
||||
"""
|
||||
Generate tags from Gallery-DL metadata.
|
||||
|
||||
Returns a list of tag dictionaries with 'name' and 'kind' keys.
|
||||
Tag format ensures uniqueness:
|
||||
- source:{platform} (kind='source')
|
||||
- post:{platform}:{artist}:{post_id} (kind='post')
|
||||
"""
|
||||
tags = []
|
||||
|
||||
platform = extract_platform(metadata)
|
||||
artist = artist_override or extract_artist(metadata)
|
||||
post_id = extract_post_id(metadata)
|
||||
|
||||
# Source platform tag
|
||||
if platform:
|
||||
tags.append({
|
||||
"name": f"source:{platform}",
|
||||
"kind": "source"
|
||||
})
|
||||
|
||||
# Post tag (unique across platform + artist + post_id)
|
||||
if platform and artist and post_id:
|
||||
# Sanitize artist name for tag (lowercase, replace spaces)
|
||||
safe_artist = artist.lower().replace(" ", "_")
|
||||
tags.append({
|
||||
"name": f"post:{platform}:{safe_artist}:{post_id}",
|
||||
"kind": "post"
|
||||
})
|
||||
|
||||
return tags
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Date Conflict Resolution
|
||||
# =============================================================================
|
||||
|
||||
def resolve_date_conflict(existing_date: Optional[datetime], new_date: Optional[datetime]) -> Optional[datetime]:
|
||||
"""
|
||||
Resolve date conflicts by keeping the earliest date.
|
||||
|
||||
Args:
|
||||
existing_date: The currently stored date (may be None)
|
||||
new_date: The new date from metadata (may be None)
|
||||
|
||||
Returns:
|
||||
The earliest non-None date, or None if both are None
|
||||
"""
|
||||
if existing_date is None:
|
||||
return new_date
|
||||
if new_date is None:
|
||||
return existing_date
|
||||
|
||||
# Return the earlier date
|
||||
return min(existing_date, new_date)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Main Enrichment Function
|
||||
# =============================================================================
|
||||
|
||||
def enrich_from_sidecar(image_path: str, existing_metadata: Optional[dict] = None) -> dict:
|
||||
"""
|
||||
Enrich image metadata from Gallery-DL sidecar JSON.
|
||||
|
||||
Args:
|
||||
image_path: Path to the image file
|
||||
existing_metadata: Optional dict with existing metadata (e.g., from EXIF)
|
||||
|
||||
Returns:
|
||||
Dictionary with enriched metadata:
|
||||
{
|
||||
"platform": str or None,
|
||||
"artist": str or None,
|
||||
"post_id": str or None,
|
||||
"post_title": str or None,
|
||||
"taken_at": datetime or None,
|
||||
"tags": [{"name": str, "kind": str}, ...],
|
||||
"raw_metadata": dict or None (the full sidecar content)
|
||||
}
|
||||
"""
|
||||
result = {
|
||||
"platform": None,
|
||||
"artist": None,
|
||||
"post_id": None,
|
||||
"post_title": None,
|
||||
"taken_at": None,
|
||||
"tags": [],
|
||||
"raw_metadata": None,
|
||||
}
|
||||
|
||||
# Find and load sidecar
|
||||
sidecar_path = find_sidecar_json(image_path)
|
||||
if not sidecar_path:
|
||||
return result
|
||||
|
||||
metadata = load_sidecar_metadata(sidecar_path)
|
||||
if not metadata:
|
||||
return result
|
||||
|
||||
result["raw_metadata"] = metadata
|
||||
|
||||
# Extract fields
|
||||
result["platform"] = extract_platform(metadata)
|
||||
result["artist"] = extract_artist(metadata)
|
||||
result["post_id"] = extract_post_id(metadata)
|
||||
result["post_title"] = extract_post_title(metadata)
|
||||
|
||||
# Extract date and resolve conflicts with existing
|
||||
sidecar_date = extract_date(metadata)
|
||||
existing_date = existing_metadata.get("taken_at") if existing_metadata else None
|
||||
result["taken_at"] = resolve_date_conflict(existing_date, sidecar_date)
|
||||
|
||||
# Generate tags
|
||||
result["tags"] = generate_source_tags(metadata, result["artist"])
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,288 @@
|
||||
# app/utils/version_migration.py
|
||||
"""
|
||||
Version tracking and data migration utilities.
|
||||
|
||||
This module handles:
|
||||
- Application version tracking via AppSettings
|
||||
- Automatic phash recomputation when version changes
|
||||
- Settings retrieval/storage from database
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
from PIL import Image
|
||||
import imagehash
|
||||
|
||||
from app import db
|
||||
from app.models import AppSettings, ImageRecord
|
||||
|
||||
log = logging.getLogger("version-migration")
|
||||
|
||||
# =============================================================================
|
||||
# Version and pHash Configuration
|
||||
# =============================================================================
|
||||
|
||||
APP_VERSION = "26.01.19.1" # Format: YY.MM.DD.iteration
|
||||
|
||||
# pHash configuration - 256-bit (hash_size=16 -> 16x16 grid)
|
||||
PHASH_HASH_SIZE = 16
|
||||
PHASH_BITS = PHASH_HASH_SIZE * PHASH_HASH_SIZE # 256
|
||||
PHASH_THRESHOLD_DEFAULT = 10 # Scaled from 2 (64-bit) to 10 (256-bit)
|
||||
|
||||
# Batch size for phash recomputation
|
||||
PHASH_RECOMPUTE_BATCH_SIZE = int(os.environ.get("PHASH_RECOMPUTE_BATCH_SIZE", "100"))
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Settings Access Functions
|
||||
# =============================================================================
|
||||
|
||||
def get_setting(key: str, default: Optional[str] = None) -> Optional[str]:
|
||||
"""
|
||||
Retrieve a setting value from the database.
|
||||
Returns default if the key doesn't exist.
|
||||
"""
|
||||
try:
|
||||
setting = AppSettings.query.filter_by(key=key).first()
|
||||
if setting is not None:
|
||||
return setting.value
|
||||
return default
|
||||
except Exception as e:
|
||||
log.warning(f"Failed to get setting '{key}': {e}")
|
||||
return default
|
||||
|
||||
|
||||
def set_setting(key: str, value: str) -> None:
|
||||
"""
|
||||
Store a setting value in the database.
|
||||
Creates the key if it doesn't exist, updates if it does.
|
||||
"""
|
||||
try:
|
||||
setting = AppSettings.query.filter_by(key=key).first()
|
||||
if setting is None:
|
||||
setting = AppSettings(key=key, value=value)
|
||||
db.session.add(setting)
|
||||
else:
|
||||
setting.value = value
|
||||
db.session.commit()
|
||||
except Exception as e:
|
||||
log.error(f"Failed to set setting '{key}': {e}")
|
||||
db.session.rollback()
|
||||
raise
|
||||
|
||||
|
||||
def get_setting_int(key: str, default: int) -> int:
|
||||
"""Get a setting as an integer."""
|
||||
val = get_setting(key)
|
||||
if val is None:
|
||||
return default
|
||||
try:
|
||||
return int(val)
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def get_setting_float(key: str, default: float) -> float:
|
||||
"""Get a setting as a float."""
|
||||
val = get_setting(key)
|
||||
if val is None:
|
||||
return default
|
||||
try:
|
||||
return float(val)
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def get_setting_bool(key: str, default: bool) -> bool:
|
||||
"""Get a setting as a boolean."""
|
||||
val = get_setting(key)
|
||||
if val is None:
|
||||
return default
|
||||
return val.lower() in ("true", "1", "yes")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Import Settings (DB-backed)
|
||||
# =============================================================================
|
||||
|
||||
def get_import_settings() -> dict:
|
||||
"""
|
||||
Load import filter settings from database, with environment variable fallbacks.
|
||||
This replaces the file-based load_import_settings() function.
|
||||
"""
|
||||
return {
|
||||
"min_width": get_setting_int("import_min_width", int(os.environ.get("IMPORT_MIN_WIDTH", "0"))),
|
||||
"min_height": get_setting_int("import_min_height", int(os.environ.get("IMPORT_MIN_HEIGHT", "0"))),
|
||||
"skip_transparent": get_setting_bool("import_skip_transparent", os.environ.get("IMPORT_SKIP_TRANSPARENT", "false").lower() == "true"),
|
||||
"transparency_threshold": get_setting_float("import_transparency_threshold", float(os.environ.get("IMPORT_TRANSPARENCY_THRESHOLD", "0.9"))),
|
||||
"phash_threshold": get_setting_int("phash_threshold", PHASH_THRESHOLD_DEFAULT),
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# pHash Computation (256-bit)
|
||||
# =============================================================================
|
||||
|
||||
def calculate_phash_256(file_path: str) -> Optional[imagehash.ImageHash]:
|
||||
"""
|
||||
Calculate a 256-bit perceptual hash for an image.
|
||||
Uses hash_size=16 for a 16x16 grid = 256 bits.
|
||||
"""
|
||||
try:
|
||||
with Image.open(file_path) as img:
|
||||
return imagehash.phash(img, hash_size=PHASH_HASH_SIZE)
|
||||
except Exception as e:
|
||||
log.debug(f"Could not compute phash for {file_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Version Migration Logic
|
||||
# =============================================================================
|
||||
|
||||
def recompute_all_phashes() -> dict:
|
||||
"""
|
||||
Recompute perceptual hashes for all images using 256-bit hashing.
|
||||
Returns statistics about the operation.
|
||||
"""
|
||||
stats = {
|
||||
"total": 0,
|
||||
"updated": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
}
|
||||
|
||||
log.info("Starting phash recomputation (256-bit)...")
|
||||
t_start = time.time()
|
||||
|
||||
batch_size = PHASH_RECOMPUTE_BATCH_SIZE
|
||||
last_id = 0
|
||||
|
||||
while True:
|
||||
# Fetch batch of images
|
||||
images = (
|
||||
ImageRecord.query
|
||||
.filter(ImageRecord.id > last_id)
|
||||
.order_by(ImageRecord.id.asc())
|
||||
.limit(batch_size)
|
||||
.all()
|
||||
)
|
||||
|
||||
if not images:
|
||||
break
|
||||
|
||||
batch_updated = 0
|
||||
for img in images:
|
||||
stats["total"] += 1
|
||||
|
||||
# Skip videos (no phash)
|
||||
ext = (img.format or "").lower()
|
||||
if ext in ("mp4", "mov") or (img.filepath and img.filepath.lower().endswith((".mp4", ".mov"))):
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
|
||||
# Skip if file doesn't exist
|
||||
if not img.filepath or not os.path.exists(img.filepath):
|
||||
log.warning(f"File not found for image id={img.id}: {img.filepath}")
|
||||
stats["failed"] += 1
|
||||
continue
|
||||
|
||||
# Compute new phash
|
||||
try:
|
||||
phash = calculate_phash_256(img.filepath)
|
||||
if phash:
|
||||
img.perceptual_hash = str(phash)
|
||||
batch_updated += 1
|
||||
stats["updated"] += 1
|
||||
else:
|
||||
stats["failed"] += 1
|
||||
except Exception as e:
|
||||
log.warning(f"Failed to compute phash for id={img.id}: {e}")
|
||||
stats["failed"] += 1
|
||||
|
||||
# Commit batch
|
||||
db.session.commit()
|
||||
last_id = images[-1].id
|
||||
log.info(f"Processed batch up to id={last_id}, updated={batch_updated}")
|
||||
|
||||
elapsed = time.time() - t_start
|
||||
log.info(
|
||||
f"pHash recomputation complete in {elapsed:.1f}s: "
|
||||
f"total={stats['total']}, updated={stats['updated']}, "
|
||||
f"skipped={stats['skipped']}, failed={stats['failed']}"
|
||||
)
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def check_and_run_migrations() -> None:
|
||||
"""
|
||||
Check the stored app version and run data migrations if needed.
|
||||
|
||||
This function:
|
||||
1. Reads the stored app_version from the database
|
||||
2. Compares it with the current APP_VERSION
|
||||
3. If different, triggers necessary data migrations (e.g., phash recomputation)
|
||||
4. Updates the stored version
|
||||
"""
|
||||
log.info(f"Checking app version (current: {APP_VERSION})...")
|
||||
|
||||
stored_version = get_setting("app_version")
|
||||
phash_version = get_setting("phash_version")
|
||||
|
||||
log.info(f"Stored version: {stored_version}, pHash version: {phash_version}")
|
||||
|
||||
# Initialize settings if this is first run
|
||||
if stored_version is None:
|
||||
log.info("First run detected, initializing settings...")
|
||||
_initialize_default_settings()
|
||||
stored_version = "0.0.0.0" # Force migration on first run
|
||||
|
||||
# Check if phash recomputation is needed
|
||||
needs_phash_recompute = (
|
||||
phash_version is None or
|
||||
phash_version != APP_VERSION or
|
||||
get_setting_int("phash_bits", 64) != PHASH_BITS
|
||||
)
|
||||
|
||||
if needs_phash_recompute:
|
||||
log.info(f"pHash migration needed (stored bits: {get_setting('phash_bits')}, new: {PHASH_BITS})")
|
||||
|
||||
# Update phash configuration
|
||||
set_setting("phash_bits", str(PHASH_BITS))
|
||||
set_setting("phash_hash_size", str(PHASH_HASH_SIZE))
|
||||
|
||||
# Recompute all hashes
|
||||
recompute_all_phashes()
|
||||
|
||||
# Mark phash version as current
|
||||
set_setting("phash_version", APP_VERSION)
|
||||
|
||||
# Update app version
|
||||
if stored_version != APP_VERSION:
|
||||
log.info(f"Updating app version from {stored_version} to {APP_VERSION}")
|
||||
set_setting("app_version", APP_VERSION)
|
||||
|
||||
log.info("Version check complete.")
|
||||
|
||||
|
||||
def _initialize_default_settings() -> None:
|
||||
"""Initialize default settings on first run."""
|
||||
defaults = {
|
||||
"app_version": APP_VERSION,
|
||||
"phash_version": None, # Will trigger recomputation
|
||||
"phash_bits": str(PHASH_BITS),
|
||||
"phash_hash_size": str(PHASH_HASH_SIZE),
|
||||
"phash_threshold": str(PHASH_THRESHOLD_DEFAULT),
|
||||
"import_min_width": "0",
|
||||
"import_min_height": "0",
|
||||
"import_skip_transparent": "false",
|
||||
"import_transparency_threshold": "0.9",
|
||||
}
|
||||
|
||||
for key, value in defaults.items():
|
||||
if get_setting(key) is None and value is not None:
|
||||
set_setting(key, value)
|
||||
Reference in New Issue
Block a user