moving styling and views to be more consistent and for gallery view to be less cumbersome.
This commit is contained in:
Bryan Van Deusen
@@ -0,0 +1,288 @@
|
||||
# app/utils/version_migration.py
|
||||
"""
|
||||
Version tracking and data migration utilities.
|
||||
|
||||
This module handles:
|
||||
- Application version tracking via AppSettings
|
||||
- Automatic phash recomputation when version changes
|
||||
- Settings retrieval/storage from database
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
from PIL import Image
|
||||
import imagehash
|
||||
|
||||
from app import db
|
||||
from app.models import AppSettings, ImageRecord
|
||||
|
||||
log = logging.getLogger("version-migration")
|
||||
|
||||
# =============================================================================
|
||||
# Version and pHash Configuration
|
||||
# =============================================================================
|
||||
|
||||
APP_VERSION = "26.01.19.1" # Format: YY.MM.DD.iteration
|
||||
|
||||
# pHash configuration - 256-bit (hash_size=16 -> 16x16 grid)
|
||||
PHASH_HASH_SIZE = 16
|
||||
PHASH_BITS = PHASH_HASH_SIZE * PHASH_HASH_SIZE # 256
|
||||
PHASH_THRESHOLD_DEFAULT = 10 # Scaled from 2 (64-bit) to 10 (256-bit)
|
||||
|
||||
# Batch size for phash recomputation
|
||||
PHASH_RECOMPUTE_BATCH_SIZE = int(os.environ.get("PHASH_RECOMPUTE_BATCH_SIZE", "100"))
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Settings Access Functions
|
||||
# =============================================================================
|
||||
|
||||
def get_setting(key: str, default: Optional[str] = None) -> Optional[str]:
|
||||
"""
|
||||
Retrieve a setting value from the database.
|
||||
Returns default if the key doesn't exist.
|
||||
"""
|
||||
try:
|
||||
setting = AppSettings.query.filter_by(key=key).first()
|
||||
if setting is not None:
|
||||
return setting.value
|
||||
return default
|
||||
except Exception as e:
|
||||
log.warning(f"Failed to get setting '{key}': {e}")
|
||||
return default
|
||||
|
||||
|
||||
def set_setting(key: str, value: str) -> None:
|
||||
"""
|
||||
Store a setting value in the database.
|
||||
Creates the key if it doesn't exist, updates if it does.
|
||||
"""
|
||||
try:
|
||||
setting = AppSettings.query.filter_by(key=key).first()
|
||||
if setting is None:
|
||||
setting = AppSettings(key=key, value=value)
|
||||
db.session.add(setting)
|
||||
else:
|
||||
setting.value = value
|
||||
db.session.commit()
|
||||
except Exception as e:
|
||||
log.error(f"Failed to set setting '{key}': {e}")
|
||||
db.session.rollback()
|
||||
raise
|
||||
|
||||
|
||||
def get_setting_int(key: str, default: int) -> int:
|
||||
"""Get a setting as an integer."""
|
||||
val = get_setting(key)
|
||||
if val is None:
|
||||
return default
|
||||
try:
|
||||
return int(val)
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def get_setting_float(key: str, default: float) -> float:
|
||||
"""Get a setting as a float."""
|
||||
val = get_setting(key)
|
||||
if val is None:
|
||||
return default
|
||||
try:
|
||||
return float(val)
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def get_setting_bool(key: str, default: bool) -> bool:
|
||||
"""Get a setting as a boolean."""
|
||||
val = get_setting(key)
|
||||
if val is None:
|
||||
return default
|
||||
return val.lower() in ("true", "1", "yes")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Import Settings (DB-backed)
|
||||
# =============================================================================
|
||||
|
||||
def get_import_settings() -> dict:
|
||||
"""
|
||||
Load import filter settings from database, with environment variable fallbacks.
|
||||
This replaces the file-based load_import_settings() function.
|
||||
"""
|
||||
return {
|
||||
"min_width": get_setting_int("import_min_width", int(os.environ.get("IMPORT_MIN_WIDTH", "0"))),
|
||||
"min_height": get_setting_int("import_min_height", int(os.environ.get("IMPORT_MIN_HEIGHT", "0"))),
|
||||
"skip_transparent": get_setting_bool("import_skip_transparent", os.environ.get("IMPORT_SKIP_TRANSPARENT", "false").lower() == "true"),
|
||||
"transparency_threshold": get_setting_float("import_transparency_threshold", float(os.environ.get("IMPORT_TRANSPARENCY_THRESHOLD", "0.9"))),
|
||||
"phash_threshold": get_setting_int("phash_threshold", PHASH_THRESHOLD_DEFAULT),
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# pHash Computation (256-bit)
|
||||
# =============================================================================
|
||||
|
||||
def calculate_phash_256(file_path: str) -> Optional[imagehash.ImageHash]:
|
||||
"""
|
||||
Calculate a 256-bit perceptual hash for an image.
|
||||
Uses hash_size=16 for a 16x16 grid = 256 bits.
|
||||
"""
|
||||
try:
|
||||
with Image.open(file_path) as img:
|
||||
return imagehash.phash(img, hash_size=PHASH_HASH_SIZE)
|
||||
except Exception as e:
|
||||
log.debug(f"Could not compute phash for {file_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Version Migration Logic
|
||||
# =============================================================================
|
||||
|
||||
def recompute_all_phashes() -> dict:
|
||||
"""
|
||||
Recompute perceptual hashes for all images using 256-bit hashing.
|
||||
Returns statistics about the operation.
|
||||
"""
|
||||
stats = {
|
||||
"total": 0,
|
||||
"updated": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
}
|
||||
|
||||
log.info("Starting phash recomputation (256-bit)...")
|
||||
t_start = time.time()
|
||||
|
||||
batch_size = PHASH_RECOMPUTE_BATCH_SIZE
|
||||
last_id = 0
|
||||
|
||||
while True:
|
||||
# Fetch batch of images
|
||||
images = (
|
||||
ImageRecord.query
|
||||
.filter(ImageRecord.id > last_id)
|
||||
.order_by(ImageRecord.id.asc())
|
||||
.limit(batch_size)
|
||||
.all()
|
||||
)
|
||||
|
||||
if not images:
|
||||
break
|
||||
|
||||
batch_updated = 0
|
||||
for img in images:
|
||||
stats["total"] += 1
|
||||
|
||||
# Skip videos (no phash)
|
||||
ext = (img.format or "").lower()
|
||||
if ext in ("mp4", "mov") or (img.filepath and img.filepath.lower().endswith((".mp4", ".mov"))):
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
|
||||
# Skip if file doesn't exist
|
||||
if not img.filepath or not os.path.exists(img.filepath):
|
||||
log.warning(f"File not found for image id={img.id}: {img.filepath}")
|
||||
stats["failed"] += 1
|
||||
continue
|
||||
|
||||
# Compute new phash
|
||||
try:
|
||||
phash = calculate_phash_256(img.filepath)
|
||||
if phash:
|
||||
img.perceptual_hash = str(phash)
|
||||
batch_updated += 1
|
||||
stats["updated"] += 1
|
||||
else:
|
||||
stats["failed"] += 1
|
||||
except Exception as e:
|
||||
log.warning(f"Failed to compute phash for id={img.id}: {e}")
|
||||
stats["failed"] += 1
|
||||
|
||||
# Commit batch
|
||||
db.session.commit()
|
||||
last_id = images[-1].id
|
||||
log.info(f"Processed batch up to id={last_id}, updated={batch_updated}")
|
||||
|
||||
elapsed = time.time() - t_start
|
||||
log.info(
|
||||
f"pHash recomputation complete in {elapsed:.1f}s: "
|
||||
f"total={stats['total']}, updated={stats['updated']}, "
|
||||
f"skipped={stats['skipped']}, failed={stats['failed']}"
|
||||
)
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def check_and_run_migrations() -> None:
|
||||
"""
|
||||
Check the stored app version and run data migrations if needed.
|
||||
|
||||
This function:
|
||||
1. Reads the stored app_version from the database
|
||||
2. Compares it with the current APP_VERSION
|
||||
3. If different, triggers necessary data migrations (e.g., phash recomputation)
|
||||
4. Updates the stored version
|
||||
"""
|
||||
log.info(f"Checking app version (current: {APP_VERSION})...")
|
||||
|
||||
stored_version = get_setting("app_version")
|
||||
phash_version = get_setting("phash_version")
|
||||
|
||||
log.info(f"Stored version: {stored_version}, pHash version: {phash_version}")
|
||||
|
||||
# Initialize settings if this is first run
|
||||
if stored_version is None:
|
||||
log.info("First run detected, initializing settings...")
|
||||
_initialize_default_settings()
|
||||
stored_version = "0.0.0.0" # Force migration on first run
|
||||
|
||||
# Check if phash recomputation is needed
|
||||
needs_phash_recompute = (
|
||||
phash_version is None or
|
||||
phash_version != APP_VERSION or
|
||||
get_setting_int("phash_bits", 64) != PHASH_BITS
|
||||
)
|
||||
|
||||
if needs_phash_recompute:
|
||||
log.info(f"pHash migration needed (stored bits: {get_setting('phash_bits')}, new: {PHASH_BITS})")
|
||||
|
||||
# Update phash configuration
|
||||
set_setting("phash_bits", str(PHASH_BITS))
|
||||
set_setting("phash_hash_size", str(PHASH_HASH_SIZE))
|
||||
|
||||
# Recompute all hashes
|
||||
recompute_all_phashes()
|
||||
|
||||
# Mark phash version as current
|
||||
set_setting("phash_version", APP_VERSION)
|
||||
|
||||
# Update app version
|
||||
if stored_version != APP_VERSION:
|
||||
log.info(f"Updating app version from {stored_version} to {APP_VERSION}")
|
||||
set_setting("app_version", APP_VERSION)
|
||||
|
||||
log.info("Version check complete.")
|
||||
|
||||
|
||||
def _initialize_default_settings() -> None:
|
||||
"""Initialize default settings on first run."""
|
||||
defaults = {
|
||||
"app_version": APP_VERSION,
|
||||
"phash_version": None, # Will trigger recomputation
|
||||
"phash_bits": str(PHASH_BITS),
|
||||
"phash_hash_size": str(PHASH_HASH_SIZE),
|
||||
"phash_threshold": str(PHASH_THRESHOLD_DEFAULT),
|
||||
"import_min_width": "0",
|
||||
"import_min_height": "0",
|
||||
"import_skip_transparent": "false",
|
||||
"import_transparency_threshold": "0.9",
|
||||
}
|
||||
|
||||
for key, value in defaults.items():
|
||||
if get_setting(key) is None and value is not None:
|
||||
set_setting(key, value)
|
||||
Reference in New Issue
Block a user