# app/utils/version_migration.py """ Version tracking and data migration utilities. This module handles: - Application version tracking via AppSettings - Automatic phash recomputation when version changes - Settings retrieval/storage from database """ import logging import os import time from typing import Optional from PIL import Image import imagehash from app import db from app.models import AppSettings, ImageRecord log = logging.getLogger("version-migration") # ============================================================================= # Version and pHash Configuration # ============================================================================= APP_VERSION = "26.01.19.1" # Format: YY.MM.DD.iteration # pHash configuration - 256-bit (hash_size=16 -> 16x16 grid) PHASH_HASH_SIZE = 16 PHASH_BITS = PHASH_HASH_SIZE * PHASH_HASH_SIZE # 256 PHASH_THRESHOLD_DEFAULT = 10 # Scaled from 2 (64-bit) to 10 (256-bit) # Batch size for phash recomputation PHASH_RECOMPUTE_BATCH_SIZE = int(os.environ.get("PHASH_RECOMPUTE_BATCH_SIZE", "100")) # ============================================================================= # Settings Access Functions # ============================================================================= def get_setting(key: str, default: Optional[str] = None) -> Optional[str]: """ Retrieve a setting value from the database. Returns default if the key doesn't exist. """ try: setting = AppSettings.query.filter_by(key=key).first() if setting is not None: return setting.value return default except Exception as e: log.warning(f"Failed to get setting '{key}': {e}") return default def set_setting(key: str, value: str) -> None: """ Store a setting value in the database. Creates the key if it doesn't exist, updates if it does. """ try: setting = AppSettings.query.filter_by(key=key).first() if setting is None: setting = AppSettings(key=key, value=value) db.session.add(setting) else: setting.value = value db.session.commit() except Exception as e: log.error(f"Failed to set setting '{key}': {e}") db.session.rollback() raise def get_setting_int(key: str, default: int) -> int: """Get a setting as an integer.""" val = get_setting(key) if val is None: return default try: return int(val) except ValueError: return default def get_setting_float(key: str, default: float) -> float: """Get a setting as a float.""" val = get_setting(key) if val is None: return default try: return float(val) except ValueError: return default def get_setting_bool(key: str, default: bool) -> bool: """Get a setting as a boolean.""" val = get_setting(key) if val is None: return default return val.lower() in ("true", "1", "yes") # ============================================================================= # Import Settings (DB-backed) # ============================================================================= def get_import_settings() -> dict: """ Load import filter settings from database, with environment variable fallbacks. This replaces the file-based load_import_settings() function. """ return { "min_width": get_setting_int("import_min_width", int(os.environ.get("IMPORT_MIN_WIDTH", "0"))), "min_height": get_setting_int("import_min_height", int(os.environ.get("IMPORT_MIN_HEIGHT", "0"))), "skip_transparent": get_setting_bool("import_skip_transparent", os.environ.get("IMPORT_SKIP_TRANSPARENT", "false").lower() == "true"), "transparency_threshold": get_setting_float("import_transparency_threshold", float(os.environ.get("IMPORT_TRANSPARENCY_THRESHOLD", "0.9"))), "phash_threshold": get_setting_int("phash_threshold", PHASH_THRESHOLD_DEFAULT), } # ============================================================================= # pHash Computation (256-bit) # ============================================================================= def calculate_phash_256(file_path: str) -> Optional[imagehash.ImageHash]: """ Calculate a 256-bit perceptual hash for an image. Uses hash_size=16 for a 16x16 grid = 256 bits. """ try: with Image.open(file_path) as img: return imagehash.phash(img, hash_size=PHASH_HASH_SIZE) except Exception as e: log.debug(f"Could not compute phash for {file_path}: {e}") return None # ============================================================================= # Version Migration Logic # ============================================================================= def recompute_all_phashes() -> dict: """ Recompute perceptual hashes for all images using 256-bit hashing. Returns statistics about the operation. """ stats = { "total": 0, "updated": 0, "skipped": 0, "failed": 0, } log.info("Starting phash recomputation (256-bit)...") t_start = time.time() batch_size = PHASH_RECOMPUTE_BATCH_SIZE last_id = 0 while True: # Fetch batch of images images = ( ImageRecord.query .filter(ImageRecord.id > last_id) .order_by(ImageRecord.id.asc()) .limit(batch_size) .all() ) if not images: break batch_updated = 0 for img in images: stats["total"] += 1 # Skip videos (no phash) ext = (img.format or "").lower() if ext in ("mp4", "mov") or (img.filepath and img.filepath.lower().endswith((".mp4", ".mov"))): stats["skipped"] += 1 continue # Skip if file doesn't exist if not img.filepath or not os.path.exists(img.filepath): log.warning(f"File not found for image id={img.id}: {img.filepath}") stats["failed"] += 1 continue # Compute new phash try: phash = calculate_phash_256(img.filepath) if phash: img.perceptual_hash = str(phash) batch_updated += 1 stats["updated"] += 1 else: stats["failed"] += 1 except Exception as e: log.warning(f"Failed to compute phash for id={img.id}: {e}") stats["failed"] += 1 # Commit batch db.session.commit() last_id = images[-1].id log.info(f"Processed batch up to id={last_id}, updated={batch_updated}") elapsed = time.time() - t_start log.info( f"pHash recomputation complete in {elapsed:.1f}s: " f"total={stats['total']}, updated={stats['updated']}, " f"skipped={stats['skipped']}, failed={stats['failed']}" ) return stats def check_and_run_migrations() -> None: """ Check the stored app version and run data migrations if needed. This function: 1. Reads the stored app_version from the database 2. Compares it with the current APP_VERSION 3. If different, triggers necessary data migrations (e.g., phash recomputation) 4. Updates the stored version """ log.info(f"Checking app version (current: {APP_VERSION})...") stored_version = get_setting("app_version") phash_version = get_setting("phash_version") log.info(f"Stored version: {stored_version}, pHash version: {phash_version}") # Initialize settings if this is first run if stored_version is None: log.info("First run detected, initializing settings...") _initialize_default_settings() stored_version = "0.0.0.0" # Force migration on first run # Check if phash recomputation is needed needs_phash_recompute = ( phash_version is None or phash_version != APP_VERSION or get_setting_int("phash_bits", 64) != PHASH_BITS ) if needs_phash_recompute: log.info(f"pHash migration needed (stored bits: {get_setting('phash_bits')}, new: {PHASH_BITS})") # Update phash configuration set_setting("phash_bits", str(PHASH_BITS)) set_setting("phash_hash_size", str(PHASH_HASH_SIZE)) # Recompute all hashes recompute_all_phashes() # Mark phash version as current set_setting("phash_version", APP_VERSION) # Update app version if stored_version != APP_VERSION: log.info(f"Updating app version from {stored_version} to {APP_VERSION}") set_setting("app_version", APP_VERSION) log.info("Version check complete.") def _initialize_default_settings() -> None: """Initialize default settings on first run.""" defaults = { "app_version": APP_VERSION, "phash_version": None, # Will trigger recomputation "phash_bits": str(PHASH_BITS), "phash_hash_size": str(PHASH_HASH_SIZE), "phash_threshold": str(PHASH_THRESHOLD_DEFAULT), "import_min_width": "0", "import_min_height": "0", "import_skip_transparent": "false", "import_transparency_threshold": "0.9", } for key, value in defaults.items(): if get_setting(key) is None and value is not None: set_setting(key, value)