From 3c3bfed75fdf9f5166abb8ca68a02fdc1cfe0026 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 3 Aug 2025 15:16:02 -0400 Subject: [PATCH] tuning import process and db handling --- app/__init__.py | 16 ++++++++++- app/utils/image_importer.py | 55 +++++++++++++++++++++++++++---------- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 5ca4fe7..a316d47 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -18,6 +18,21 @@ def create_app(config_class='config.Config'): login_manager.init_app(app) login_manager.login_view = 'auth.login' + with app.app_context(): + # Enable connection ping only for PostgreSQL + if app.config['SQLALCHEMY_DATABASE_URI'].startswith("postgresql"): + from sqlalchemy import event, text + from sqlalchemy.exc import DisconnectionError + + @event.listens_for(db.engine, "engine_connect") + def ping_connection(connection, branch): + if branch: + return + try: + connection.execution_options(isolation_level="AUTOCOMMIT").scalar(text("SELECT 1")) + except: + raise DisconnectionError() + from app.main import main from app.auth import auth @@ -28,5 +43,4 @@ def create_app(config_class='config.Config'): def datetimeformat(value, format="%Y-%m-%d"): return value.strftime(format) if value else "" - return app diff --git a/app/utils/image_importer.py b/app/utils/image_importer.py index 484a1f8..9fa757e 100644 --- a/app/utils/image_importer.py +++ b/app/utils/image_importer.py @@ -15,6 +15,12 @@ THUMB_SIZE = (400, 400) def import_images_task(source_dir, dest_dir): imported = [] + batch_size = 10 + batch_counter = 0 + existing_phashes = [ + (imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height) + for img in ImageRecord.query.filter(ImageRecord.perceptual_hash != None).all() + ] for artist_dir in os.listdir(source_dir): artist_path = os.path.join(source_dir, artist_dir) @@ -26,18 +32,26 @@ def import_images_task(source_dir, dest_dir): if file.lower().endswith(('.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff')): full_path = os.path.join(root, file) filename = os.path.basename(full_path) - file_hash = calculate_hash(full_path) + print(f"[INFO] Processing: {full_path}") + file_size = os.path.getsize(full_path) + + if ImageRecord.query.filter_by(filename=filename, file_size=file_size).first(): + print(f"[SKIP] Duplicate by name+size: {filename}") + continue + + file_hash = calculate_hash(full_path) if ImageRecord.query.filter_by(hash=file_hash).first(): + print(f"[SKIP] Duplicate by hash: {filename}") continue metadata = extract_metadata(full_path) - if not metadata['width'] or not metadata['height']: + print(f"[SKIP] Missing dimension data for {filename}") continue phash = calculate_perceptual_hash(full_path) - if phash and is_similar_image(phash, metadata['width'], metadata['height']): + if phash and is_similar_image(phash, metadata['width'], metadata['height'], existing_phashes): print(f"[SKIP] {filename} is visually similar to an existing larger image.") continue @@ -46,11 +60,10 @@ def import_images_task(source_dir, dest_dir): dest_path = os.path.join(target_dir, filename) shutil.copy2(full_path, dest_path) - # Generate thumbnail AFTER copying to /images try: thumb_path = generate_thumbnail(dest_path) except Exception as e: - print(f"[WARN] Failed to generate thumbnail: {e}") + print(f"[WARN] Failed to generate thumbnail for {filename}: {e}") thumb_path = None record = ImageRecord( @@ -71,9 +84,24 @@ def import_images_task(source_dir, dest_dir): db.session.add(record) imported.append(filename) - db.session.commit() - time.sleep(1) + if phash: + existing_phashes.append((phash, metadata['width'], metadata['height'])) + batch_counter += 1 + + if batch_counter >= batch_size: + db.session.commit() + print(f"[INFO] Committed batch of {batch_size} images.") + batch_counter = 0 + + if 'sqlite' in db.engine.url.drivername: + time.sleep(1) + + if batch_counter > 0: + db.session.commit() + print(f"[INFO] Committed final batch of {batch_counter} images.") + + print(f"[INFO] Import complete. Total images imported: {len(imported)}") return f"Imported {len(imported)} images." @@ -116,18 +144,17 @@ def calculate_perceptual_hash(file_path): print(f"[WARN] Failed to compute perceptual hash for {file_path}: {e}") return None -def is_similar_image(phash, width, height, threshold=2): - for img in ImageRecord.query.filter(ImageRecord.perceptual_hash != None).all(): +def is_similar_image(phash, width, height, existing_phashes, threshold=2): + for existing, ew, eh in existing_phashes: try: - existing_phash = imagehash.hex_to_hash(img.perceptual_hash) - distance = phash - existing_phash - if distance <= threshold: - if img.width >= width and img.height >= height: - return True + distance = phash - existing + if distance <= threshold and ew >= width and eh >= height: + return True except Exception as e: print(f"[WARN] Failed pHash comparison: {e}") return False + def extract_metadata(file_path): metadata = { 'file_size': None,