tuning import process and db handling

app/__init__.py

@@ -18,6 +18,21 @@ def create_app(config_class='config.Config'):
     login_manager.init_app(app)
     login_manager.login_view = 'auth.login'
 
+    with app.app_context():
+        # Enable connection ping only for PostgreSQL
+        if app.config['SQLALCHEMY_DATABASE_URI'].startswith("postgresql"):
+            from sqlalchemy import event, text
+            from sqlalchemy.exc import DisconnectionError
+
+            @event.listens_for(db.engine, "engine_connect")
+            def ping_connection(connection, branch):
+                if branch:
+                    return
+                try:
+                    connection.execution_options(isolation_level="AUTOCOMMIT").scalar(text("SELECT 1"))
+                except:
+                    raise DisconnectionError()
+
     from app.main import main
     from app.auth import auth
 
@@ -28,5 +43,4 @@ def create_app(config_class='config.Config'):
     def datetimeformat(value, format="%Y-%m-%d"):
         return value.strftime(format) if value else ""
 
-
     return app

app/utils/image_importer.py

@@ -15,6 +15,12 @@ THUMB_SIZE = (400, 400)
 
 def import_images_task(source_dir, dest_dir):
     imported = []
+    batch_size = 10
+    batch_counter = 0
+    existing_phashes = [
+        (imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height)
+        for img in ImageRecord.query.filter(ImageRecord.perceptual_hash != None).all()
+    ]
 
     for artist_dir in os.listdir(source_dir):
         artist_path = os.path.join(source_dir, artist_dir)
@@ -26,18 +32,26 @@ def import_images_task(source_dir, dest_dir):
                 if file.lower().endswith(('.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff')):
                     full_path = os.path.join(root, file)
                     filename = os.path.basename(full_path)
-                    file_hash = calculate_hash(full_path)
 
+                    print(f"[INFO] Processing: {full_path}")
+                    file_size = os.path.getsize(full_path)
+
+                    if ImageRecord.query.filter_by(filename=filename, file_size=file_size).first():
+                        print(f"[SKIP] Duplicate by name+size: {filename}")
+                        continue
+                    
+                    file_hash = calculate_hash(full_path)
                     if ImageRecord.query.filter_by(hash=file_hash).first():
+                        print(f"[SKIP] Duplicate by hash: {filename}")
                         continue
 
                     metadata = extract_metadata(full_path)
-
                     if not metadata['width'] or not metadata['height']:
+                        print(f"[SKIP] Missing dimension data for {filename}")
                         continue
 
                     phash = calculate_perceptual_hash(full_path)
-                    if phash and is_similar_image(phash, metadata['width'], metadata['height']):
+                    if phash and is_similar_image(phash, metadata['width'], metadata['height'], existing_phashes):
                         print(f"[SKIP] {filename} is visually similar to an existing larger image.")
                         continue
 
@@ -46,11 +60,10 @@ def import_images_task(source_dir, dest_dir):
                     dest_path = os.path.join(target_dir, filename)
                     shutil.copy2(full_path, dest_path)
 
-                    # Generate thumbnail AFTER copying to /images
                     try:
                         thumb_path = generate_thumbnail(dest_path)
                     except Exception as e:
-                        print(f"[WARN] Failed to generate thumbnail: {e}")
+                        print(f"[WARN] Failed to generate thumbnail for {filename}: {e}")
                         thumb_path = None
 
                     record = ImageRecord(
@@ -71,9 +84,24 @@ def import_images_task(source_dir, dest_dir):
                     db.session.add(record)
                     imported.append(filename)
 
-                    db.session.commit()
-                    time.sleep(1)
+                    if phash:
+                        existing_phashes.append((phash, metadata['width'], metadata['height']))
 
+                    batch_counter += 1
+
+                    if batch_counter >= batch_size:
+                        db.session.commit()
+                        print(f"[INFO] Committed batch of {batch_size} images.")
+                        batch_counter = 0
+
+                        if 'sqlite' in db.engine.url.drivername:
+                            time.sleep(1)
+
+    if batch_counter > 0:
+        db.session.commit()
+        print(f"[INFO] Committed final batch of {batch_counter} images.")
+
+    print(f"[INFO] Import complete. Total images imported: {len(imported)}")
     return f"Imported {len(imported)} images."
 
 
@@ -116,18 +144,17 @@ def calculate_perceptual_hash(file_path):
         print(f"[WARN] Failed to compute perceptual hash for {file_path}: {e}")
         return None
 
-def is_similar_image(phash, width, height, threshold=2):
-    for img in ImageRecord.query.filter(ImageRecord.perceptual_hash != None).all():
+def is_similar_image(phash, width, height, existing_phashes, threshold=2):
+    for existing, ew, eh in existing_phashes:
         try:
-            existing_phash = imagehash.hex_to_hash(img.perceptual_hash)
-            distance = phash - existing_phash
-            if distance <= threshold:
-                if img.width >= width and img.height >= height:
-                    return True
+            distance = phash - existing
+            if distance <= threshold and ew >= width and eh >= height:
+                return True
         except Exception as e:
             print(f"[WARN] Failed pHash comparison: {e}")
     return False
 
+
 def extract_metadata(file_path):
     metadata = {
         'file_size': None,