fixed pagination issues and thumbnail generation issues

app/static/style.css

@@ -329,18 +329,25 @@ header {
   gap: .25rem;
 }
 
+/* Tag chips (truncate long names nicely) */
 .tag-chip {
-  font: 12px/1.6 system-ui, sans-serif;
+  display: inline-block;             /* required for ellipsis */
+  max-width: clamp(8rem, 22vw, 14rem); /* responsive cap */
   padding: 2px 8px;
   border-radius: 999px;
   background: rgba(255,255,255,0.92);
   color: #334155;
   text-decoration: none;
   border: 1px solid rgba(199,210,254,0.9);
-  white-space: nowrap;
+  white-space: nowrap;               /* single line */
+  overflow: hidden;                  /* hide overflow */
+  text-overflow: ellipsis;           /* … */
+  vertical-align: middle;
+  font: 12px/1.6 system-ui, sans-serif;
 }
 .tag-chip:hover { background: rgba(255,255,255,1); }
 
+
 .tag-editor { margin-top: .5rem; }
 .tag-form { margin-top: .4rem; display: flex; gap: .5rem; }
 .tag-form input { flex: 1; }
@@ -541,3 +548,28 @@ header {
 }
 .form-button:hover { background-color: #434190; }
 .form-footer-text { margin-top: 1rem; text-align: center; }
+
+/* Fixed-height, 2-line title area so all cards match height */
+.card-title {
+  /* reserve exactly two lines of space for the title */
+  line-height: 1.3;
+  min-height: calc(1.3em * 2);
+  max-height: calc(1.3em * 2);
+  overflow: hidden;
+  margin-bottom: 0.5rem;
+}
+
+/* Multi-line clamp with ellipsis (where supported) */
+.card-title-text {
+  margin: 0;
+  font-size: 1.1rem;
+  line-height: 1.3;
+  display: -webkit-box;
+  -webkit-line-clamp: 2;           /* clamp to 2 lines */
+  -webkit-box-orient: vertical;
+  overflow: hidden;
+
+  /* good fallbacks for very long single words / no spaces */
+  overflow-wrap: anywhere;          /* allows breaking long tokens */
+  word-break: break-word;
+}

app/templates/_pagination.html

@@ -1,4 +1,4 @@
-{% set per_page = request.args.get('per_page') %}
+<!-- /app/templates/_pagination.html -->
 {% if images.pages > 1 %}
 <div class="pagination-container">
   {% set total_pages = images.pages %}
@@ -6,12 +6,25 @@
   {% set window_size = 7 %}
   {% set endpoint = request.endpoint %}
 
+  {# Build a merged param dict: query args + path args #}
+  {% set params = request.args.to_dict(flat=True) %}
+  {% for k, v in request.view_args.items() %}
+    {% set _ = params.update({k: v}) %}
+  {% endfor %}
+
+  {# Helper to make a page URL while preserving all other params #}
+  {% macro page_url(p) -%}
+    {%- set pmap = params.copy() -%}
+    {%- set _ = pmap.update({'page': p}) -%}
+    {{ url_for(endpoint, **pmap) }}
+  {%- endmacro %}
+
   {# First and Prev #}
   {% if images.has_prev %}
     {% if current_page > 1 %}
-      <a class="pagination-link" href="{{ url_for(endpoint, page=1, per_page=per_page, **request.view_args) }}">« First</a>
+      <a class="pagination-link" href="{{ page_url(1) }}">« First</a>
     {% endif %}
-    <a class="pagination-link" href="{{ url_for(endpoint, page=images.prev_num, per_page=per_page, **request.view_args) }}">← Previous</a>
+    <a class="pagination-link" href="{{ page_url(images.prev_num) }}">← Previous</a>
   {% endif %}
 
   {# Page window logic #}
@@ -31,7 +44,7 @@
 
   {# Left ellipsis #}
   {% if start_page > 1 %}
-    <a class="pagination-link" href="{{ url_for(endpoint, page=1, per_page=per_page, **request.view_args) }}">1</a>
+    <a class="pagination-link" href="{{ page_url(1) }}">1</a>
     {% if start_page > 2 %}
       <span class="pagination-ellipsis">…</span>
     {% endif %}
@@ -39,7 +52,7 @@
 
   {# Page numbers #}
   {% for p in range(start_page, end_page + 1) %}
-    <a class="pagination-link {% if p == current_page %}active{% endif %}" href="{{ url_for(endpoint, page=p, per_page=per_page, **request.view_args) }}">{{ p }}</a>
+    <a class="pagination-link {% if p == current_page %}active{% endif %}" href="{{ page_url(p) }}">{{ p }}</a>
   {% endfor %}
 
   {# Right ellipsis #}
@@ -47,14 +60,14 @@
     {% if end_page < total_pages - 1 %}
       <span class="pagination-ellipsis">…</span>
     {% endif %}
-    <a class="pagination-link" href="{{ url_for(endpoint, page=total_pages, per_page=per_page, **request.view_args) }}">{{ total_pages }}</a>
+    <a class="pagination-link" href="{{ page_url(total_pages) }}">{{ total_pages }}</a>
   {% endif %}
 
   {# Next and Last #}
   {% if images.has_next %}
-    <a class="pagination-link" href="{{ url_for(endpoint, page=images.next_num, per_page=per_page, **request.view_args) }}">Next →</a>
+    <a class="pagination-link" href="{{ page_url(images.next_num) }}">Next →</a>
     {% if current_page < total_pages %}
-      <a class="pagination-link" href="{{ url_for(endpoint, page=total_pages, per_page=per_page, **request.view_args) }}">Last »</a>
+      <a class="pagination-link" href="{{ page_url(total_pages) }}">Last »</a>
     {% endif %}
   {% endif %}
 </div>

app/templates/tags_list.html

@@ -28,10 +28,12 @@
   {% for tag in tag_data %}
     <a href="{{ url_for('main.gallery', tag=tag.tag) }}" class="artist-card-link">
       <div class="artist-card">
-        <h3>
+        <div class="card-title" title="{{ tag.name }}">
+          <h3 class="card-title-text">
             {% if tag.kind == 'artist' %}🎨{% elif tag.kind == 'archive' %}🗜️{% else %}# {% endif %}
             {{ tag.name }}
           </h3>
+        </div>
         <div class="artist-preview">
           {% for image in tag.images %}
             <div class="artist-thumb"

app/utils/image_importer.py

@@ -15,13 +15,19 @@ import tempfile
 import time
 import uuid
 
-from PIL import Image
+from PIL import Image, ImageOps
 import exifread
 import imagehash
 
+from pyunpack import Archive
+
 from app import db
 from app.models import ImageRecord, Tag, ArchiveRecord
 
+FFMPEG_THUMB_TIMEOUT = int(os.environ.get("THUMBS_FFMPEG_TIMEOUT", "60"))  # seconds
+Image.MAX_IMAGE_PIXELS = int(os.environ.get("PIL_MAX_IMAGE_PIXELS", "178956970"))
+ARCHIVE_NUM_WIDTH = int(os.environ.get("ARCHIVE_NUM_WIDTH", "4"))
+
 
 # =============================================================================
 # Configuration & Constants
@@ -250,15 +256,13 @@ def process_archive(archive_path, dest_dir, artist, existing_phashes):
     archive_size = os.path.getsize(archive_path)
     archive_hash = calculate_hash(archive_path)
 
-    # Skip if archive already processed
+    # Skip if this exact archive file was already processed (hash-based)
     existing_archive = ArchiveRecord.query.filter_by(hash=archive_hash).first()
     if existing_archive:
         print(f"[SKIP] Archive already imported: {archive_path}")
         return 0
 
-    tag_name = f"archive:{artist}/{os.path.basename(archive_path)}"
     tmpdir = tempfile.mkdtemp(prefix="extract_")
-
     touched_records = []   # ImageRecord objects we imported or matched
     imported_or_tagged = 0
     created_tag = None
@@ -288,6 +292,8 @@ def process_archive(archive_path, dest_dir, artist, existing_phashes):
 
         # Only now: create/attach the archive tag if we actually touched images
         if touched_records:
+            # Incremental name like 'archive:{artist}/0001'
+            tag_name = compute_next_archive_tag_name(artist)
             created_tag = Tag.query.filter_by(name=tag_name).first()
             if not created_tag:
                 created_tag = Tag(name=tag_name, kind='archive')
@@ -301,13 +307,13 @@ def process_archive(archive_path, dest_dir, artist, existing_phashes):
                     newly_tagged += 1
             imported_or_tagged += newly_tagged
 
-        # Record this archive as processed.
+        # Record this archive as processed (tag may be None if no media)
         arch = ArchiveRecord(
             filename=archive_path,
             file_size=archive_size,
             hash=archive_hash,
             artist=artist,
-            tag=created_tag  # None if we had no media
+            tag=created_tag
         )
         db.session.add(arch)
         db.session.commit()
@@ -430,27 +436,89 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash
 # Helpers: Thumbnails, metadata, hashing, similarity
 # =============================================================================
 
-def generate_thumbnail(image_path: str, size: tuple[int, int] = (400, 400), overwrite: bool = False) -> str:
+def generate_thumbnail(
+    image_path,
+    size=(400, 400),
+    overwrite=False,
+    prefer_jpeg=False,      # False = auto-pick PNG for alpha images; True = always JPEG
+    jpeg_bg=(0, 0, 0),      # background used when flattening to JPEG
+):
     """
-    Save thumbnails mirrored under /images/thumbs/<artist>/<unique_filename>.
+    Save thumbnails mirrored under /images/thumbs/<...>.
-    (unique_filename includes the hash suffix, so no collisions)
+    If prefer_jpeg is False (default), transparent images save as PNG, others as JPEG.
+    If prefer_jpeg is True, all thumbs are JPEG with transparency flattened to `jpeg_bg`.
+    Returns the absolute path to the thumbnail.
     """
     images_root = Path("/images").resolve()
     image_path = Path(image_path).resolve()
-    rel_path = image_path.relative_to(images_root)  # raises if not under /images
 
-    thumb_path = images_root / "thumbs" / rel_path
+    try:
-    thumb_path = thumb_path.with_suffix(".jpg") if thumb_path.suffix.lower() not in (".jpg", ".jpeg") else thumb_path
+        rel_path = image_path.relative_to(images_root)
-    thumb_path.parent.mkdir(parents=True, exist_ok=True)
+    except ValueError:
+        raise ValueError(f"{image_path} is not under {images_root}")
 
-    if not overwrite and thumb_path.exists():
+    # Base path under /images/thumbs with same subfolders/filename (we may change the suffix)
-        return str(thumb_path)
+    thumb_base = (images_root / "thumbs" / rel_path).with_suffix("")  # drop original suffix for control
 
+    # Decide output extension/format
     with Image.open(image_path) as img:
+        img = ImageOps.exif_transpose(img)  # honor camera rotation
         img.thumbnail(size)
-        img.save(thumb_path)
 
-    return str(thumb_path)
+        has_alpha = _has_alpha(img)
+
+        if prefer_jpeg:
+            # Force JPEG for consistency
+            out_path = thumb_base.with_suffix(".jpg")
+            out_path.parent.mkdir(parents=True, exist_ok=True)
+            if not overwrite and out_path.exists():
+                return str(out_path)
+            img_rgb = _flatten_to_rgb(img, bg=jpeg_bg)
+            img_rgb.save(out_path, format="JPEG", quality=85, optimize=True, progressive=True)
+            return str(out_path)
+        else:
+            # Auto-pick: PNG for alpha, JPEG otherwise
+            if has_alpha:
+                out_path = thumb_base.with_suffix(".png")
+                out_path.parent.mkdir(parents=True, exist_ok=True)
+                if not overwrite and out_path.exists():
+                    return str(out_path)
+                # Keep transparency if present
+                # Convert palette to RGBA for reliable PNG writing
+                if img.mode == "P":
+                    img = img.convert("RGBA")
+                img.save(out_path, format="PNG", optimize=True)
+                return str(out_path)
+            else:
+                out_path = thumb_base.with_suffix(".jpg")
+                out_path.parent.mkdir(parents=True, exist_ok=True)
+                if not overwrite and out_path.exists():
+                    return str(out_path)
+                if img.mode != "RGB":
+                    img = img.convert("RGB")
+                img.save(out_path, format="JPEG", quality=85, optimize=True, progressive=True)
+                return str(out_path)
+
+def generate_video_thumbnail(video_path: str, output_path: str, time_position: str = "00:00:01") -> str | None:
+    """
+    Extract a single JPG frame from a video using ffmpeg and write it to output_path.
+    Returns output_path on success, or None on failure.
+    """
+    try:
+        os.makedirs(os.path.dirname(output_path), exist_ok=True)
+        cmd = [
+            "ffmpeg", "-y",
+            "-ss", time_position,
+            "-i", str(video_path),
+            "-vframes", "1",
+            "-vf", f"scale={THUMB_SIZE[0]}:-1",
+            str(output_path),
+        ]
+        subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=FFMPEG_THUMB_TIMEOUT)
+        return output_path
+    except subprocess.CalledProcessError as e:
+        print(f"[WARN] Failed to extract video thumbnail for {video_path}: {e}")
+        return None
 
 
 def generate_video_thumbnail_mirrored(video_path: str, time_position: str = "00:00:01") -> str | None:
@@ -458,21 +526,11 @@ def generate_video_thumbnail_mirrored(video_path: str, time_position: str = "00:
     Mirror the stored (hashed) video path under /images/thumbs/... and save as .jpg.
     """
     images_root = Path("/images").resolve()
-    video_path = Path(video_path).resolve()
+    video_path_p = Path(video_path).resolve()
-    rel_path = video_path.relative_to(images_root)  # raises if not under /images
+    rel_path = video_path_p.relative_to(images_root)  # raises if not under /images
     thumb_path = (images_root / "thumbs" / rel_path).with_suffix(".jpg")
-    thumb_path.parent.mkdir(parents=True, exist_ok=True)
+    return generate_video_thumbnail(str(video_path_p), str(thumb_path), time_position=time_position)
 
-    cmd = [
-        "ffmpeg", "-i", str(video_path), "-ss", time_position,
-        "-vframes", "1", "-vf", f"scale={THUMB_SIZE[0]}:-1", str(thumb_path)
-    ]
-    try:
-        subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        return str(thumb_path)
-    except subprocess.CalledProcessError as e:
-        print(f"[WARN] Failed to extract video thumbnail for {video_path}: {e}")
-        return None
 
 
 def calculate_hash(file_path: str) -> str:
@@ -556,6 +614,33 @@ def extract_metadata(file_path: str) -> dict:
 
     return metadata
 
+def _has_alpha(img: Image.Image) -> bool:
+    """Return True if the image has any transparency channel."""
+    if img.mode in ("RGBA", "LA"):
+        return True
+    if img.mode == "P" and "transparency" in img.info:
+        return True
+    return False
+
+def _flatten_to_rgb(img: Image.Image, bg=(0, 0, 0)) -> Image.Image:
+    """
+    Flatten an RGBA/LA/Palette-with-alpha image onto a solid background,
+    returning an RGB image suitable for saving as JPEG.
+    """
+    # Normalize to RGBA so we can use the alpha channel as a mask
+    if img.mode == "P":
+        img = img.convert("RGBA")
+    if img.mode in ("RGBA", "LA"):
+        # Ensure we have an explicit alpha channel as last band
+        if img.mode == "LA":
+            img = img.convert("RGBA")
+        alpha = img.split()[-1]  # A channel
+        bg_img = Image.new("RGB", img.size, bg)
+        bg_img.paste(img, mask=alpha)
+        return bg_img
+    # No alpha, just convert to RGB
+    return img.convert("RGB")
+
 
 # =============================================================================
 # Helpers: Multi-part detection
@@ -597,3 +682,71 @@ def is_first_volume(path: str) -> bool:
         return True
 
     return True
+
+def compute_next_archive_tag_name(artist: str, width: int = ARCHIVE_NUM_WIDTH) -> str:
+    """
+    Return a unique incremental tag name like: 'archive:{artist}/0001', '0002', ...
+    - Looks only at Tag(kind='archive') with names starting 'archive:{artist}/'
+    - Finds max numeric suffix and returns next number (zero-padded).
+    - Ensures no collision even if tags were renamed.
+    """
+    prefix = f"archive:{artist}/"
+    # Pull existing tag names for this artist/prefix
+    rows = (Tag.query
+              .with_entities(Tag.name)
+              .filter(Tag.kind == 'archive',
+                      Tag.name.like(prefix + '%'))
+              .all())
+
+    max_n = 0
+    for (name,) in rows:
+        tail = name[len(prefix):]
+        if tail.isdigit():
+            try:
+                n = int(tail)
+                if n > max_n:
+                    max_n = n
+            except Exception:
+                pass
+
+    # Propose next number and make sure it doesn't already exist
+    n = max_n + 1
+    while True:
+        candidate = f"{prefix}{str(n).zfill(width)}"
+        if not Tag.query.filter_by(name=candidate).first():
+            return candidate
+        n += 1
+
+def compute_next_archive_tag_name(artist: str, width: int = ARCHIVE_NUM_WIDTH) -> str:
+    """
+    Return a unique incremental tag name like: 'archive:{artist}/0001', '0002', ...
+    - Looks only at Tag(kind='archive') with names starting 'archive:{artist}/'
+    - Finds max numeric suffix and returns next number (zero-padded).
+    - Ensures no collision even if tags were renamed.
+    """
+    prefix = f"archive:{artist}/"
+    # Pull existing tag names for this artist/prefix
+    rows = (Tag.query
+              .with_entities(Tag.name)
+              .filter(Tag.kind == 'archive',
+                      Tag.name.like(prefix + '%'))
+              .all())
+
+    max_n = 0
+    for (name,) in rows:
+        tail = name[len(prefix):]
+        if tail.isdigit():
+            try:
+                n = int(tail)
+                if n > max_n:
+                    max_n = n
+            except Exception:
+                pass
+
+    # Propose next number and make sure it doesn't already exist
+    n = max_n + 1
+    while True:
+        candidate = f"{prefix}{str(n).zfill(width)}"
+        if not Tag.query.filter_by(name=candidate).first():
+            return candidate
+        n += 1

image_import_worker.py

@@ -1,7 +1,8 @@
 # image_import_worker.py
-import time
 import os
+import time
 from sqlalchemy.exc import OperationalError
+
 from app import create_app
 from app.utils.image_importer import import_images_task
 
@@ -11,81 +12,279 @@ SOURCE_DIR = "/import"
 DEST_DIR       = "/images"
 CHECK_INTERVAL = 10  # seconds
 
+# Verbosity controls (set via environment)
+THUMBS_VERBOSE     = os.getenv("THUMBS_VERBOSE", "0").lower() in ("1", "true", "yes")
+THUMBS_LOG_EVERY   = int(os.getenv("THUMBS_LOG_EVERY", "50"))   # log every N images if verbose
+THUMBS_BATCH_SIZE  = int(os.getenv("THUMBS_BATCH_SIZE", "500")) # DB chunk size per commit
+
 app = create_app()
 
-from sqlalchemy.exc import OperationalError
-import time
 
 def monitor_and_import():
     with app.app_context():
-        print("[Image Importer] Monitoring for trigger flags...")
+        print("[Image Importer] Monitoring for trigger flags... (CHECK_INTERVAL=%ss)" % CHECK_INTERVAL, flush=True)
 
         while True:
-            if os.path.exists(TRIGGER_FLAG):
+            did_work = False
-                print("[Image Importer] Import flag found. Starting import...")
-                max_retries = 3
-                attempt = 0
-
-                while attempt < max_retries:
-                    try:
-                        result = import_images_task(SOURCE_DIR, DEST_DIR)
-                        print(f"[Image Importer] {result}")
-                        break  # Exit retry loop if successful
-                    except OperationalError as e:
-                        attempt += 1
-                        print(f"[Image Importer] Database error during import (attempt {attempt}): {e}")
-                        if attempt >= max_retries:
-                            print("[Image Importer] Max retries reached. Import failed.")
-                        else:
-                            sleep_time = 2 ** attempt
-                            print(f"[Image Importer] Retrying in {sleep_time} seconds...")
-                            time.sleep(sleep_time)
-                    except Exception as e:
-                        print(f"[Image Importer] Unexpected error during import: {e}")
-                        break
-                    finally:
-                        try:
-                            os.remove(TRIGGER_FLAG)
-                            print("[Image Importer] Import flag cleared.")
-                        except Exception as e:
-                            print(f"[Image Importer] Failed to remove import flag: {e}")
 
+            # ------------------------------------------------------------------
+            # THUMBNAIL REGEN (PRIORITIZED) — chunked loop, detailed progress
+            # ------------------------------------------------------------------
             if os.path.exists(THUMBNAIL_FLAG):
-                print("[Image Importer] Thumbnail regeneration triggered.")
+                print("[Image Importer] Thumbnail regeneration flag detected.", flush=True)
+
+                RUN_FLAG = THUMBNAIL_FLAG + ".run"
+                ERR_FLAG = THUMBNAIL_FLAG + ".err"
+
+                # Claim atomically so only one worker runs
+                try:
+                    os.replace(THUMBNAIL_FLAG, RUN_FLAG)
+                    print(f"[Image Importer] Claimed thumbnail job -> {RUN_FLAG}", flush=True)
+                except Exception as e:
+                    print(f"[Image Importer] Could not claim thumbnail job: {e}", flush=True)
+                    time.sleep(CHECK_INTERVAL)
+                    continue
+
                 max_retries = 3
                 attempt = 0
 
                 while attempt < max_retries:
                     try:
+                        from pathlib import Path
+                        from sqlalchemy import text
+                        from app import db
                         from app.models import ImageRecord
-                        from app.utils.image_importer import generate_thumbnail
+                        from app.utils.image_importer import generate_thumbnail, generate_video_thumbnail
 
-                        for img in ImageRecord.query.all():
+                        # Early diagnostics
+                        db.session.execute(text("SELECT 1"))
+                        total = db.session.query(ImageRecord).count()
+                        print(f"[Thumbs] Starting regen; ImageRecord count = {total}", flush=True)
+
+                        images_root = Path("/images").resolve()
+                        batch_size = THUMBS_BATCH_SIZE
+
+                        processed = 0      # total processed rows
+                        writes = 0         # files written/overwritten
+                        path_updates = 0   # DB thumb_path changes
+                        failures = 0
+                        last_id = 0
+                        t_start = time.time()
+
+                        while True:
+                            rows = (
+                                ImageRecord.query
+                                .filter(ImageRecord.id > last_id)
+                                .order_by(ImageRecord.id.asc())
+                                .limit(batch_size)
+                                .all()
+                            )
+                            if not rows:
+                                break
+
+                            batch_start_id = rows[0].id
+                            batch_end_id   = rows[-1].id
+                            batch_processed = 0
+                            batch_writes    = 0
+                            batch_updates   = 0
+                            batch_failures  = 0
+                            bt0 = time.time()
+
+                            for img in rows:
                                 try:
-                                generate_thumbnail(img.filepath, overwrite=True)
+                                    processed += 1
-                                print(f"[Thumbnail] Regenerated for {img.filename}")
+                                    batch_processed += 1
+
+                                    ext = (img.format or "").lower()
+                                    is_video = ext in ("mp4", "mov") or img.filepath.lower().endswith((".mp4", ".mov"))
+
+                                    # Determine output path for videos if needed
+                                    if is_video:
+                                        out_path = img.thumb_path
+                                        if not out_path:
+                                            try:
+                                                rel = Path(img.filepath).resolve().relative_to(images_root)
+                                            except Exception:
+                                                batch_failures += 1
+                                                failures += 1
+                                                if THUMBS_VERBOSE:
+                                                    print(f"[Thumbs] SKIP (not under /images): id={img.id} {img.filepath}", flush=True)
+                                                continue
+                                            out_path = str((images_root / "thumbs" / rel).with_suffix(".jpg"))
+
+                                        new_path = generate_video_thumbnail(img.filepath, out_path)
+                                    else:
+                                        new_path = generate_thumbnail(img.filepath, overwrite=True)
+
+                                    # Count file writes: our helpers return the path on success
+                                    if new_path:
+                                        writes += 1
+                                        batch_writes += 1
+
+                                    # Update DB if path changed
+                                    if new_path and new_path != img.thumb_path:
+                                        img.thumb_path = new_path
+                                        path_updates += 1
+                                        batch_updates += 1
+
+                                    # Optional per-file log
+                                    if THUMBS_VERBOSE and (processed <= 10 or processed % THUMBS_LOG_EVERY == 0):
+                                        kind = "video" if is_video else "image"
+                                        print(f"[Thumbs] #{processed} ({kind}) id={img.id} name='{img.filename}' -> {new_path or 'None'}", flush=True)
+
                                 except Exception as e:
-                                print(f"[Thumbnail] Failed for {img.filename}: {e}")
+                                    failures += 1
-                        break  # Success
+                                    batch_failures += 1
+                                    print(f"[Thumbs] Failed for id={img.id} name='{img.filename}': {e}", flush=True)
+
+                            # Commit AFTER each chunk; safe for Postgres (no server-side cursor)
+                            db.session.commit()
+                            last_id = rows[-1].id
+                            db.session.expunge_all()  # keep memory stable
+
+                            # Batch progress
+                            bt = time.time() - bt0
+                            rate = batch_processed / bt if bt > 0 else 0
+                            print(
+                                f"[Thumbs] Batch {batch_start_id}-{batch_end_id}: "
+                                f"processed={batch_processed}, writes={batch_writes}, "
+                                f"db_updates={batch_updates}, failures={batch_failures} "
+                                f"({rate:.1f} items/s) — totals: processed={processed}, writes={writes}, "
+                                f"db_updates={path_updates}, failures={failures} (last_id={last_id})",
+                                flush=True
+                            )
+
+                        # Final summary
+                        elapsed = time.time() - t_start
+                        overall_rate = processed / elapsed if elapsed > 0 else 0
+                        print(
+                            f"[Image Importer] Thumbnail regen COMPLETE in {elapsed:.1f}s — "
+                            f"processed={processed}, writes={writes}, db_updates={path_updates}, failures={failures} "
+                            f"({overall_rate:.1f} items/s).",
+                            flush=True
+                        )
+
+                        # Success — remove the .run flag and exit
+                        try:
+                            os.remove(RUN_FLAG)
+                            print("[Image Importer] Thumbnail .run flag cleared.", flush=True)
+                        except Exception as e:
+                            print(f"[Image Importer] Could not remove .run flag: {e}", flush=True)
+                        break  # success
+
                     except OperationalError as e:
                         attempt += 1
-                        print(f"[Image Importer] Database error during thumbnail regen (attempt {attempt}): {e}")
+                        print(f"[Image Importer] DB error during thumbnail regen (attempt {attempt}): {e}", flush=True)
+                        # Recover the session for the next attempt
+                        try:
+                            from app import db
+                            db.session.rollback()
+                        except Exception:
+                            pass
+                        time.sleep(2 ** attempt)
                         if attempt >= max_retries:
-                            print("[Image Importer] Max retries reached. Thumbnail regeneration failed.")
+                            try:
+                                os.replace(RUN_FLAG, ERR_FLAG)
+                                print(f"[Image Importer] Moved thumbnail flag to {ERR_FLAG}", flush=True)
+                            except Exception as re:
+                                print(f"[Image Importer] Could not mark error flag: {re}", flush=True)
+                    except Exception as e:
+                        print(f"[Image Importer] Unexpected error during thumbnail regen: {e}", flush=True)
+                        try:
+                            os.replace(RUN_FLAG, ERR_FLAG)
+                            print(f"[Image Importer] Moved thumbnail flag to {ERR_FLAG}", flush=True)
+                        except Exception as re:
+                            print(f"[Image Importer] Could not mark error flag: {re}", flush=True)
+                        break
+
+                did_work = True  # handled a job (success or fail)
+
+            # ------------------------------------------------------------------
+            # IMPORT TRIGGER  (robust: claim -> run -> clear/err; rollback on fail)
+            # ------------------------------------------------------------------
+            if os.path.exists(TRIGGER_FLAG):
+                print("[Image Importer] Import flag found. Starting import...", flush=True)
+
+                RUN_FLAG = TRIGGER_FLAG + ".run"
+                ERR_FLAG = TRIGGER_FLAG + ".err"
+
+                # Claim atomically so only one worker takes it
+                try:
+                    os.replace(TRIGGER_FLAG, RUN_FLAG)
+                    print(f"[Image Importer] Claimed import job -> {RUN_FLAG}", flush=True)
+                except Exception as e:
+                    print(f"[Image Importer] Could not claim import job: {e}", flush=True)
+                    time.sleep(CHECK_INTERVAL)
+                    continue
+
+                max_retries = 3
+                attempt = 0
+
+                while attempt < max_retries:
+                    try:
+                        from sqlalchemy import text
+                        from app import db
+
+                        # Fast DB ping to catch stale pool conns
+                        db.session.execute(text("SELECT 1"))
+
+                        # Run the import
+                        result = import_images_task(SOURCE_DIR, DEST_DIR)
+                        print(f"[Image Importer] {result}", flush=True)
+
+                        # Success — remove the .run flag and exit
+                        try:
+                            os.remove(RUN_FLAG)
+                            print("[Image Importer] Import .run flag cleared.", flush=True)
+                        except Exception as e:
+                            print(f"[Image Importer] Could not remove import .run flag: {e}", flush=True)
+                        break  # success
+
+                    except OperationalError as e:
+                        attempt += 1
+                        print(f"[Image Importer] DB error during import (attempt {attempt}): {e}", flush=True)
+                        # IMPORTANT: rollback failed transaction & drop pool to force clean conns
+                        try:
+                            from app import db
+                            db.session.rollback()
+                            db.engine.dispose()
+                        except Exception:
+                            pass
+                        if attempt >= max_retries:
+                            try:
+                                os.replace(RUN_FLAG, ERR_FLAG)
+                                print(f"[Image Importer] Moved import flag to {ERR_FLAG}", flush=True)
+                            except Exception as re:
+                                print(f"[Image Importer] Could not mark import error flag: {re}", flush=True)
                         else:
                             sleep_time = 2 ** attempt
-                            print(f"[Image Importer] Retrying in {sleep_time} seconds...")
+                            print(f"[Image Importer] Retrying in {sleep_time} seconds...", flush=True)
                             time.sleep(sleep_time)
+
                     except Exception as e:
-                        print(f"[Image Importer] Unexpected error during thumbnail regen: {e}")
+                        print(f"[Image Importer] Unexpected error during import: {e}", flush=True)
-                        break
+                        # Make sure the session is usable next time
-                    finally:
                         try:
-                            os.remove(THUMBNAIL_FLAG)
+                            from app import db
-                            print("[Image Importer] Thumbnail flag cleared.")
+                            db.session.rollback()
-                        except Exception as e:
+                            db.engine.dispose()
-                            print(f"[Image Importer] Failed to remove thumbnail flag: {e}")
+                        except Exception:
+                            pass
+                        try:
+                            os.replace(RUN_FLAG, ERR_FLAG)
+                            print(f"[Image Importer] Moved import flag to {ERR_FLAG}", flush=True)
+                        except Exception as re:
+                            print(f"[Image Importer] Could not mark import error flag: {re}", flush=True)
+                        break
+
+                did_work = True
+
+            # ------------------------------------------------------------------
+            # IDLE SLEEP
+            # ------------------------------------------------------------------
+            if not did_work:
+                time.sleep(CHECK_INTERVAL)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     monitor_and_import()