str:
+def generate_thumbnail(
+ image_path,
+ size=(400, 400),
+ overwrite=False,
+ prefer_jpeg=False, # False = auto-pick PNG for alpha images; True = always JPEG
+ jpeg_bg=(0, 0, 0), # background used when flattening to JPEG
+):
"""
- Save thumbnails mirrored under /images/thumbs/
/.
- (unique_filename includes the hash suffix, so no collisions)
+ Save thumbnails mirrored under /images/thumbs/<...>.
+ If prefer_jpeg is False (default), transparent images save as PNG, others as JPEG.
+ If prefer_jpeg is True, all thumbs are JPEG with transparency flattened to `jpeg_bg`.
+ Returns the absolute path to the thumbnail.
"""
images_root = Path("/images").resolve()
image_path = Path(image_path).resolve()
- rel_path = image_path.relative_to(images_root) # raises if not under /images
- thumb_path = images_root / "thumbs" / rel_path
- thumb_path = thumb_path.with_suffix(".jpg") if thumb_path.suffix.lower() not in (".jpg", ".jpeg") else thumb_path
- thumb_path.parent.mkdir(parents=True, exist_ok=True)
+ try:
+ rel_path = image_path.relative_to(images_root)
+ except ValueError:
+ raise ValueError(f"{image_path} is not under {images_root}")
- if not overwrite and thumb_path.exists():
- return str(thumb_path)
+ # Base path under /images/thumbs with same subfolders/filename (we may change the suffix)
+ thumb_base = (images_root / "thumbs" / rel_path).with_suffix("") # drop original suffix for control
+ # Decide output extension/format
with Image.open(image_path) as img:
+ img = ImageOps.exif_transpose(img) # honor camera rotation
img.thumbnail(size)
- img.save(thumb_path)
- return str(thumb_path)
+ has_alpha = _has_alpha(img)
+
+ if prefer_jpeg:
+ # Force JPEG for consistency
+ out_path = thumb_base.with_suffix(".jpg")
+ out_path.parent.mkdir(parents=True, exist_ok=True)
+ if not overwrite and out_path.exists():
+ return str(out_path)
+ img_rgb = _flatten_to_rgb(img, bg=jpeg_bg)
+ img_rgb.save(out_path, format="JPEG", quality=85, optimize=True, progressive=True)
+ return str(out_path)
+ else:
+ # Auto-pick: PNG for alpha, JPEG otherwise
+ if has_alpha:
+ out_path = thumb_base.with_suffix(".png")
+ out_path.parent.mkdir(parents=True, exist_ok=True)
+ if not overwrite and out_path.exists():
+ return str(out_path)
+ # Keep transparency if present
+ # Convert palette to RGBA for reliable PNG writing
+ if img.mode == "P":
+ img = img.convert("RGBA")
+ img.save(out_path, format="PNG", optimize=True)
+ return str(out_path)
+ else:
+ out_path = thumb_base.with_suffix(".jpg")
+ out_path.parent.mkdir(parents=True, exist_ok=True)
+ if not overwrite and out_path.exists():
+ return str(out_path)
+ if img.mode != "RGB":
+ img = img.convert("RGB")
+ img.save(out_path, format="JPEG", quality=85, optimize=True, progressive=True)
+ return str(out_path)
+
+def generate_video_thumbnail(video_path: str, output_path: str, time_position: str = "00:00:01") -> str | None:
+ """
+ Extract a single JPG frame from a video using ffmpeg and write it to output_path.
+ Returns output_path on success, or None on failure.
+ """
+ try:
+ os.makedirs(os.path.dirname(output_path), exist_ok=True)
+ cmd = [
+ "ffmpeg", "-y",
+ "-ss", time_position,
+ "-i", str(video_path),
+ "-vframes", "1",
+ "-vf", f"scale={THUMB_SIZE[0]}:-1",
+ str(output_path),
+ ]
+ subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=FFMPEG_THUMB_TIMEOUT)
+ return output_path
+ except subprocess.CalledProcessError as e:
+ print(f"[WARN] Failed to extract video thumbnail for {video_path}: {e}")
+ return None
def generate_video_thumbnail_mirrored(video_path: str, time_position: str = "00:00:01") -> str | None:
@@ -458,21 +526,11 @@ def generate_video_thumbnail_mirrored(video_path: str, time_position: str = "00:
Mirror the stored (hashed) video path under /images/thumbs/... and save as .jpg.
"""
images_root = Path("/images").resolve()
- video_path = Path(video_path).resolve()
- rel_path = video_path.relative_to(images_root) # raises if not under /images
+ video_path_p = Path(video_path).resolve()
+ rel_path = video_path_p.relative_to(images_root) # raises if not under /images
thumb_path = (images_root / "thumbs" / rel_path).with_suffix(".jpg")
- thumb_path.parent.mkdir(parents=True, exist_ok=True)
+ return generate_video_thumbnail(str(video_path_p), str(thumb_path), time_position=time_position)
- cmd = [
- "ffmpeg", "-i", str(video_path), "-ss", time_position,
- "-vframes", "1", "-vf", f"scale={THUMB_SIZE[0]}:-1", str(thumb_path)
- ]
- try:
- subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
- return str(thumb_path)
- except subprocess.CalledProcessError as e:
- print(f"[WARN] Failed to extract video thumbnail for {video_path}: {e}")
- return None
def calculate_hash(file_path: str) -> str:
@@ -556,6 +614,33 @@ def extract_metadata(file_path: str) -> dict:
return metadata
+def _has_alpha(img: Image.Image) -> bool:
+ """Return True if the image has any transparency channel."""
+ if img.mode in ("RGBA", "LA"):
+ return True
+ if img.mode == "P" and "transparency" in img.info:
+ return True
+ return False
+
+def _flatten_to_rgb(img: Image.Image, bg=(0, 0, 0)) -> Image.Image:
+ """
+ Flatten an RGBA/LA/Palette-with-alpha image onto a solid background,
+ returning an RGB image suitable for saving as JPEG.
+ """
+ # Normalize to RGBA so we can use the alpha channel as a mask
+ if img.mode == "P":
+ img = img.convert("RGBA")
+ if img.mode in ("RGBA", "LA"):
+ # Ensure we have an explicit alpha channel as last band
+ if img.mode == "LA":
+ img = img.convert("RGBA")
+ alpha = img.split()[-1] # A channel
+ bg_img = Image.new("RGB", img.size, bg)
+ bg_img.paste(img, mask=alpha)
+ return bg_img
+ # No alpha, just convert to RGB
+ return img.convert("RGB")
+
# =============================================================================
# Helpers: Multi-part detection
@@ -596,4 +681,72 @@ def is_first_volume(path: str) -> bool:
if name.endswith((".zip", ".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz")):
return True
- return True
\ No newline at end of file
+ return True
+
+def compute_next_archive_tag_name(artist: str, width: int = ARCHIVE_NUM_WIDTH) -> str:
+ """
+ Return a unique incremental tag name like: 'archive:{artist}/0001', '0002', ...
+ - Looks only at Tag(kind='archive') with names starting 'archive:{artist}/'
+ - Finds max numeric suffix and returns next number (zero-padded).
+ - Ensures no collision even if tags were renamed.
+ """
+ prefix = f"archive:{artist}/"
+ # Pull existing tag names for this artist/prefix
+ rows = (Tag.query
+ .with_entities(Tag.name)
+ .filter(Tag.kind == 'archive',
+ Tag.name.like(prefix + '%'))
+ .all())
+
+ max_n = 0
+ for (name,) in rows:
+ tail = name[len(prefix):]
+ if tail.isdigit():
+ try:
+ n = int(tail)
+ if n > max_n:
+ max_n = n
+ except Exception:
+ pass
+
+ # Propose next number and make sure it doesn't already exist
+ n = max_n + 1
+ while True:
+ candidate = f"{prefix}{str(n).zfill(width)}"
+ if not Tag.query.filter_by(name=candidate).first():
+ return candidate
+ n += 1
+
+def compute_next_archive_tag_name(artist: str, width: int = ARCHIVE_NUM_WIDTH) -> str:
+ """
+ Return a unique incremental tag name like: 'archive:{artist}/0001', '0002', ...
+ - Looks only at Tag(kind='archive') with names starting 'archive:{artist}/'
+ - Finds max numeric suffix and returns next number (zero-padded).
+ - Ensures no collision even if tags were renamed.
+ """
+ prefix = f"archive:{artist}/"
+ # Pull existing tag names for this artist/prefix
+ rows = (Tag.query
+ .with_entities(Tag.name)
+ .filter(Tag.kind == 'archive',
+ Tag.name.like(prefix + '%'))
+ .all())
+
+ max_n = 0
+ for (name,) in rows:
+ tail = name[len(prefix):]
+ if tail.isdigit():
+ try:
+ n = int(tail)
+ if n > max_n:
+ max_n = n
+ except Exception:
+ pass
+
+ # Propose next number and make sure it doesn't already exist
+ n = max_n + 1
+ while True:
+ candidate = f"{prefix}{str(n).zfill(width)}"
+ if not Tag.query.filter_by(name=candidate).first():
+ return candidate
+ n += 1
\ No newline at end of file
diff --git a/image_import_worker.py b/image_import_worker.py
index e9cd167..61ded1c 100644
--- a/image_import_worker.py
+++ b/image_import_worker.py
@@ -1,91 +1,290 @@
# image_import_worker.py
-import time
import os
+import time
from sqlalchemy.exc import OperationalError
+
from app import create_app
from app.utils.image_importer import import_images_task
THUMBNAIL_FLAG = "/import/thumbnail.flag"
-TRIGGER_FLAG = "/import/trigger.flag"
-SOURCE_DIR = "/import"
-DEST_DIR = "/images"
+TRIGGER_FLAG = "/import/trigger.flag"
+SOURCE_DIR = "/import"
+DEST_DIR = "/images"
CHECK_INTERVAL = 10 # seconds
+# Verbosity controls (set via environment)
+THUMBS_VERBOSE = os.getenv("THUMBS_VERBOSE", "0").lower() in ("1", "true", "yes")
+THUMBS_LOG_EVERY = int(os.getenv("THUMBS_LOG_EVERY", "50")) # log every N images if verbose
+THUMBS_BATCH_SIZE = int(os.getenv("THUMBS_BATCH_SIZE", "500")) # DB chunk size per commit
+
app = create_app()
-from sqlalchemy.exc import OperationalError
-import time
def monitor_and_import():
with app.app_context():
- print("[Image Importer] Monitoring for trigger flags...")
+ print("[Image Importer] Monitoring for trigger flags... (CHECK_INTERVAL=%ss)" % CHECK_INTERVAL, flush=True)
while True:
- if os.path.exists(TRIGGER_FLAG):
- print("[Image Importer] Import flag found. Starting import...")
- max_retries = 3
- attempt = 0
-
- while attempt < max_retries:
- try:
- result = import_images_task(SOURCE_DIR, DEST_DIR)
- print(f"[Image Importer] {result}")
- break # Exit retry loop if successful
- except OperationalError as e:
- attempt += 1
- print(f"[Image Importer] Database error during import (attempt {attempt}): {e}")
- if attempt >= max_retries:
- print("[Image Importer] Max retries reached. Import failed.")
- else:
- sleep_time = 2 ** attempt
- print(f"[Image Importer] Retrying in {sleep_time} seconds...")
- time.sleep(sleep_time)
- except Exception as e:
- print(f"[Image Importer] Unexpected error during import: {e}")
- break
- finally:
- try:
- os.remove(TRIGGER_FLAG)
- print("[Image Importer] Import flag cleared.")
- except Exception as e:
- print(f"[Image Importer] Failed to remove import flag: {e}")
+ did_work = False
+ # ------------------------------------------------------------------
+ # THUMBNAIL REGEN (PRIORITIZED) — chunked loop, detailed progress
+ # ------------------------------------------------------------------
if os.path.exists(THUMBNAIL_FLAG):
- print("[Image Importer] Thumbnail regeneration triggered.")
+ print("[Image Importer] Thumbnail regeneration flag detected.", flush=True)
+
+ RUN_FLAG = THUMBNAIL_FLAG + ".run"
+ ERR_FLAG = THUMBNAIL_FLAG + ".err"
+
+ # Claim atomically so only one worker runs
+ try:
+ os.replace(THUMBNAIL_FLAG, RUN_FLAG)
+ print(f"[Image Importer] Claimed thumbnail job -> {RUN_FLAG}", flush=True)
+ except Exception as e:
+ print(f"[Image Importer] Could not claim thumbnail job: {e}", flush=True)
+ time.sleep(CHECK_INTERVAL)
+ continue
+
max_retries = 3
attempt = 0
while attempt < max_retries:
try:
+ from pathlib import Path
+ from sqlalchemy import text
+ from app import db
from app.models import ImageRecord
- from app.utils.image_importer import generate_thumbnail
+ from app.utils.image_importer import generate_thumbnail, generate_video_thumbnail
+
+ # Early diagnostics
+ db.session.execute(text("SELECT 1"))
+ total = db.session.query(ImageRecord).count()
+ print(f"[Thumbs] Starting regen; ImageRecord count = {total}", flush=True)
+
+ images_root = Path("/images").resolve()
+ batch_size = THUMBS_BATCH_SIZE
+
+ processed = 0 # total processed rows
+ writes = 0 # files written/overwritten
+ path_updates = 0 # DB thumb_path changes
+ failures = 0
+ last_id = 0
+ t_start = time.time()
+
+ while True:
+ rows = (
+ ImageRecord.query
+ .filter(ImageRecord.id > last_id)
+ .order_by(ImageRecord.id.asc())
+ .limit(batch_size)
+ .all()
+ )
+ if not rows:
+ break
+
+ batch_start_id = rows[0].id
+ batch_end_id = rows[-1].id
+ batch_processed = 0
+ batch_writes = 0
+ batch_updates = 0
+ batch_failures = 0
+ bt0 = time.time()
+
+ for img in rows:
+ try:
+ processed += 1
+ batch_processed += 1
+
+ ext = (img.format or "").lower()
+ is_video = ext in ("mp4", "mov") or img.filepath.lower().endswith((".mp4", ".mov"))
+
+ # Determine output path for videos if needed
+ if is_video:
+ out_path = img.thumb_path
+ if not out_path:
+ try:
+ rel = Path(img.filepath).resolve().relative_to(images_root)
+ except Exception:
+ batch_failures += 1
+ failures += 1
+ if THUMBS_VERBOSE:
+ print(f"[Thumbs] SKIP (not under /images): id={img.id} {img.filepath}", flush=True)
+ continue
+ out_path = str((images_root / "thumbs" / rel).with_suffix(".jpg"))
+
+ new_path = generate_video_thumbnail(img.filepath, out_path)
+ else:
+ new_path = generate_thumbnail(img.filepath, overwrite=True)
+
+ # Count file writes: our helpers return the path on success
+ if new_path:
+ writes += 1
+ batch_writes += 1
+
+ # Update DB if path changed
+ if new_path and new_path != img.thumb_path:
+ img.thumb_path = new_path
+ path_updates += 1
+ batch_updates += 1
+
+ # Optional per-file log
+ if THUMBS_VERBOSE and (processed <= 10 or processed % THUMBS_LOG_EVERY == 0):
+ kind = "video" if is_video else "image"
+ print(f"[Thumbs] #{processed} ({kind}) id={img.id} name='{img.filename}' -> {new_path or 'None'}", flush=True)
+
+ except Exception as e:
+ failures += 1
+ batch_failures += 1
+ print(f"[Thumbs] Failed for id={img.id} name='{img.filename}': {e}", flush=True)
+
+ # Commit AFTER each chunk; safe for Postgres (no server-side cursor)
+ db.session.commit()
+ last_id = rows[-1].id
+ db.session.expunge_all() # keep memory stable
+
+ # Batch progress
+ bt = time.time() - bt0
+ rate = batch_processed / bt if bt > 0 else 0
+ print(
+ f"[Thumbs] Batch {batch_start_id}-{batch_end_id}: "
+ f"processed={batch_processed}, writes={batch_writes}, "
+ f"db_updates={batch_updates}, failures={batch_failures} "
+ f"({rate:.1f} items/s) — totals: processed={processed}, writes={writes}, "
+ f"db_updates={path_updates}, failures={failures} (last_id={last_id})",
+ flush=True
+ )
+
+ # Final summary
+ elapsed = time.time() - t_start
+ overall_rate = processed / elapsed if elapsed > 0 else 0
+ print(
+ f"[Image Importer] Thumbnail regen COMPLETE in {elapsed:.1f}s — "
+ f"processed={processed}, writes={writes}, db_updates={path_updates}, failures={failures} "
+ f"({overall_rate:.1f} items/s).",
+ flush=True
+ )
+
+ # Success — remove the .run flag and exit
+ try:
+ os.remove(RUN_FLAG)
+ print("[Image Importer] Thumbnail .run flag cleared.", flush=True)
+ except Exception as e:
+ print(f"[Image Importer] Could not remove .run flag: {e}", flush=True)
+ break # success
- for img in ImageRecord.query.all():
- try:
- generate_thumbnail(img.filepath, overwrite=True)
- print(f"[Thumbnail] Regenerated for {img.filename}")
- except Exception as e:
- print(f"[Thumbnail] Failed for {img.filename}: {e}")
- break # Success
except OperationalError as e:
attempt += 1
- print(f"[Image Importer] Database error during thumbnail regen (attempt {attempt}): {e}")
+ print(f"[Image Importer] DB error during thumbnail regen (attempt {attempt}): {e}", flush=True)
+ # Recover the session for the next attempt
+ try:
+ from app import db
+ db.session.rollback()
+ except Exception:
+ pass
+ time.sleep(2 ** attempt)
if attempt >= max_retries:
- print("[Image Importer] Max retries reached. Thumbnail regeneration failed.")
+ try:
+ os.replace(RUN_FLAG, ERR_FLAG)
+ print(f"[Image Importer] Moved thumbnail flag to {ERR_FLAG}", flush=True)
+ except Exception as re:
+ print(f"[Image Importer] Could not mark error flag: {re}", flush=True)
+ except Exception as e:
+ print(f"[Image Importer] Unexpected error during thumbnail regen: {e}", flush=True)
+ try:
+ os.replace(RUN_FLAG, ERR_FLAG)
+ print(f"[Image Importer] Moved thumbnail flag to {ERR_FLAG}", flush=True)
+ except Exception as re:
+ print(f"[Image Importer] Could not mark error flag: {re}", flush=True)
+ break
+
+ did_work = True # handled a job (success or fail)
+
+ # ------------------------------------------------------------------
+ # IMPORT TRIGGER (robust: claim -> run -> clear/err; rollback on fail)
+ # ------------------------------------------------------------------
+ if os.path.exists(TRIGGER_FLAG):
+ print("[Image Importer] Import flag found. Starting import...", flush=True)
+
+ RUN_FLAG = TRIGGER_FLAG + ".run"
+ ERR_FLAG = TRIGGER_FLAG + ".err"
+
+ # Claim atomically so only one worker takes it
+ try:
+ os.replace(TRIGGER_FLAG, RUN_FLAG)
+ print(f"[Image Importer] Claimed import job -> {RUN_FLAG}", flush=True)
+ except Exception as e:
+ print(f"[Image Importer] Could not claim import job: {e}", flush=True)
+ time.sleep(CHECK_INTERVAL)
+ continue
+
+ max_retries = 3
+ attempt = 0
+
+ while attempt < max_retries:
+ try:
+ from sqlalchemy import text
+ from app import db
+
+ # Fast DB ping to catch stale pool conns
+ db.session.execute(text("SELECT 1"))
+
+ # Run the import
+ result = import_images_task(SOURCE_DIR, DEST_DIR)
+ print(f"[Image Importer] {result}", flush=True)
+
+ # Success — remove the .run flag and exit
+ try:
+ os.remove(RUN_FLAG)
+ print("[Image Importer] Import .run flag cleared.", flush=True)
+ except Exception as e:
+ print(f"[Image Importer] Could not remove import .run flag: {e}", flush=True)
+ break # success
+
+ except OperationalError as e:
+ attempt += 1
+ print(f"[Image Importer] DB error during import (attempt {attempt}): {e}", flush=True)
+ # IMPORTANT: rollback failed transaction & drop pool to force clean conns
+ try:
+ from app import db
+ db.session.rollback()
+ db.engine.dispose()
+ except Exception:
+ pass
+ if attempt >= max_retries:
+ try:
+ os.replace(RUN_FLAG, ERR_FLAG)
+ print(f"[Image Importer] Moved import flag to {ERR_FLAG}", flush=True)
+ except Exception as re:
+ print(f"[Image Importer] Could not mark import error flag: {re}", flush=True)
else:
sleep_time = 2 ** attempt
- print(f"[Image Importer] Retrying in {sleep_time} seconds...")
+ print(f"[Image Importer] Retrying in {sleep_time} seconds...", flush=True)
time.sleep(sleep_time)
+
except Exception as e:
- print(f"[Image Importer] Unexpected error during thumbnail regen: {e}")
- break
- finally:
+ print(f"[Image Importer] Unexpected error during import: {e}", flush=True)
+ # Make sure the session is usable next time
try:
- os.remove(THUMBNAIL_FLAG)
- print("[Image Importer] Thumbnail flag cleared.")
- except Exception as e:
- print(f"[Image Importer] Failed to remove thumbnail flag: {e}")
+ from app import db
+ db.session.rollback()
+ db.engine.dispose()
+ except Exception:
+ pass
+ try:
+ os.replace(RUN_FLAG, ERR_FLAG)
+ print(f"[Image Importer] Moved import flag to {ERR_FLAG}", flush=True)
+ except Exception as re:
+ print(f"[Image Importer] Could not mark import error flag: {re}", flush=True)
+ break
+
+ did_work = True
+
+ # ------------------------------------------------------------------
+ # IDLE SLEEP
+ # ------------------------------------------------------------------
+ if not did_work:
+ time.sleep(CHECK_INTERVAL)
-if __name__ == '__main__':
+if __name__ == "__main__":
monitor_and_import()