bvandeusen/imagerepo
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed pagination issues and thumbnail generation issues

Browse Source
This commit is contained in:
Bryan Van Deusen
2025-08-16 10:25:16 -04:00
parent 3ff34ec9c2
commit a77724dccc
5 changed files with 502 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/static/style.css
+37 -5
View File
@@ -329,17 +329,24 @@ header {
gap: .25rem;
}
/* Tag chips (truncate long names nicely) */
.tag-chip {
font: 12px/1.6 system-ui, sans-serif;
display: inline-block; /* required for ellipsis */
max-width: clamp(8rem, 22vw, 14rem); /* responsive cap */
padding: 2px 8px;
border-radius: 999px;
background: rgba(255, 255, 255, 0.92);
background: rgba(255,255,255,0.92);
color: #334155;
text-decoration: none;
border: 1px solid rgba(199, 210, 254, 0.9);
white-space: nowrap;
border: 1px solid rgba(199,210,254,0.9);
white-space: nowrap; /* single line */
overflow: hidden; /* hide overflow */
text-overflow: ellipsis; /* … */
vertical-align: middle;
font: 12px/1.6 system-ui, sans-serif;
}
.tag-chip:hover { background: rgba(255, 255, 255, 1); }
.tag-chip:hover { background: rgba(255,255,255,1); }
.tag-editor { margin-top: .5rem; }
.tag-form { margin-top: .4rem; display: flex; gap: .5rem; }
@@ -541,3 +548,28 @@ header {
}
.form-button:hover { background-color: #434190; }
.form-footer-text { margin-top: 1rem; text-align: center; }
/* Fixed-height, 2-line title area so all cards match height */
.card-title {
/* reserve exactly two lines of space for the title */
line-height: 1.3;
min-height: calc(1.3em * 2);
max-height: calc(1.3em * 2);
overflow: hidden;
margin-bottom: 0.5rem;
}
/* Multi-line clamp with ellipsis (where supported) */
.card-title-text {
margin: 0;
font-size: 1.1rem;
line-height: 1.3;
display: -webkit-box;
-webkit-line-clamp: 2; /* clamp to 2 lines */
-webkit-box-orient: vertical;
overflow: hidden;
/* good fallbacks for very long single words / no spaces */
overflow-wrap: anywhere; /* allows breaking long tokens */
word-break: break-word;
}
app/templates/_pagination.html
+21 -8
View File
@@ -1,4 +1,4 @@
{% set per_page = request.args.get('per_page') %}
<!-- /app/templates/_pagination.html -->
{% if images.pages > 1 %}
<div class="pagination-container">
{% set total_pages = images.pages %}
@@ -6,12 +6,25 @@
{% set window_size = 7 %}
{% set endpoint = request.endpoint %}
{# Build a merged param dict: query args + path args #}
{% set params = request.args.to_dict(flat=True) %}
{% for k, v in request.view_args.items() %}
{% set _ = params.update({k: v}) %}
{% endfor %}
{# Helper to make a page URL while preserving all other params #}
{% macro page_url(p) -%}
{%- set pmap = params.copy() -%}
{%- set _ = pmap.update({'page': p}) -%}
{{ url_for(endpoint, **pmap) }}
{%- endmacro %}
{# First and Prev #}
{% if images.has_prev %}
{% if current_page > 1 %}
<a class="pagination-link" href="{{ url_for(endpoint, page=1, per_page=per_page, **request.view_args) }}">« First</a>
<a class="pagination-link" href="{{ page_url(1) }}">« First</a>
{% endif %}
<a class="pagination-link" href="{{ url_for(endpoint, page=images.prev_num, per_page=per_page, **request.view_args) }}">← Previous</a>
<a class="pagination-link" href="{{ page_url(images.prev_num) }}">← Previous</a>
{% endif %}
{# Page window logic #}
@@ -31,7 +44,7 @@
{# Left ellipsis #}
{% if start_page > 1 %}
<a class="pagination-link" href="{{ url_for(endpoint, page=1, per_page=per_page, **request.view_args) }}">1</a>
<a class="pagination-link" href="{{ page_url(1) }}">1</a>
{% if start_page > 2 %}
<span class="pagination-ellipsis">…</span>
{% endif %}
@@ -39,7 +52,7 @@
{# Page numbers #}
{% for p in range(start_page, end_page + 1) %}
<a class="pagination-link {% if p == current_page %}active{% endif %}" href="{{ url_for(endpoint, page=p, per_page=per_page, **request.view_args) }}">{{ p }}</a>
<a class="pagination-link {% if p == current_page %}active{% endif %}" href="{{ page_url(p) }}">{{ p }}</a>
{% endfor %}
{# Right ellipsis #}
@@ -47,14 +60,14 @@
{% if end_page < total_pages - 1 %}
<span class="pagination-ellipsis">…</span>
{% endif %}
<a class="pagination-link" href="{{ url_for(endpoint, page=total_pages, per_page=per_page, **request.view_args) }}">{{ total_pages }}</a>
<a class="pagination-link" href="{{ page_url(total_pages) }}">{{ total_pages }}</a>
{% endif %}
{# Next and Last #}
{% if images.has_next %}
<a class="pagination-link" href="{{ url_for(endpoint, page=images.next_num, per_page=per_page, **request.view_args) }}">Next →</a>
<a class="pagination-link" href="{{ page_url(images.next_num) }}">Next →</a>
{% if current_page < total_pages %}
<a class="pagination-link" href="{{ url_for(endpoint, page=total_pages, per_page=per_page, **request.view_args) }}">Last »</a>
<a class="pagination-link" href="{{ page_url(total_pages) }}">Last »</a>
{% endif %}
{% endif %}
</div>
app/templates/tags_list.html
+6 -4
View File
@@ -28,10 +28,12 @@
{% for tag in tag_data %}
<a href="{{ url_for('main.gallery', tag=tag.tag) }}" class="artist-card-link">
<div class="artist-card">
<h3>
{% if tag.kind == 'artist' %}🎨{% elif tag.kind == 'archive' %}🗜️{% else %}# {% endif %}
{{ tag.name }}
</h3>
<div class="card-title" title="{{ tag.name }}">
<h3 class="card-title-text">
{% if tag.kind == 'artist' %}🎨{% elif tag.kind == 'archive' %}🗜️{% else %}# {% endif %}
{{ tag.name }}
</h3>
</div>
<div class="artist-preview">
{% for image in tag.images %}
<div class="artist-thumb"
app/utils/image_importer.py
+184 -31
View File
@@ -15,13 +15,19 @@ import tempfile
import time
import uuid
from PIL import Image
from PIL import Image, ImageOps
import exifread
import imagehash
from pyunpack import Archive
from app import db
from app.models import ImageRecord, Tag, ArchiveRecord
FFMPEG_THUMB_TIMEOUT = int(os.environ.get("THUMBS_FFMPEG_TIMEOUT", "60")) # seconds
Image.MAX_IMAGE_PIXELS = int(os.environ.get("PIL_MAX_IMAGE_PIXELS", "178956970"))
ARCHIVE_NUM_WIDTH = int(os.environ.get("ARCHIVE_NUM_WIDTH", "4"))
# =============================================================================
# Configuration & Constants
@@ -250,15 +256,13 @@ def process_archive(archive_path, dest_dir, artist, existing_phashes):
archive_size = os.path.getsize(archive_path)
archive_hash = calculate_hash(archive_path)
# Skip if archive already processed
# Skip if this exact archive file was already processed (hash-based)
existing_archive = ArchiveRecord.query.filter_by(hash=archive_hash).first()
if existing_archive:
print(f"[SKIP] Archive already imported: {archive_path}")
return 0
tag_name = f"archive:{artist}/{os.path.basename(archive_path)}"
tmpdir = tempfile.mkdtemp(prefix="extract_")
touched_records = [] # ImageRecord objects we imported or matched
imported_or_tagged = 0
created_tag = None
@@ -288,6 +292,8 @@ def process_archive(archive_path, dest_dir, artist, existing_phashes):
# Only now: create/attach the archive tag if we actually touched images
if touched_records:
# Incremental name like 'archive:{artist}/0001'
tag_name = compute_next_archive_tag_name(artist)
created_tag = Tag.query.filter_by(name=tag_name).first()
if not created_tag:
created_tag = Tag(name=tag_name, kind='archive')
@@ -301,13 +307,13 @@ def process_archive(archive_path, dest_dir, artist, existing_phashes):
newly_tagged += 1
imported_or_tagged += newly_tagged
# Record this archive as processed.
# Record this archive as processed (tag may be None if no media)
arch = ArchiveRecord(
filename=archive_path,
file_size=archive_size,
hash=archive_hash,
artist=artist,
tag=created_tag # None if we had no media
tag=created_tag
)
db.session.add(arch)
db.session.commit()
@@ -430,27 +436,89 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash
# Helpers: Thumbnails, metadata, hashing, similarity
# =============================================================================
def generate_thumbnail(image_path: str, size: tuple[int, int] = (400, 400), overwrite: bool = False) -> str:
def generate_thumbnail(
image_path,
size=(400, 400),
overwrite=False,
prefer_jpeg=False, # False = auto-pick PNG for alpha images; True = always JPEG
jpeg_bg=(0, 0, 0), # background used when flattening to JPEG
):
"""
Save thumbnails mirrored under /images/thumbs/<artist>/<unique_filename>.
(unique_filename includes the hash suffix, so no collisions)
Save thumbnails mirrored under /images/thumbs/<...>.
If prefer_jpeg is False (default), transparent images save as PNG, others as JPEG.
If prefer_jpeg is True, all thumbs are JPEG with transparency flattened to `jpeg_bg`.
Returns the absolute path to the thumbnail.
"""
images_root = Path("/images").resolve()
image_path = Path(image_path).resolve()
rel_path = image_path.relative_to(images_root) # raises if not under /images
thumb_path = images_root / "thumbs" / rel_path
thumb_path = thumb_path.with_suffix(".jpg") if thumb_path.suffix.lower() not in (".jpg", ".jpeg") else thumb_path
thumb_path.parent.mkdir(parents=True, exist_ok=True)
try:
rel_path = image_path.relative_to(images_root)
except ValueError:
raise ValueError(f"{image_path} is not under {images_root}")
if not overwrite and thumb_path.exists():
return str(thumb_path)
# Base path under /images/thumbs with same subfolders/filename (we may change the suffix)
thumb_base = (images_root / "thumbs" / rel_path).with_suffix("") # drop original suffix for control
# Decide output extension/format
with Image.open(image_path) as img:
img = ImageOps.exif_transpose(img) # honor camera rotation
img.thumbnail(size)
img.save(thumb_path)
return str(thumb_path)
has_alpha = _has_alpha(img)
if prefer_jpeg:
# Force JPEG for consistency
out_path = thumb_base.with_suffix(".jpg")
out_path.parent.mkdir(parents=True, exist_ok=True)
if not overwrite and out_path.exists():
return str(out_path)
img_rgb = _flatten_to_rgb(img, bg=jpeg_bg)
img_rgb.save(out_path, format="JPEG", quality=85, optimize=True, progressive=True)
return str(out_path)
else:
# Auto-pick: PNG for alpha, JPEG otherwise
if has_alpha:
out_path = thumb_base.with_suffix(".png")
out_path.parent.mkdir(parents=True, exist_ok=True)
if not overwrite and out_path.exists():
return str(out_path)
# Keep transparency if present
# Convert palette to RGBA for reliable PNG writing
if img.mode == "P":
img = img.convert("RGBA")
img.save(out_path, format="PNG", optimize=True)
return str(out_path)
else:
out_path = thumb_base.with_suffix(".jpg")
out_path.parent.mkdir(parents=True, exist_ok=True)
if not overwrite and out_path.exists():
return str(out_path)
if img.mode != "RGB":
img = img.convert("RGB")
img.save(out_path, format="JPEG", quality=85, optimize=True, progressive=True)
return str(out_path)
def generate_video_thumbnail(video_path: str, output_path: str, time_position: str = "00:00:01") -> str | None:
"""
Extract a single JPG frame from a video using ffmpeg and write it to output_path.
Returns output_path on success, or None on failure.
"""
try:
os.makedirs(os.path.dirname(output_path), exist_ok=True)
cmd = [
"ffmpeg", "-y",
"-ss", time_position,
"-i", str(video_path),
"-vframes", "1",
"-vf", f"scale={THUMB_SIZE[0]}:-1",
str(output_path),
]
subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=FFMPEG_THUMB_TIMEOUT)
return output_path
except subprocess.CalledProcessError as e:
print(f"[WARN] Failed to extract video thumbnail for {video_path}: {e}")
return None
def generate_video_thumbnail_mirrored(video_path: str, time_position: str = "00:00:01") -> str | None:
@@ -458,21 +526,11 @@ def generate_video_thumbnail_mirrored(video_path: str, time_position: str = "00:
Mirror the stored (hashed) video path under /images/thumbs/... and save as .jpg.
"""
images_root = Path("/images").resolve()
video_path = Path(video_path).resolve()
rel_path = video_path.relative_to(images_root) # raises if not under /images
video_path_p = Path(video_path).resolve()
rel_path = video_path_p.relative_to(images_root) # raises if not under /images
thumb_path = (images_root / "thumbs" / rel_path).with_suffix(".jpg")
thumb_path.parent.mkdir(parents=True, exist_ok=True)
return generate_video_thumbnail(str(video_path_p), str(thumb_path), time_position=time_position)
cmd = [
"ffmpeg", "-i", str(video_path), "-ss", time_position,
"-vframes", "1", "-vf", f"scale={THUMB_SIZE[0]}:-1", str(thumb_path)
]
try:
subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
return str(thumb_path)
except subprocess.CalledProcessError as e:
print(f"[WARN] Failed to extract video thumbnail for {video_path}: {e}")
return None
def calculate_hash(file_path: str) -> str:
@@ -556,6 +614,33 @@ def extract_metadata(file_path: str) -> dict:
return metadata
def _has_alpha(img: Image.Image) -> bool:
"""Return True if the image has any transparency channel."""
if img.mode in ("RGBA", "LA"):
return True
if img.mode == "P" and "transparency" in img.info:
return True
return False
def _flatten_to_rgb(img: Image.Image, bg=(0, 0, 0)) -> Image.Image:
"""
Flatten an RGBA/LA/Palette-with-alpha image onto a solid background,
returning an RGB image suitable for saving as JPEG.
"""
# Normalize to RGBA so we can use the alpha channel as a mask
if img.mode == "P":
img = img.convert("RGBA")
if img.mode in ("RGBA", "LA"):
# Ensure we have an explicit alpha channel as last band
if img.mode == "LA":
img = img.convert("RGBA")
alpha = img.split()[-1] # A channel
bg_img = Image.new("RGB", img.size, bg)
bg_img.paste(img, mask=alpha)
return bg_img
# No alpha, just convert to RGB
return img.convert("RGB")
# =============================================================================
# Helpers: Multi-part detection
@@ -596,4 +681,72 @@ def is_first_volume(path: str) -> bool:
if name.endswith((".zip", ".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz")):
return True
return True
return True
def compute_next_archive_tag_name(artist: str, width: int = ARCHIVE_NUM_WIDTH) -> str:
"""
Return a unique incremental tag name like: 'archive:{artist}/0001', '0002', ...
- Looks only at Tag(kind='archive') with names starting 'archive:{artist}/'
- Finds max numeric suffix and returns next number (zero-padded).
- Ensures no collision even if tags were renamed.
"""
prefix = f"archive:{artist}/"
# Pull existing tag names for this artist/prefix
rows = (Tag.query
.with_entities(Tag.name)
.filter(Tag.kind == 'archive',
Tag.name.like(prefix + '%'))
.all())
max_n = 0
for (name,) in rows:
tail = name[len(prefix):]
if tail.isdigit():
try:
n = int(tail)
if n > max_n:
max_n = n
except Exception:
pass
# Propose next number and make sure it doesn't already exist
n = max_n + 1
while True:
candidate = f"{prefix}{str(n).zfill(width)}"
if not Tag.query.filter_by(name=candidate).first():
return candidate
n += 1
def compute_next_archive_tag_name(artist: str, width: int = ARCHIVE_NUM_WIDTH) -> str:
"""
Return a unique incremental tag name like: 'archive:{artist}/0001', '0002', ...
- Looks only at Tag(kind='archive') with names starting 'archive:{artist}/'
- Finds max numeric suffix and returns next number (zero-padded).
- Ensures no collision even if tags were renamed.
"""
prefix = f"archive:{artist}/"
# Pull existing tag names for this artist/prefix
rows = (Tag.query
.with_entities(Tag.name)
.filter(Tag.kind == 'archive',
Tag.name.like(prefix + '%'))
.all())
max_n = 0
for (name,) in rows:
tail = name[len(prefix):]
if tail.isdigit():
try:
n = int(tail)
if n > max_n:
max_n = n
except Exception:
pass
# Propose next number and make sure it doesn't already exist
n = max_n + 1
while True:
candidate = f"{prefix}{str(n).zfill(width)}"
if not Tag.query.filter_by(name=candidate).first():
return candidate
n += 1
image_import_worker.py
+254 -55
View File
@@ -1,91 +1,290 @@
# image_import_worker.py
import time
import os
import time
from sqlalchemy.exc import OperationalError
from app import create_app
from app.utils.image_importer import import_images_task
THUMBNAIL_FLAG = "/import/thumbnail.flag"
TRIGGER_FLAG = "/import/trigger.flag"
SOURCE_DIR = "/import"
DEST_DIR = "/images"
TRIGGER_FLAG = "/import/trigger.flag"
SOURCE_DIR = "/import"
DEST_DIR = "/images"
CHECK_INTERVAL = 10 # seconds
# Verbosity controls (set via environment)
THUMBS_VERBOSE = os.getenv("THUMBS_VERBOSE", "0").lower() in ("1", "true", "yes")
THUMBS_LOG_EVERY = int(os.getenv("THUMBS_LOG_EVERY", "50")) # log every N images if verbose
THUMBS_BATCH_SIZE = int(os.getenv("THUMBS_BATCH_SIZE", "500")) # DB chunk size per commit
app = create_app()
from sqlalchemy.exc import OperationalError
import time
def monitor_and_import():
with app.app_context():
print("[Image Importer] Monitoring for trigger flags...")
print("[Image Importer] Monitoring for trigger flags... (CHECK_INTERVAL=%ss)" % CHECK_INTERVAL, flush=True)
while True:
if os.path.exists(TRIGGER_FLAG):
print("[Image Importer] Import flag found. Starting import...")
max_retries = 3
attempt = 0
while attempt < max_retries:
try:
result = import_images_task(SOURCE_DIR, DEST_DIR)
print(f"[Image Importer] {result}")
break # Exit retry loop if successful
except OperationalError as e:
attempt += 1
print(f"[Image Importer] Database error during import (attempt {attempt}): {e}")
if attempt >= max_retries:
print("[Image Importer] Max retries reached. Import failed.")
else:
sleep_time = 2 ** attempt
print(f"[Image Importer] Retrying in {sleep_time} seconds...")
time.sleep(sleep_time)
except Exception as e:
print(f"[Image Importer] Unexpected error during import: {e}")
break
finally:
try:
os.remove(TRIGGER_FLAG)
print("[Image Importer] Import flag cleared.")
except Exception as e:
print(f"[Image Importer] Failed to remove import flag: {e}")
did_work = False
# ------------------------------------------------------------------
# THUMBNAIL REGEN (PRIORITIZED) — chunked loop, detailed progress
# ------------------------------------------------------------------
if os.path.exists(THUMBNAIL_FLAG):
print("[Image Importer] Thumbnail regeneration triggered.")
print("[Image Importer] Thumbnail regeneration flag detected.", flush=True)
RUN_FLAG = THUMBNAIL_FLAG + ".run"
ERR_FLAG = THUMBNAIL_FLAG + ".err"
# Claim atomically so only one worker runs
try:
os.replace(THUMBNAIL_FLAG, RUN_FLAG)
print(f"[Image Importer] Claimed thumbnail job -> {RUN_FLAG}", flush=True)
except Exception as e:
print(f"[Image Importer] Could not claim thumbnail job: {e}", flush=True)
time.sleep(CHECK_INTERVAL)
continue
max_retries = 3
attempt = 0
while attempt < max_retries:
try:
from pathlib import Path
from sqlalchemy import text
from app import db
from app.models import ImageRecord
from app.utils.image_importer import generate_thumbnail
from app.utils.image_importer import generate_thumbnail, generate_video_thumbnail
# Early diagnostics
db.session.execute(text("SELECT 1"))
total = db.session.query(ImageRecord).count()
print(f"[Thumbs] Starting regen; ImageRecord count = {total}", flush=True)
images_root = Path("/images").resolve()
batch_size = THUMBS_BATCH_SIZE
processed = 0 # total processed rows
writes = 0 # files written/overwritten
path_updates = 0 # DB thumb_path changes
failures = 0
last_id = 0
t_start = time.time()
while True:
rows = (
ImageRecord.query
.filter(ImageRecord.id > last_id)
.order_by(ImageRecord.id.asc())
.limit(batch_size)
.all()
)
if not rows:
break
batch_start_id = rows[0].id
batch_end_id = rows[-1].id
batch_processed = 0
batch_writes = 0
batch_updates = 0
batch_failures = 0
bt0 = time.time()
for img in rows:
try:
processed += 1
batch_processed += 1
ext = (img.format or "").lower()
is_video = ext in ("mp4", "mov") or img.filepath.lower().endswith((".mp4", ".mov"))
# Determine output path for videos if needed
if is_video:
out_path = img.thumb_path
if not out_path:
try:
rel = Path(img.filepath).resolve().relative_to(images_root)
except Exception:
batch_failures += 1
failures += 1
if THUMBS_VERBOSE:
print(f"[Thumbs] SKIP (not under /images): id={img.id} {img.filepath}", flush=True)
continue
out_path = str((images_root / "thumbs" / rel).with_suffix(".jpg"))
new_path = generate_video_thumbnail(img.filepath, out_path)
else:
new_path = generate_thumbnail(img.filepath, overwrite=True)
# Count file writes: our helpers return the path on success
if new_path:
writes += 1
batch_writes += 1
# Update DB if path changed
if new_path and new_path != img.thumb_path:
img.thumb_path = new_path
path_updates += 1
batch_updates += 1
# Optional per-file log
if THUMBS_VERBOSE and (processed <= 10 or processed % THUMBS_LOG_EVERY == 0):
kind = "video" if is_video else "image"
print(f"[Thumbs] #{processed} ({kind}) id={img.id} name='{img.filename}' -> {new_path or 'None'}", flush=True)
except Exception as e:
failures += 1
batch_failures += 1
print(f"[Thumbs] Failed for id={img.id} name='{img.filename}': {e}", flush=True)
# Commit AFTER each chunk; safe for Postgres (no server-side cursor)
db.session.commit()
last_id = rows[-1].id
db.session.expunge_all() # keep memory stable
# Batch progress
bt = time.time() - bt0
rate = batch_processed / bt if bt > 0 else 0
print(
f"[Thumbs] Batch {batch_start_id}-{batch_end_id}: "
f"processed={batch_processed}, writes={batch_writes}, "
f"db_updates={batch_updates}, failures={batch_failures} "
f"({rate:.1f} items/s) — totals: processed={processed}, writes={writes}, "
f"db_updates={path_updates}, failures={failures} (last_id={last_id})",
flush=True
)
# Final summary
elapsed = time.time() - t_start
overall_rate = processed / elapsed if elapsed > 0 else 0
print(
f"[Image Importer] Thumbnail regen COMPLETE in {elapsed:.1f}s — "
f"processed={processed}, writes={writes}, db_updates={path_updates}, failures={failures} "
f"({overall_rate:.1f} items/s).",
flush=True
)
# Success — remove the .run flag and exit
try:
os.remove(RUN_FLAG)
print("[Image Importer] Thumbnail .run flag cleared.", flush=True)
except Exception as e:
print(f"[Image Importer] Could not remove .run flag: {e}", flush=True)
break # success
for img in ImageRecord.query.all():
try:
generate_thumbnail(img.filepath, overwrite=True)
print(f"[Thumbnail] Regenerated for {img.filename}")
except Exception as e:
print(f"[Thumbnail] Failed for {img.filename}: {e}")
break # Success
except OperationalError as e:
attempt += 1
print(f"[Image Importer] Database error during thumbnail regen (attempt {attempt}): {e}")
print(f"[Image Importer] DB error during thumbnail regen (attempt {attempt}): {e}", flush=True)
# Recover the session for the next attempt
try:
from app import db
db.session.rollback()
except Exception:
pass
time.sleep(2 ** attempt)
if attempt >= max_retries:
print("[Image Importer] Max retries reached. Thumbnail regeneration failed.")
try:
os.replace(RUN_FLAG, ERR_FLAG)
print(f"[Image Importer] Moved thumbnail flag to {ERR_FLAG}", flush=True)
except Exception as re:
print(f"[Image Importer] Could not mark error flag: {re}", flush=True)
except Exception as e:
print(f"[Image Importer] Unexpected error during thumbnail regen: {e}", flush=True)
try:
os.replace(RUN_FLAG, ERR_FLAG)
print(f"[Image Importer] Moved thumbnail flag to {ERR_FLAG}", flush=True)
except Exception as re:
print(f"[Image Importer] Could not mark error flag: {re}", flush=True)
break
did_work = True # handled a job (success or fail)
# ------------------------------------------------------------------
# IMPORT TRIGGER (robust: claim -> run -> clear/err; rollback on fail)
# ------------------------------------------------------------------
if os.path.exists(TRIGGER_FLAG):
print("[Image Importer] Import flag found. Starting import...", flush=True)
RUN_FLAG = TRIGGER_FLAG + ".run"
ERR_FLAG = TRIGGER_FLAG + ".err"
# Claim atomically so only one worker takes it
try:
os.replace(TRIGGER_FLAG, RUN_FLAG)
print(f"[Image Importer] Claimed import job -> {RUN_FLAG}", flush=True)
except Exception as e:
print(f"[Image Importer] Could not claim import job: {e}", flush=True)
time.sleep(CHECK_INTERVAL)
continue
max_retries = 3
attempt = 0
while attempt < max_retries:
try:
from sqlalchemy import text
from app import db
# Fast DB ping to catch stale pool conns
db.session.execute(text("SELECT 1"))
# Run the import
result = import_images_task(SOURCE_DIR, DEST_DIR)
print(f"[Image Importer] {result}", flush=True)
# Success — remove the .run flag and exit
try:
os.remove(RUN_FLAG)
print("[Image Importer] Import .run flag cleared.", flush=True)
except Exception as e:
print(f"[Image Importer] Could not remove import .run flag: {e}", flush=True)
break # success
except OperationalError as e:
attempt += 1
print(f"[Image Importer] DB error during import (attempt {attempt}): {e}", flush=True)
# IMPORTANT: rollback failed transaction & drop pool to force clean conns
try:
from app import db
db.session.rollback()
db.engine.dispose()
except Exception:
pass
if attempt >= max_retries:
try:
os.replace(RUN_FLAG, ERR_FLAG)
print(f"[Image Importer] Moved import flag to {ERR_FLAG}", flush=True)
except Exception as re:
print(f"[Image Importer] Could not mark import error flag: {re}", flush=True)
else:
sleep_time = 2 ** attempt
print(f"[Image Importer] Retrying in {sleep_time} seconds...")
print(f"[Image Importer] Retrying in {sleep_time} seconds...", flush=True)
time.sleep(sleep_time)
except Exception as e:
print(f"[Image Importer] Unexpected error during thumbnail regen: {e}")
break
finally:
print(f"[Image Importer] Unexpected error during import: {e}", flush=True)
# Make sure the session is usable next time
try:
os.remove(THUMBNAIL_FLAG)
print("[Image Importer] Thumbnail flag cleared.")
except Exception as e:
print(f"[Image Importer] Failed to remove thumbnail flag: {e}")
from app import db
db.session.rollback()
db.engine.dispose()
except Exception:
pass
try:
os.replace(RUN_FLAG, ERR_FLAG)
print(f"[Image Importer] Moved import flag to {ERR_FLAG}", flush=True)
except Exception as re:
print(f"[Image Importer] Could not mark import error flag: {re}", flush=True)
break
did_work = True
# ------------------------------------------------------------------
# IDLE SLEEP
# ------------------------------------------------------------------
if not did_work:
time.sleep(CHECK_INTERVAL)
if __name__ == '__main__':
if __name__ == "__main__":
monitor_and_import()