132 lines
4.5 KiB
Python
132 lines
4.5 KiB
Python
"""generate_thumbnail task: PIL/ffmpeg thumbnail generation on the thumbnail queue.
|
|
|
|
Lives separately from import_file because thumbnails can be regenerated en
|
|
masse (FC-2c adds the 'regenerate all' admin action) and they're CPU-bound
|
|
so they deserve their own queue lane.
|
|
"""
|
|
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy.exc import DBAPIError, OperationalError
|
|
|
|
from ..celery_app import celery
|
|
from ..models import ImageRecord
|
|
from ..services.importer import is_video
|
|
from ..services.thumbnailer import Thumbnailer
|
|
from ._sync_engine import sync_session_factory as _sync_session_factory
|
|
|
|
IMAGES_ROOT = Path("/images")
|
|
|
|
THUMB_MAGIC_JPEG = b"\xff\xd8\xff"
|
|
THUMB_MAGIC_PNG = b"\x89PNG\r\n\x1a\n"
|
|
|
|
|
|
def _thumb_is_valid(path: Path) -> bool:
|
|
"""Return True iff `path` exists and starts with a JPEG or PNG magic header.
|
|
|
|
The on-disk thumbnail format is set by services/thumbnailer.py — JPEG for
|
|
opaque sources, PNG for alpha sources. Anything else (missing file, OSError,
|
|
truncated, wrong magic) is invalid.
|
|
"""
|
|
try:
|
|
with path.open("rb") as f:
|
|
head = f.read(12)
|
|
except OSError:
|
|
return False
|
|
if len(head) < 8:
|
|
return False
|
|
if head[:3] == THUMB_MAGIC_JPEG:
|
|
return True
|
|
if head[:8] == THUMB_MAGIC_PNG:
|
|
return True
|
|
return False
|
|
|
|
|
|
@celery.task(
|
|
name="backend.app.tasks.thumbnail.generate_thumbnail",
|
|
bind=True,
|
|
autoretry_for=(OperationalError, DBAPIError, OSError),
|
|
retry_backoff=5,
|
|
retry_backoff_max=60,
|
|
retry_jitter=True,
|
|
max_retries=3,
|
|
soft_time_limit=120,
|
|
time_limit=180,
|
|
)
|
|
def generate_thumbnail(self, image_id: int) -> dict:
|
|
SessionLocal = _sync_session_factory()
|
|
with SessionLocal() as session:
|
|
record = session.get(ImageRecord, image_id)
|
|
if record is None:
|
|
return {"status": "missing", "image_id": image_id}
|
|
|
|
thumbnailer = Thumbnailer(images_root=IMAGES_ROOT)
|
|
source = Path(record.path)
|
|
try:
|
|
if is_video(source):
|
|
result = thumbnailer.generate_video_thumbnail(source, record.sha256)
|
|
else:
|
|
result = thumbnailer.generate_image_thumbnail(source, record.sha256)
|
|
except Exception as exc: # pragma: no cover — thumbnail failure is non-fatal
|
|
return {"status": "failed", "image_id": image_id, "error": f"{type(exc).__name__}: {exc}"}
|
|
|
|
record.thumbnail_path = str(result.path)
|
|
session.add(record)
|
|
session.commit()
|
|
return {"status": "ok", "image_id": image_id, "path": str(result.path)}
|
|
|
|
|
|
@celery.task(
|
|
name="backend.app.tasks.thumbnail.backfill_thumbnails",
|
|
bind=True,
|
|
)
|
|
def backfill_thumbnails(self) -> dict:
|
|
"""Scan ImageRecord and enqueue generate_thumbnail for rows whose
|
|
thumbnail is missing, gone from disk, or has wrong magic bytes.
|
|
|
|
Keyset paginates by id ASC, page size 500. NULLs out thumbnail_path for
|
|
rows that point at a missing or corrupt file before enqueueing — keeps
|
|
the DB self-consistent on partial runs and makes re-runs safe.
|
|
|
|
Returns {"enqueued": N, "ok": M, "regenerated": K} where:
|
|
- enqueued = total generate_thumbnail.delay() calls
|
|
- ok = rows whose existing thumbnail file is valid (skipped)
|
|
- regenerated = subset of enqueued that had a non-NULL thumbnail_path
|
|
cleared (i.e. missing + corrupt)
|
|
"""
|
|
from sqlalchemy import select, update
|
|
|
|
SessionLocal = _sync_session_factory()
|
|
enqueued = 0
|
|
ok = 0
|
|
regenerated = 0
|
|
last_id = 0
|
|
with SessionLocal() as session:
|
|
while True:
|
|
rows = session.execute(
|
|
select(ImageRecord.id, ImageRecord.thumbnail_path)
|
|
.where(ImageRecord.id > last_id)
|
|
.order_by(ImageRecord.id.asc())
|
|
.limit(500)
|
|
).all()
|
|
if not rows:
|
|
break
|
|
for image_id, thumb_path in rows:
|
|
if thumb_path is None:
|
|
generate_thumbnail.delay(image_id)
|
|
enqueued += 1
|
|
elif _thumb_is_valid(Path(thumb_path)):
|
|
ok += 1
|
|
else:
|
|
session.execute(
|
|
update(ImageRecord)
|
|
.where(ImageRecord.id == image_id)
|
|
.values(thumbnail_path=None)
|
|
)
|
|
generate_thumbnail.delay(image_id)
|
|
enqueued += 1
|
|
regenerated += 1
|
|
session.commit()
|
|
last_id = rows[-1][0]
|
|
return {"enqueued": enqueued, "ok": ok, "regenerated": regenerated}
|