feat(deep-scan): keyset-paginated backfill_phash task
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,16 +1,22 @@
|
||||
"""Periodic maintenance: recover stuck import tasks, garbage-collect old finished tasks."""
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from PIL import Image
|
||||
from sqlalchemy import create_engine, delete, select, update
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..config import get_config
|
||||
from ..models import ImportTask
|
||||
from ..models import ImageRecord, ImportTask
|
||||
from ..utils.phash import compute_phash
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
STUCK_THRESHOLD_MINUTES = 30
|
||||
OLD_TASK_DAYS = 7
|
||||
PHASH_PAGE = 500
|
||||
|
||||
|
||||
def _sync_session_factory():
|
||||
@@ -71,3 +77,41 @@ def cleanup_old_tasks() -> int:
|
||||
)
|
||||
session.commit()
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.backfill_phash")
|
||||
def backfill_phash() -> int:
|
||||
"""Recompute phash for stored images that have none (imported before
|
||||
FC-2d-i+ii). Keyset-paginated by id (restart-safe), NULL-only fill,
|
||||
idempotent. Videos legitimately keep phash NULL. A missing/unreadable
|
||||
file is logged and left NULL — never fails the task."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
updated = 0
|
||||
last_id = 0
|
||||
with SessionLocal() as session:
|
||||
while True:
|
||||
rows = session.execute(
|
||||
select(ImageRecord)
|
||||
.where(ImageRecord.id > last_id)
|
||||
.where(ImageRecord.phash.is_(None))
|
||||
.where(ImageRecord.mime.like("image/%"))
|
||||
.order_by(ImageRecord.id.asc())
|
||||
.limit(PHASH_PAGE)
|
||||
).scalars().all()
|
||||
if not rows:
|
||||
break
|
||||
for rec in rows:
|
||||
try:
|
||||
with Image.open(rec.path) as im:
|
||||
ph = compute_phash(im)
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"backfill_phash: unreadable %s: %s", rec.path, exc
|
||||
)
|
||||
ph = None
|
||||
if ph is not None and rec.phash is None:
|
||||
rec.phash = ph
|
||||
updated += 1
|
||||
session.commit()
|
||||
last_id = rows[-1].id
|
||||
return updated
|
||||
|
||||
Reference in New Issue
Block a user