feat(integrity): structural verification + supersede-on-replace pipeline
Adds per-image integrity tracking so corrupt files are detected, excluded
from random/showcase/ML/suggestion paths, and recoverable by dropping a
fresh copy in /import — closing the gap that surfaced as the WD14
'6 bytes not processed' OSError.
Schema (migration l26042501)
- image_record.integrity_status: unknown | ok | truncated | unreadable | missing
- image_record.integrity_checked_at: timestamptz
- partial index on status <> 'ok' for cheap report/filter queries
Verifier
- app/services/integrity.py: verify_path() dispatches by extension
- PIL two-stage (verify + load with LOAD_TRUNCATED_IMAGES disabled)
- ffprobe for video, zipfile.testzip for archives
- Truncation-vs-unreadable distinction via PIL message hints
Pipeline
- verify_media_integrity Celery task: per-image, idempotent
- verify_unverified_images sweep: only_unknown by default, skips
paths in active import tasks
- Hooked into the end of import_media_file (new + archive paths) and
the supersede branch
- supersede_image() resets status to 'unknown' so the post-supersede
verify writes a fresh truth
- Supersede-on-replace: a fresh /import/<artist>/<filename> matching
a flagged-corrupt record routes through _supersede_existing,
preserving tags/series/embeddings
Exclusions
- /, /api/random-images, tag_and_embed, ml.backfill enqueue, and
get_suggestions all filter integrity_status IN ('ok', 'unknown') so
flagged rows don't poison the gallery, ML, or suggestion math.
'unknown' is treated as healthy so post-migration data stays visible
until the sweep runs.
UI / report
- Settings -> Maintenance: 'Verify unknown' + 'Force re-verify all'
- GET /api/integrity/failed (paginated list of flagged rows)
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -6,12 +6,14 @@ task was removed on 2026-04-21 as part of the bare-name refactor.
|
||||
"""
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from celery import shared_task
|
||||
from sqlalchemy import text
|
||||
|
||||
from app import db
|
||||
from app.models import Tag
|
||||
from app.models import ImageRecord, ImportTask, Tag
|
||||
from app.services.integrity import verify_path
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -205,6 +207,94 @@ def sync_character_fandoms_to_images() -> dict:
|
||||
}
|
||||
|
||||
|
||||
@shared_task(
|
||||
name='app.tasks.maintenance.verify_media_integrity',
|
||||
soft_time_limit=60,
|
||||
time_limit=120,
|
||||
)
|
||||
def verify_media_integrity(image_id: int) -> dict:
|
||||
"""Run structural verification on one image and persist the result.
|
||||
|
||||
Idempotent — safe to re-enqueue freely; each call re-checks the file
|
||||
on disk and writes a fresh `integrity_checked_at` timestamp. Used both
|
||||
by the post-import hook (so newly-imported rows land verified) and by
|
||||
the sweep task (which re-verifies the existing library).
|
||||
"""
|
||||
image = ImageRecord.query.get(image_id)
|
||||
if image is None:
|
||||
return {'image_id': image_id, 'status': 'no_record'}
|
||||
|
||||
status, detail = verify_path(image.filepath)
|
||||
image.integrity_status = status
|
||||
image.integrity_checked_at = datetime.now(timezone.utc)
|
||||
db.session.commit()
|
||||
|
||||
if status != 'ok':
|
||||
log.warning(
|
||||
"verify_media_integrity: image_id=%s path=%r → %s (%s)",
|
||||
image_id, image.filepath, status, detail,
|
||||
)
|
||||
return {'image_id': image_id, 'status': status, 'detail': detail}
|
||||
|
||||
|
||||
@shared_task(
|
||||
name='app.tasks.maintenance.verify_unverified_images',
|
||||
soft_time_limit=300,
|
||||
time_limit=600,
|
||||
)
|
||||
def verify_unverified_images(only_unknown: bool = True) -> dict:
|
||||
"""Sweep every ImageRecord and enqueue per-image verify tasks.
|
||||
|
||||
Skips paths that are currently the target of an in-flight import — same
|
||||
contract `deep_scan_directory` uses, so a half-written file mid-import
|
||||
doesn't get false-flagged. By default only revisits rows whose status is
|
||||
still 'unknown' (post-migration default + brand-new rows that haven't
|
||||
been picked up by the import-time hook yet); pass only_unknown=False to
|
||||
force a full re-verify across the library.
|
||||
"""
|
||||
active_paths = {
|
||||
row[0] for row in db.session.execute(text("""
|
||||
SELECT source_path FROM import_task
|
||||
WHERE status IN ('pending', 'queued', 'processing')
|
||||
""")).fetchall()
|
||||
}
|
||||
|
||||
q = ImageRecord.query.filter(ImageRecord.filepath.isnot(None))
|
||||
if only_unknown:
|
||||
q = q.filter(ImageRecord.integrity_status == 'unknown')
|
||||
|
||||
enqueued = 0
|
||||
skipped_active = 0
|
||||
last_id = 0
|
||||
BATCH = 500
|
||||
while True:
|
||||
rows = (
|
||||
q.filter(ImageRecord.id > last_id)
|
||||
.order_by(ImageRecord.id.asc())
|
||||
.limit(BATCH)
|
||||
.all()
|
||||
)
|
||||
if not rows:
|
||||
break
|
||||
for r in rows:
|
||||
if r.filepath in active_paths:
|
||||
skipped_active += 1
|
||||
continue
|
||||
verify_media_integrity.delay(r.id)
|
||||
enqueued += 1
|
||||
last_id = rows[-1].id
|
||||
|
||||
log.info(
|
||||
"verify_unverified_images: enqueued=%d skipped_active=%d only_unknown=%s",
|
||||
enqueued, skipped_active, only_unknown,
|
||||
)
|
||||
return {
|
||||
'enqueued': enqueued,
|
||||
'skipped_active': skipped_active,
|
||||
'only_unknown': only_unknown,
|
||||
}
|
||||
|
||||
|
||||
@shared_task(
|
||||
name='app.tasks.maintenance.sweep_blocklisted_tag_from_images',
|
||||
soft_time_limit=60,
|
||||
|
||||
Reference in New Issue
Block a user