feat(integrity): structural verification + supersede-on-replace pipeline
Adds per-image integrity tracking so corrupt files are detected, excluded
from random/showcase/ML/suggestion paths, and recoverable by dropping a
fresh copy in /import — closing the gap that surfaced as the WD14
'6 bytes not processed' OSError.
Schema (migration l26042501)
- image_record.integrity_status: unknown | ok | truncated | unreadable | missing
- image_record.integrity_checked_at: timestamptz
- partial index on status <> 'ok' for cheap report/filter queries
Verifier
- app/services/integrity.py: verify_path() dispatches by extension
- PIL two-stage (verify + load with LOAD_TRUNCATED_IMAGES disabled)
- ffprobe for video, zipfile.testzip for archives
- Truncation-vs-unreadable distinction via PIL message hints
Pipeline
- verify_media_integrity Celery task: per-image, idempotent
- verify_unverified_images sweep: only_unknown by default, skips
paths in active import tasks
- Hooked into the end of import_media_file (new + archive paths) and
the supersede branch
- supersede_image() resets status to 'unknown' so the post-supersede
verify writes a fresh truth
- Supersede-on-replace: a fresh /import/<artist>/<filename> matching
a flagged-corrupt record routes through _supersede_existing,
preserving tags/series/embeddings
Exclusions
- /, /api/random-images, tag_and_embed, ml.backfill enqueue, and
get_suggestions all filter integrity_status IN ('ok', 'unknown') so
flagged rows don't poison the gallery, ML, or suggestion math.
'unknown' is treated as healthy so post-migration data stays visible
until the sweep runs.
UI / report
- Settings -> Maintenance: 'Verify unknown' + 'Force re-verify all'
- GET /api/integrity/failed (paginated list of flagged rows)
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -231,6 +231,31 @@ def import_media_file(self, task_id: int):
|
||||
|
||||
return _skip_task(task, 'Duplicate by hash', result_image_id=existing.id)
|
||||
|
||||
# Supersede-by-name for previously-flagged corrupt records.
|
||||
# If the user (or the downloader) drops a fresh copy of a file we
|
||||
# earlier marked corrupt at /import/<artist>/<filename>, the bytes
|
||||
# don't match (so the hash dedup above missed it) but the source
|
||||
# path does. Treat it as a supersede so tags/series/embeddings
|
||||
# carry over and the old corrupt /images/... file gets replaced.
|
||||
flagged = (
|
||||
ImageRecord.query
|
||||
.filter(ImageRecord.integrity_status.in_(('truncated', 'unreadable', 'missing')))
|
||||
.filter(ImageRecord.filename == filename)
|
||||
.filter(ImageRecord.filepath.like(f"%/{artist}/%"))
|
||||
.first()
|
||||
)
|
||||
if flagged is not None:
|
||||
phash = calculate_perceptual_hash(src_path)
|
||||
log.info(
|
||||
f"Supersede-on-replace: fresh /import/{artist}/{filename} replaces "
|
||||
f"flagged image_id={flagged.id} (status={flagged.integrity_status})"
|
||||
)
|
||||
return _supersede_existing(
|
||||
task, flagged, src_path, file_hash, phash,
|
||||
metadata, artist, dest_dir, filename,
|
||||
archive_path, archive_sidecar_path,
|
||||
)
|
||||
|
||||
# pHash similarity check
|
||||
# Quick mode: SKIP pHash comparison (relies on content hash for duplicates)
|
||||
# Deep mode: Full pHash comparison for similarity detection
|
||||
@@ -372,6 +397,14 @@ def import_media_file(self, task_id: int):
|
||||
# Never let an enqueue failure roll back an otherwise-successful import.
|
||||
log.warning(f"Could not enqueue tag_and_embed for image {record.id}: {ml_err}")
|
||||
|
||||
# Queue structural integrity verification. Runs cheaply on the
|
||||
# maintenance queue; result lands as integrity_status on the row.
|
||||
try:
|
||||
from app.tasks.maintenance import verify_media_integrity
|
||||
verify_media_integrity.delay(record.id)
|
||||
except Exception as v_err:
|
||||
log.warning(f"Could not enqueue verify_media_integrity for image {record.id}: {v_err}")
|
||||
|
||||
# Update batch stats
|
||||
if task.batch_id:
|
||||
from app.tasks.scan import update_batch_stats
|
||||
@@ -760,6 +793,13 @@ def _process_archive_file(src_path: str, artist: str, dest_dir: str,
|
||||
if sidecar_path:
|
||||
apply_sidecar_metadata.delay(record.id, sidecar_path)
|
||||
|
||||
# Queue integrity verification for the extracted file too.
|
||||
try:
|
||||
from app.tasks.maintenance import verify_media_integrity
|
||||
verify_media_integrity.delay(record.id)
|
||||
except Exception as v_err:
|
||||
log.warning(f"Could not enqueue verify_media_integrity for image {record.id}: {v_err}")
|
||||
|
||||
log.debug(f"Imported from archive: {actual_filename} -> {dest_path}")
|
||||
|
||||
return {
|
||||
@@ -936,6 +976,14 @@ def _supersede_existing(task: ImportTask, old_record: ImageRecord, src_path: str
|
||||
from app.tasks.scan import update_batch_stats
|
||||
update_batch_stats.delay(task.batch_id)
|
||||
|
||||
# Re-verify the superseded file. supersede_image already cleared
|
||||
# integrity_status to 'unknown'; this resolves it back to a real value.
|
||||
try:
|
||||
from app.tasks.maintenance import verify_media_integrity
|
||||
verify_media_integrity.delay(record.id)
|
||||
except Exception as v_err:
|
||||
log.warning(f"Could not enqueue verify_media_integrity (supersede) for image {record.id}: {v_err}")
|
||||
|
||||
return {
|
||||
'status': 'superseded',
|
||||
'image_id': record.id,
|
||||
|
||||
Reference in New Issue
Block a user