feat(integrity): structural verification + supersede-on-replace pipeline

Adds per-image integrity tracking so corrupt files are detected, excluded
from random/showcase/ML/suggestion paths, and recoverable by dropping a
fresh copy in /import — closing the gap that surfaced as the WD14
'6 bytes not processed' OSError.

Schema (migration l26042501)
- image_record.integrity_status: unknown | ok | truncated | unreadable | missing
- image_record.integrity_checked_at: timestamptz
- partial index on status <> 'ok' for cheap report/filter queries

Verifier
- app/services/integrity.py: verify_path() dispatches by extension
- PIL two-stage (verify + load with LOAD_TRUNCATED_IMAGES disabled)
- ffprobe for video, zipfile.testzip for archives
- Truncation-vs-unreadable distinction via PIL message hints

Pipeline
- verify_media_integrity Celery task: per-image, idempotent
- verify_unverified_images sweep: only_unknown by default, skips
  paths in active import tasks
- Hooked into the end of import_media_file (new + archive paths) and
  the supersede branch
- supersede_image() resets status to 'unknown' so the post-supersede
  verify writes a fresh truth
- Supersede-on-replace: a fresh /import/<artist>/<filename> matching
  a flagged-corrupt record routes through _supersede_existing,
  preserving tags/series/embeddings

Exclusions
- /, /api/random-images, tag_and_embed, ml.backfill enqueue, and
  get_suggestions all filter integrity_status IN ('ok', 'unknown') so
  flagged rows don't poison the gallery, ML, or suggestion math.
  'unknown' is treated as healthy so post-migration data stays visible
  until the sweep runs.

UI / report
- Settings -> Maintenance: 'Verify unknown' + 'Force re-verify all'
- GET /api/integrity/failed (paginated list of flagged rows)

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

app/tasks/import_file.py

@@ -231,6 +231,31 @@ def import_media_file(self, task_id: int):
 
             return _skip_task(task, 'Duplicate by hash', result_image_id=existing.id)
 
+        # Supersede-by-name for previously-flagged corrupt records.
+        # If the user (or the downloader) drops a fresh copy of a file we
+        # earlier marked corrupt at /import/<artist>/<filename>, the bytes
+        # don't match (so the hash dedup above missed it) but the source
+        # path does. Treat it as a supersede so tags/series/embeddings
+        # carry over and the old corrupt /images/... file gets replaced.
+        flagged = (
+            ImageRecord.query
+            .filter(ImageRecord.integrity_status.in_(('truncated', 'unreadable', 'missing')))
+            .filter(ImageRecord.filename == filename)
+            .filter(ImageRecord.filepath.like(f"%/{artist}/%"))
+            .first()
+        )
+        if flagged is not None:
+            phash = calculate_perceptual_hash(src_path)
+            log.info(
+                f"Supersede-on-replace: fresh /import/{artist}/{filename} replaces "
+                f"flagged image_id={flagged.id} (status={flagged.integrity_status})"
+            )
+            return _supersede_existing(
+                task, flagged, src_path, file_hash, phash,
+                metadata, artist, dest_dir, filename,
+                archive_path, archive_sidecar_path,
+            )
+
         # pHash similarity check
         # Quick mode: SKIP pHash comparison (relies on content hash for duplicates)
         # Deep mode: Full pHash comparison for similarity detection
@@ -372,6 +397,14 @@ def import_media_file(self, task_id: int):
             # Never let an enqueue failure roll back an otherwise-successful import.
             log.warning(f"Could not enqueue tag_and_embed for image {record.id}: {ml_err}")
 
+        # Queue structural integrity verification. Runs cheaply on the
+        # maintenance queue; result lands as integrity_status on the row.
+        try:
+            from app.tasks.maintenance import verify_media_integrity
+            verify_media_integrity.delay(record.id)
+        except Exception as v_err:
+            log.warning(f"Could not enqueue verify_media_integrity for image {record.id}: {v_err}")
+
         # Update batch stats
         if task.batch_id:
             from app.tasks.scan import update_batch_stats
@@ -760,6 +793,13 @@ def _process_archive_file(src_path: str, artist: str, dest_dir: str,
     if sidecar_path:
         apply_sidecar_metadata.delay(record.id, sidecar_path)
 
+    # Queue integrity verification for the extracted file too.
+    try:
+        from app.tasks.maintenance import verify_media_integrity
+        verify_media_integrity.delay(record.id)
+    except Exception as v_err:
+        log.warning(f"Could not enqueue verify_media_integrity for image {record.id}: {v_err}")
+
     log.debug(f"Imported from archive: {actual_filename} -> {dest_path}")
 
     return {
@@ -936,6 +976,14 @@ def _supersede_existing(task: ImportTask, old_record: ImageRecord, src_path: str
         from app.tasks.scan import update_batch_stats
         update_batch_stats.delay(task.batch_id)
 
+    # Re-verify the superseded file. supersede_image already cleared
+    # integrity_status to 'unknown'; this resolves it back to a real value.
+    try:
+        from app.tasks.maintenance import verify_media_integrity
+        verify_media_integrity.delay(record.id)
+    except Exception as v_err:
+        log.warning(f"Could not enqueue verify_media_integrity (supersede) for image {record.id}: {v_err}")
+
     return {
         'status': 'superseded',
         'image_id': record.id,

app/tasks/maintenance.py

@@ -6,12 +6,14 @@ task was removed on 2026-04-21 as part of the bare-name refactor.
 """
 import logging
 import re
+from datetime import datetime, timezone
 
 from celery import shared_task
 from sqlalchemy import text
 
 from app import db
-from app.models import Tag
+from app.models import ImageRecord, ImportTask, Tag
+from app.services.integrity import verify_path
 
 log = logging.getLogger(__name__)
 
@@ -205,6 +207,94 @@ def sync_character_fandoms_to_images() -> dict:
     }
 
 
+@shared_task(
+    name='app.tasks.maintenance.verify_media_integrity',
+    soft_time_limit=60,
+    time_limit=120,
+)
+def verify_media_integrity(image_id: int) -> dict:
+    """Run structural verification on one image and persist the result.
+
+    Idempotent — safe to re-enqueue freely; each call re-checks the file
+    on disk and writes a fresh `integrity_checked_at` timestamp. Used both
+    by the post-import hook (so newly-imported rows land verified) and by
+    the sweep task (which re-verifies the existing library).
+    """
+    image = ImageRecord.query.get(image_id)
+    if image is None:
+        return {'image_id': image_id, 'status': 'no_record'}
+
+    status, detail = verify_path(image.filepath)
+    image.integrity_status = status
+    image.integrity_checked_at = datetime.now(timezone.utc)
+    db.session.commit()
+
+    if status != 'ok':
+        log.warning(
+            "verify_media_integrity: image_id=%s path=%r → %s (%s)",
+            image_id, image.filepath, status, detail,
+        )
+    return {'image_id': image_id, 'status': status, 'detail': detail}
+
+
+@shared_task(
+    name='app.tasks.maintenance.verify_unverified_images',
+    soft_time_limit=300,
+    time_limit=600,
+)
+def verify_unverified_images(only_unknown: bool = True) -> dict:
+    """Sweep every ImageRecord and enqueue per-image verify tasks.
+
+    Skips paths that are currently the target of an in-flight import — same
+    contract `deep_scan_directory` uses, so a half-written file mid-import
+    doesn't get false-flagged. By default only revisits rows whose status is
+    still 'unknown' (post-migration default + brand-new rows that haven't
+    been picked up by the import-time hook yet); pass only_unknown=False to
+    force a full re-verify across the library.
+    """
+    active_paths = {
+        row[0] for row in db.session.execute(text("""
+            SELECT source_path FROM import_task
+            WHERE status IN ('pending', 'queued', 'processing')
+        """)).fetchall()
+    }
+
+    q = ImageRecord.query.filter(ImageRecord.filepath.isnot(None))
+    if only_unknown:
+        q = q.filter(ImageRecord.integrity_status == 'unknown')
+
+    enqueued = 0
+    skipped_active = 0
+    last_id = 0
+    BATCH = 500
+    while True:
+        rows = (
+            q.filter(ImageRecord.id > last_id)
+            .order_by(ImageRecord.id.asc())
+            .limit(BATCH)
+            .all()
+        )
+        if not rows:
+            break
+        for r in rows:
+            if r.filepath in active_paths:
+                skipped_active += 1
+                continue
+            verify_media_integrity.delay(r.id)
+            enqueued += 1
+        last_id = rows[-1].id
+
+    log.info(
+        "verify_unverified_images: enqueued=%d skipped_active=%d only_unknown=%s",
+        enqueued, skipped_active, only_unknown,
+    )
+    return {
+        'enqueued': enqueued,
+        'skipped_active': skipped_active,
+        'only_unknown': only_unknown,
+    }
+
+
 @shared_task(
     name='app.tasks.maintenance.sweep_blocklisted_tag_from_images',
     soft_time_limit=60,

app/tasks/ml.py

@@ -100,6 +100,15 @@ def tag_and_embed(self, image_id: int):
         log.warning(f"tag_and_embed: file missing at {image.filepath}")
         return {'status': 'file_missing'}
 
+    # Skip flagged-corrupt rows. The verifier already saw the file fail
+    # structurally; running inference would either crash or produce noise.
+    # 'unknown' (pre-sweep / mid-import) still proceeds.
+    if image.integrity_status not in ('ok', 'unknown'):
+        log.info(
+            f"tag_and_embed: skipping image {image_id} — integrity={image.integrity_status}"
+        )
+        return {'status': 'skipped_integrity', 'integrity': image.integrity_status}
+
     is_video = image.filepath.lower().endswith(VIDEO_EXTS)
 
     try:
@@ -187,6 +196,7 @@ def backfill(self, batch_size: int = 50, pause_seconds: float = 0.5):
                 ),
             )
             .filter(ImageRecord.id > last_id)
+            .filter(ImageRecord.integrity_status.in_(('ok', 'unknown')))
             .filter(
                 (ImageTagPrediction.image_id.is_(None))
                 | (ImageEmbedding.image_id.is_(None))