feat(integrity): structural verification + supersede-on-replace pipeline
Adds per-image integrity tracking so corrupt files are detected, excluded
from random/showcase/ML/suggestion paths, and recoverable by dropping a
fresh copy in /import — closing the gap that surfaced as the WD14
'6 bytes not processed' OSError.
Schema (migration l26042501)
- image_record.integrity_status: unknown | ok | truncated | unreadable | missing
- image_record.integrity_checked_at: timestamptz
- partial index on status <> 'ok' for cheap report/filter queries
Verifier
- app/services/integrity.py: verify_path() dispatches by extension
- PIL two-stage (verify + load with LOAD_TRUNCATED_IMAGES disabled)
- ffprobe for video, zipfile.testzip for archives
- Truncation-vs-unreadable distinction via PIL message hints
Pipeline
- verify_media_integrity Celery task: per-image, idempotent
- verify_unverified_images sweep: only_unknown by default, skips
paths in active import tasks
- Hooked into the end of import_media_file (new + archive paths) and
the supersede branch
- supersede_image() resets status to 'unknown' so the post-supersede
verify writes a fresh truth
- Supersede-on-replace: a fresh /import/<artist>/<filename> matching
a flagged-corrupt record routes through _supersede_existing,
preserving tags/series/embeddings
Exclusions
- /, /api/random-images, tag_and_embed, ml.backfill enqueue, and
get_suggestions all filter integrity_status IN ('ok', 'unknown') so
flagged rows don't poison the gallery, ML, or suggestion math.
'unknown' is treated as healthy so post-migration data stays visible
until the sweep runs.
UI / report
- Settings -> Maintenance: 'Verify unknown' + 'Force re-verify all'
- GET /api/integrity/failed (paginated list of flagged rows)
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
+72
-2
@@ -38,6 +38,7 @@ def index():
|
||||
images = (
|
||||
ImageRecord.query
|
||||
.options(joinedload(ImageRecord.tags))
|
||||
.filter(ImageRecord.integrity_status.in_(('ok', 'unknown')))
|
||||
.order_by(func.random())
|
||||
.limit(20)
|
||||
.all()
|
||||
@@ -65,8 +66,14 @@ def random_images_api():
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Build query
|
||||
q = ImageRecord.query.options(joinedload(ImageRecord.tags))
|
||||
# Build query — exclude flagged-corrupt rows so the shuffle never serves
|
||||
# a thumbnail-broken image. 'unknown' rows (pre-sweep / freshly imported)
|
||||
# are treated as healthy until the verifier proves otherwise.
|
||||
q = (
|
||||
ImageRecord.query
|
||||
.options(joinedload(ImageRecord.tags))
|
||||
.filter(ImageRecord.integrity_status.in_(('ok', 'unknown')))
|
||||
)
|
||||
|
||||
if exclude_ids:
|
||||
q = q.filter(~ImageRecord.id.in_(exclude_ids))
|
||||
@@ -648,6 +655,69 @@ def trigger_sync_character_fandoms_to_images():
|
||||
return redirect(url_for('main.settings', tab='maintenance'))
|
||||
|
||||
|
||||
@main.post('/settings/maintenance/verify-images')
|
||||
def trigger_verify_unverified_images():
|
||||
"""Sweep ImageRecords and enqueue per-image structural verification.
|
||||
|
||||
Form param `mode=all` forces re-verify across the whole library; the
|
||||
default skips rows already in a non-'unknown' state so re-clicking the
|
||||
button doesn't thrash through cleanly-verified data.
|
||||
"""
|
||||
from app.tasks.maintenance import verify_unverified_images
|
||||
only_unknown = request.form.get('mode', 'unknown') != 'all'
|
||||
verify_unverified_images.apply_async(
|
||||
kwargs={'only_unknown': only_unknown}, queue='maintenance',
|
||||
)
|
||||
return redirect(url_for('main.settings', tab='maintenance'))
|
||||
|
||||
|
||||
@main.get('/api/integrity/failed')
|
||||
def api_integrity_failed():
|
||||
"""List ImageRecords whose structural verification failed.
|
||||
|
||||
Query params:
|
||||
- status: one of 'truncated' | 'unreadable' | 'missing' (optional;
|
||||
omit to return every non-'ok' / non-'unknown' row)
|
||||
- limit: page size (default 100, max 500)
|
||||
- offset: pagination cursor
|
||||
"""
|
||||
status_filter = (request.args.get('status') or '').strip()
|
||||
limit = min(request.args.get('limit', 100, type=int), 500)
|
||||
offset = max(request.args.get('offset', 0, type=int), 0)
|
||||
|
||||
q = ImageRecord.query.filter(
|
||||
ImageRecord.integrity_status.in_(('truncated', 'unreadable', 'missing'))
|
||||
)
|
||||
if status_filter in ('truncated', 'unreadable', 'missing'):
|
||||
q = q.filter(ImageRecord.integrity_status == status_filter)
|
||||
|
||||
total = q.count()
|
||||
rows = (
|
||||
q.order_by(ImageRecord.integrity_checked_at.desc().nullslast(),
|
||||
ImageRecord.id.desc())
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
.all()
|
||||
)
|
||||
return jsonify(
|
||||
ok=True,
|
||||
total=total,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
items=[
|
||||
{
|
||||
'id': r.id,
|
||||
'filename': r.filename,
|
||||
'filepath': r.filepath,
|
||||
'integrity_status': r.integrity_status,
|
||||
'integrity_checked_at': r.integrity_checked_at.isoformat()
|
||||
if r.integrity_checked_at else None,
|
||||
}
|
||||
for r in rows
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# Tag add/remove endpoints
|
||||
# ----------------------------
|
||||
|
||||
Reference in New Issue
Block a user