feat(integrity): structural verification + supersede-on-replace pipeline
Adds per-image integrity tracking so corrupt files are detected, excluded
from random/showcase/ML/suggestion paths, and recoverable by dropping a
fresh copy in /import — closing the gap that surfaced as the WD14
'6 bytes not processed' OSError.
Schema (migration l26042501)
- image_record.integrity_status: unknown | ok | truncated | unreadable | missing
- image_record.integrity_checked_at: timestamptz
- partial index on status <> 'ok' for cheap report/filter queries
Verifier
- app/services/integrity.py: verify_path() dispatches by extension
- PIL two-stage (verify + load with LOAD_TRUNCATED_IMAGES disabled)
- ffprobe for video, zipfile.testzip for archives
- Truncation-vs-unreadable distinction via PIL message hints
Pipeline
- verify_media_integrity Celery task: per-image, idempotent
- verify_unverified_images sweep: only_unknown by default, skips
paths in active import tasks
- Hooked into the end of import_media_file (new + archive paths) and
the supersede branch
- supersede_image() resets status to 'unknown' so the post-supersede
verify writes a fresh truth
- Supersede-on-replace: a fresh /import/<artist>/<filename> matching
a flagged-corrupt record routes through _supersede_existing,
preserving tags/series/embeddings
Exclusions
- /, /api/random-images, tag_and_embed, ml.backfill enqueue, and
get_suggestions all filter integrity_status IN ('ok', 'unknown') so
flagged rows don't poison the gallery, ML, or suggestion math.
'unknown' is treated as healthy so post-migration data stays visible
until the sweep runs.
UI / report
- Settings -> Maintenance: 'Verify unknown' + 'Force re-verify all'
- GET /api/integrity/failed (paginated list of flagged rows)
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -73,6 +73,8 @@ def make_celery(app=None):
|
|||||||
'app.tasks.scan.update_batch_stats': {'queue': 'maintenance'},
|
'app.tasks.scan.update_batch_stats': {'queue': 'maintenance'},
|
||||||
'app.tasks.maintenance.sweep_blocklisted_tag_from_images': {'queue': 'maintenance'},
|
'app.tasks.maintenance.sweep_blocklisted_tag_from_images': {'queue': 'maintenance'},
|
||||||
'app.tasks.maintenance.sync_character_fandoms_to_images': {'queue': 'maintenance'},
|
'app.tasks.maintenance.sync_character_fandoms_to_images': {'queue': 'maintenance'},
|
||||||
|
'app.tasks.maintenance.verify_media_integrity': {'queue': 'maintenance'},
|
||||||
|
'app.tasks.maintenance.verify_unverified_images': {'queue': 'maintenance'},
|
||||||
|
|
||||||
# Import tasks - handled by worker (heavy processing)
|
# Import tasks - handled by worker (heavy processing)
|
||||||
'app.tasks.import_file.*': {'queue': 'import'},
|
'app.tasks.import_file.*': {'queue': 'import'},
|
||||||
|
|||||||
+72
-2
@@ -38,6 +38,7 @@ def index():
|
|||||||
images = (
|
images = (
|
||||||
ImageRecord.query
|
ImageRecord.query
|
||||||
.options(joinedload(ImageRecord.tags))
|
.options(joinedload(ImageRecord.tags))
|
||||||
|
.filter(ImageRecord.integrity_status.in_(('ok', 'unknown')))
|
||||||
.order_by(func.random())
|
.order_by(func.random())
|
||||||
.limit(20)
|
.limit(20)
|
||||||
.all()
|
.all()
|
||||||
@@ -65,8 +66,14 @@ def random_images_api():
|
|||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Build query
|
# Build query — exclude flagged-corrupt rows so the shuffle never serves
|
||||||
q = ImageRecord.query.options(joinedload(ImageRecord.tags))
|
# a thumbnail-broken image. 'unknown' rows (pre-sweep / freshly imported)
|
||||||
|
# are treated as healthy until the verifier proves otherwise.
|
||||||
|
q = (
|
||||||
|
ImageRecord.query
|
||||||
|
.options(joinedload(ImageRecord.tags))
|
||||||
|
.filter(ImageRecord.integrity_status.in_(('ok', 'unknown')))
|
||||||
|
)
|
||||||
|
|
||||||
if exclude_ids:
|
if exclude_ids:
|
||||||
q = q.filter(~ImageRecord.id.in_(exclude_ids))
|
q = q.filter(~ImageRecord.id.in_(exclude_ids))
|
||||||
@@ -648,6 +655,69 @@ def trigger_sync_character_fandoms_to_images():
|
|||||||
return redirect(url_for('main.settings', tab='maintenance'))
|
return redirect(url_for('main.settings', tab='maintenance'))
|
||||||
|
|
||||||
|
|
||||||
|
@main.post('/settings/maintenance/verify-images')
|
||||||
|
def trigger_verify_unverified_images():
|
||||||
|
"""Sweep ImageRecords and enqueue per-image structural verification.
|
||||||
|
|
||||||
|
Form param `mode=all` forces re-verify across the whole library; the
|
||||||
|
default skips rows already in a non-'unknown' state so re-clicking the
|
||||||
|
button doesn't thrash through cleanly-verified data.
|
||||||
|
"""
|
||||||
|
from app.tasks.maintenance import verify_unverified_images
|
||||||
|
only_unknown = request.form.get('mode', 'unknown') != 'all'
|
||||||
|
verify_unverified_images.apply_async(
|
||||||
|
kwargs={'only_unknown': only_unknown}, queue='maintenance',
|
||||||
|
)
|
||||||
|
return redirect(url_for('main.settings', tab='maintenance'))
|
||||||
|
|
||||||
|
|
||||||
|
@main.get('/api/integrity/failed')
|
||||||
|
def api_integrity_failed():
|
||||||
|
"""List ImageRecords whose structural verification failed.
|
||||||
|
|
||||||
|
Query params:
|
||||||
|
- status: one of 'truncated' | 'unreadable' | 'missing' (optional;
|
||||||
|
omit to return every non-'ok' / non-'unknown' row)
|
||||||
|
- limit: page size (default 100, max 500)
|
||||||
|
- offset: pagination cursor
|
||||||
|
"""
|
||||||
|
status_filter = (request.args.get('status') or '').strip()
|
||||||
|
limit = min(request.args.get('limit', 100, type=int), 500)
|
||||||
|
offset = max(request.args.get('offset', 0, type=int), 0)
|
||||||
|
|
||||||
|
q = ImageRecord.query.filter(
|
||||||
|
ImageRecord.integrity_status.in_(('truncated', 'unreadable', 'missing'))
|
||||||
|
)
|
||||||
|
if status_filter in ('truncated', 'unreadable', 'missing'):
|
||||||
|
q = q.filter(ImageRecord.integrity_status == status_filter)
|
||||||
|
|
||||||
|
total = q.count()
|
||||||
|
rows = (
|
||||||
|
q.order_by(ImageRecord.integrity_checked_at.desc().nullslast(),
|
||||||
|
ImageRecord.id.desc())
|
||||||
|
.limit(limit)
|
||||||
|
.offset(offset)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return jsonify(
|
||||||
|
ok=True,
|
||||||
|
total=total,
|
||||||
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
|
items=[
|
||||||
|
{
|
||||||
|
'id': r.id,
|
||||||
|
'filename': r.filename,
|
||||||
|
'filepath': r.filepath,
|
||||||
|
'integrity_status': r.integrity_status,
|
||||||
|
'integrity_checked_at': r.integrity_checked_at.isoformat()
|
||||||
|
if r.integrity_checked_at else None,
|
||||||
|
}
|
||||||
|
for r in rows
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# ----------------------------
|
# ----------------------------
|
||||||
# Tag add/remove endpoints
|
# Tag add/remove endpoints
|
||||||
# ----------------------------
|
# ----------------------------
|
||||||
|
|||||||
@@ -34,6 +34,12 @@ class ImageRecord(db.Model):
|
|||||||
taken_at = db.Column(db.DateTime, nullable=True)
|
taken_at = db.Column(db.DateTime, nullable=True)
|
||||||
imported_at = db.Column(db.DateTime, default=db.func.now())
|
imported_at = db.Column(db.DateTime, default=db.func.now())
|
||||||
is_thumbnail = db.Column(db.Boolean, default=False)
|
is_thumbnail = db.Column(db.Boolean, default=False)
|
||||||
|
# Structural-integrity state set by app.services.integrity.verify_path.
|
||||||
|
# 'ok' rows are healthy; everything else is excluded from random/showcase/ML/
|
||||||
|
# suggestion paths. See migration l26042501 for the value enumeration.
|
||||||
|
integrity_status = db.Column(db.String(16), nullable=False, default='unknown',
|
||||||
|
server_default='unknown')
|
||||||
|
integrity_checked_at = db.Column(db.DateTime(timezone=True), nullable=True)
|
||||||
tags = db.relationship(
|
tags = db.relationship(
|
||||||
"Tag",
|
"Tag",
|
||||||
secondary=image_tags,
|
secondary=image_tags,
|
||||||
|
|||||||
@@ -0,0 +1,171 @@
|
|||||||
|
"""Structural-integrity verification for media files.
|
||||||
|
|
||||||
|
Public surface: `verify_path(path) -> (status, detail)`.
|
||||||
|
Status values match the `image_record.integrity_status` column:
|
||||||
|
- 'ok': passed all checks
|
||||||
|
- 'truncated': structurally a valid file up to a point, then missing
|
||||||
|
trailing bytes (the WD14 '6 bytes not processed' case)
|
||||||
|
- 'unreadable': can't open at all, or container is malformed enough that
|
||||||
|
even the head doesn't parse
|
||||||
|
- 'missing': filepath doesn't exist on disk
|
||||||
|
|
||||||
|
Dispatch is by file extension. Images go through PIL (with truncated-image
|
||||||
|
loading explicitly disabled so we surface the very thing we want to detect).
|
||||||
|
Videos go through ffprobe. Archives go through zipfile.testzip / tarfile
|
||||||
|
where the stdlib supports it. Anything else returns 'ok' (we don't know
|
||||||
|
how to validate it, so we don't lie about its state).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
# We deliberately import the PIL.ImageFile module here without flipping
|
||||||
|
# LOAD_TRUNCATED_IMAGES — opposite of wd14.py's runtime tolerance, since
|
||||||
|
# this module's job is to *detect* truncation, not paper over it.
|
||||||
|
from PIL import Image, UnidentifiedImageError
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
IMAGE_EXTS = ('.jpg', '.jpeg', '.jfif', '.png', '.gif', '.bmp', '.tiff', '.webp')
|
||||||
|
VIDEO_EXTS = ('.mp4', '.mov', '.avi', '.mkv', '.webm', '.m4v', '.wmv', '.flv')
|
||||||
|
ZIP_EXTS = ('.zip', '.cbr', '.cbz')
|
||||||
|
|
||||||
|
|
||||||
|
def verify_path(path: str) -> tuple[str, str | None]:
|
||||||
|
"""Verify a single file. Returns (status, detail-or-None).
|
||||||
|
|
||||||
|
Detail is a short human-readable string for the failure case, useful
|
||||||
|
for the report endpoint and logs. None when status='ok'.
|
||||||
|
"""
|
||||||
|
if not os.path.exists(path):
|
||||||
|
return 'missing', None
|
||||||
|
try:
|
||||||
|
size = os.path.getsize(path)
|
||||||
|
except OSError as e:
|
||||||
|
return 'missing', f'stat failed: {e}'
|
||||||
|
if size == 0:
|
||||||
|
return 'unreadable', 'zero-byte file'
|
||||||
|
|
||||||
|
ext = os.path.splitext(path)[1].lower()
|
||||||
|
|
||||||
|
if ext in IMAGE_EXTS:
|
||||||
|
return _verify_image(path)
|
||||||
|
if ext in VIDEO_EXTS:
|
||||||
|
return _verify_video(path)
|
||||||
|
if ext in ZIP_EXTS:
|
||||||
|
return _verify_zip(path)
|
||||||
|
# Unknown extension: don't fail it, but don't lie that we verified it.
|
||||||
|
# Caller writes 'ok' but the kind=unknown case is rare in this library.
|
||||||
|
return 'ok', None
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_image(path: str) -> tuple[str, str | None]:
|
||||||
|
"""PIL-based two-stage check.
|
||||||
|
|
||||||
|
Stage 1: open + verify(). Validates header, structure, and (for most
|
||||||
|
formats) walks the entire stream to confirm framing — this is what
|
||||||
|
catches the 6-byte truncation. verify() invalidates the Image after,
|
||||||
|
so callers can't draw from it (we don't need to).
|
||||||
|
|
||||||
|
Stage 2: re-open + load(). Some PIL formats (notably JPEG) only flag
|
||||||
|
truncation during load(), not verify(). Re-opening with truncated
|
||||||
|
loading disabled lets a malformed scan trigger OSError here.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
with Image.open(path) as img:
|
||||||
|
img.verify()
|
||||||
|
except (OSError, SyntaxError, UnidentifiedImageError, ValueError) as e:
|
||||||
|
msg = str(e).lower()
|
||||||
|
if _looks_truncated(msg):
|
||||||
|
return 'truncated', str(e)
|
||||||
|
return 'unreadable', str(e)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Force a full decode without the LOAD_TRUNCATED_IMAGES safety net
|
||||||
|
# that wd14/siglip enabled at module-import time.
|
||||||
|
from PIL import ImageFile
|
||||||
|
prev = ImageFile.LOAD_TRUNCATED_IMAGES
|
||||||
|
ImageFile.LOAD_TRUNCATED_IMAGES = False
|
||||||
|
try:
|
||||||
|
with Image.open(path) as img:
|
||||||
|
img.load()
|
||||||
|
finally:
|
||||||
|
ImageFile.LOAD_TRUNCATED_IMAGES = prev
|
||||||
|
except (OSError, SyntaxError, ValueError) as e:
|
||||||
|
msg = str(e).lower()
|
||||||
|
if _looks_truncated(msg):
|
||||||
|
return 'truncated', str(e)
|
||||||
|
return 'unreadable', str(e)
|
||||||
|
|
||||||
|
return 'ok', None
|
||||||
|
|
||||||
|
|
||||||
|
# Phrases PIL emits across formats when the file ends earlier than the
|
||||||
|
# decoder expected. Centralized so both verify-stages agree on the mapping.
|
||||||
|
_TRUNCATED_HINTS = (
|
||||||
|
'trunc', # 'image file is truncated', 'truncated file read'
|
||||||
|
'not processed', # 'N bytes not processed' (the original wd14 case)
|
||||||
|
'broken png', # PNG short final chunk
|
||||||
|
'premature', # JPEG/PNG premature end
|
||||||
|
'unexpected end', # GIF / generic end-of-stream
|
||||||
|
'not enough', # 'not enough data'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _looks_truncated(lower_msg: str) -> bool:
|
||||||
|
return any(h in lower_msg for h in _TRUNCATED_HINTS)
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_video(path: str) -> tuple[str, str | None]:
|
||||||
|
"""ffprobe-based container walk.
|
||||||
|
|
||||||
|
`-v error` keeps stdout/stderr quiet on success and emits the first
|
||||||
|
structural complaint on failure. Any non-zero exit is treated as a
|
||||||
|
failed verification. ffprobe distinguishes 'truncated' poorly across
|
||||||
|
formats, so we lean on stderr text to map it.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
['ffprobe', '-v', 'error', '-show_format', '-show_streams', '-i', path],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
except FileNotFoundError:
|
||||||
|
log.warning("ffprobe not on PATH; skipping video verify for %s", path)
|
||||||
|
return 'ok', None
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
return 'unreadable', 'ffprobe timed out'
|
||||||
|
|
||||||
|
if result.returncode == 0:
|
||||||
|
return 'ok', None
|
||||||
|
err = (result.stderr or '').strip().lower()
|
||||||
|
if any(s in err for s in ('truncated', 'partial', 'eof', 'end of file', 'invalid data found')):
|
||||||
|
return 'truncated', result.stderr.strip()
|
||||||
|
return 'unreadable', result.stderr.strip() or f'ffprobe exit {result.returncode}'
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_zip(path: str) -> tuple[str, str | None]:
|
||||||
|
"""zipfile.testzip walks every CRC. None = clean; bad-name = corrupt.
|
||||||
|
|
||||||
|
BadZipFile / LargeZipFile are unreadable cases (header malformed or
|
||||||
|
too big to test), distinct from a CRC mismatch on a single member
|
||||||
|
which we treat as truncated/partial.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
with zipfile.ZipFile(path) as zf:
|
||||||
|
bad = zf.testzip()
|
||||||
|
except zipfile.BadZipFile as e:
|
||||||
|
return 'unreadable', str(e)
|
||||||
|
except zipfile.LargeZipFile as e:
|
||||||
|
return 'unreadable', str(e)
|
||||||
|
except OSError as e:
|
||||||
|
return 'unreadable', str(e)
|
||||||
|
|
||||||
|
if bad is None:
|
||||||
|
return 'ok', None
|
||||||
|
return 'truncated', f'CRC failed for member: {bad}'
|
||||||
@@ -248,7 +248,13 @@ def get_suggestions(
|
|||||||
The optional `cfg` and `existing_all` kwargs let callers that loop over many
|
The optional `cfg` and `existing_all` kwargs let callers that loop over many
|
||||||
images amortize the config fetch and the Tag-table scan.
|
images amortize the config fetch and the Tag-table scan.
|
||||||
"""
|
"""
|
||||||
if ImageRecord.query.get(image_id) is None:
|
img = ImageRecord.query.get(image_id)
|
||||||
|
if img is None:
|
||||||
|
return {'character': [], 'copyright': [], 'general': []}
|
||||||
|
# Don't compute suggestions for files we know are corrupt — the source
|
||||||
|
# WD14 predictions on them are unreliable, and the user shouldn't be
|
||||||
|
# nudged to attach tags to a broken image.
|
||||||
|
if img.integrity_status not in ('ok', 'unknown'):
|
||||||
return {'character': [], 'copyright': [], 'general': []}
|
return {'character': [], 'copyright': [], 'general': []}
|
||||||
|
|
||||||
if cfg is None:
|
if cfg is None:
|
||||||
|
|||||||
@@ -231,6 +231,31 @@ def import_media_file(self, task_id: int):
|
|||||||
|
|
||||||
return _skip_task(task, 'Duplicate by hash', result_image_id=existing.id)
|
return _skip_task(task, 'Duplicate by hash', result_image_id=existing.id)
|
||||||
|
|
||||||
|
# Supersede-by-name for previously-flagged corrupt records.
|
||||||
|
# If the user (or the downloader) drops a fresh copy of a file we
|
||||||
|
# earlier marked corrupt at /import/<artist>/<filename>, the bytes
|
||||||
|
# don't match (so the hash dedup above missed it) but the source
|
||||||
|
# path does. Treat it as a supersede so tags/series/embeddings
|
||||||
|
# carry over and the old corrupt /images/... file gets replaced.
|
||||||
|
flagged = (
|
||||||
|
ImageRecord.query
|
||||||
|
.filter(ImageRecord.integrity_status.in_(('truncated', 'unreadable', 'missing')))
|
||||||
|
.filter(ImageRecord.filename == filename)
|
||||||
|
.filter(ImageRecord.filepath.like(f"%/{artist}/%"))
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if flagged is not None:
|
||||||
|
phash = calculate_perceptual_hash(src_path)
|
||||||
|
log.info(
|
||||||
|
f"Supersede-on-replace: fresh /import/{artist}/{filename} replaces "
|
||||||
|
f"flagged image_id={flagged.id} (status={flagged.integrity_status})"
|
||||||
|
)
|
||||||
|
return _supersede_existing(
|
||||||
|
task, flagged, src_path, file_hash, phash,
|
||||||
|
metadata, artist, dest_dir, filename,
|
||||||
|
archive_path, archive_sidecar_path,
|
||||||
|
)
|
||||||
|
|
||||||
# pHash similarity check
|
# pHash similarity check
|
||||||
# Quick mode: SKIP pHash comparison (relies on content hash for duplicates)
|
# Quick mode: SKIP pHash comparison (relies on content hash for duplicates)
|
||||||
# Deep mode: Full pHash comparison for similarity detection
|
# Deep mode: Full pHash comparison for similarity detection
|
||||||
@@ -372,6 +397,14 @@ def import_media_file(self, task_id: int):
|
|||||||
# Never let an enqueue failure roll back an otherwise-successful import.
|
# Never let an enqueue failure roll back an otherwise-successful import.
|
||||||
log.warning(f"Could not enqueue tag_and_embed for image {record.id}: {ml_err}")
|
log.warning(f"Could not enqueue tag_and_embed for image {record.id}: {ml_err}")
|
||||||
|
|
||||||
|
# Queue structural integrity verification. Runs cheaply on the
|
||||||
|
# maintenance queue; result lands as integrity_status on the row.
|
||||||
|
try:
|
||||||
|
from app.tasks.maintenance import verify_media_integrity
|
||||||
|
verify_media_integrity.delay(record.id)
|
||||||
|
except Exception as v_err:
|
||||||
|
log.warning(f"Could not enqueue verify_media_integrity for image {record.id}: {v_err}")
|
||||||
|
|
||||||
# Update batch stats
|
# Update batch stats
|
||||||
if task.batch_id:
|
if task.batch_id:
|
||||||
from app.tasks.scan import update_batch_stats
|
from app.tasks.scan import update_batch_stats
|
||||||
@@ -760,6 +793,13 @@ def _process_archive_file(src_path: str, artist: str, dest_dir: str,
|
|||||||
if sidecar_path:
|
if sidecar_path:
|
||||||
apply_sidecar_metadata.delay(record.id, sidecar_path)
|
apply_sidecar_metadata.delay(record.id, sidecar_path)
|
||||||
|
|
||||||
|
# Queue integrity verification for the extracted file too.
|
||||||
|
try:
|
||||||
|
from app.tasks.maintenance import verify_media_integrity
|
||||||
|
verify_media_integrity.delay(record.id)
|
||||||
|
except Exception as v_err:
|
||||||
|
log.warning(f"Could not enqueue verify_media_integrity for image {record.id}: {v_err}")
|
||||||
|
|
||||||
log.debug(f"Imported from archive: {actual_filename} -> {dest_path}")
|
log.debug(f"Imported from archive: {actual_filename} -> {dest_path}")
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -936,6 +976,14 @@ def _supersede_existing(task: ImportTask, old_record: ImageRecord, src_path: str
|
|||||||
from app.tasks.scan import update_batch_stats
|
from app.tasks.scan import update_batch_stats
|
||||||
update_batch_stats.delay(task.batch_id)
|
update_batch_stats.delay(task.batch_id)
|
||||||
|
|
||||||
|
# Re-verify the superseded file. supersede_image already cleared
|
||||||
|
# integrity_status to 'unknown'; this resolves it back to a real value.
|
||||||
|
try:
|
||||||
|
from app.tasks.maintenance import verify_media_integrity
|
||||||
|
verify_media_integrity.delay(record.id)
|
||||||
|
except Exception as v_err:
|
||||||
|
log.warning(f"Could not enqueue verify_media_integrity (supersede) for image {record.id}: {v_err}")
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'status': 'superseded',
|
'status': 'superseded',
|
||||||
'image_id': record.id,
|
'image_id': record.id,
|
||||||
|
|||||||
@@ -6,12 +6,14 @@ task was removed on 2026-04-21 as part of the bare-name refactor.
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
|
|
||||||
from app import db
|
from app import db
|
||||||
from app.models import Tag
|
from app.models import ImageRecord, ImportTask, Tag
|
||||||
|
from app.services.integrity import verify_path
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -205,6 +207,94 @@ def sync_character_fandoms_to_images() -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@shared_task(
|
||||||
|
name='app.tasks.maintenance.verify_media_integrity',
|
||||||
|
soft_time_limit=60,
|
||||||
|
time_limit=120,
|
||||||
|
)
|
||||||
|
def verify_media_integrity(image_id: int) -> dict:
|
||||||
|
"""Run structural verification on one image and persist the result.
|
||||||
|
|
||||||
|
Idempotent — safe to re-enqueue freely; each call re-checks the file
|
||||||
|
on disk and writes a fresh `integrity_checked_at` timestamp. Used both
|
||||||
|
by the post-import hook (so newly-imported rows land verified) and by
|
||||||
|
the sweep task (which re-verifies the existing library).
|
||||||
|
"""
|
||||||
|
image = ImageRecord.query.get(image_id)
|
||||||
|
if image is None:
|
||||||
|
return {'image_id': image_id, 'status': 'no_record'}
|
||||||
|
|
||||||
|
status, detail = verify_path(image.filepath)
|
||||||
|
image.integrity_status = status
|
||||||
|
image.integrity_checked_at = datetime.now(timezone.utc)
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
if status != 'ok':
|
||||||
|
log.warning(
|
||||||
|
"verify_media_integrity: image_id=%s path=%r → %s (%s)",
|
||||||
|
image_id, image.filepath, status, detail,
|
||||||
|
)
|
||||||
|
return {'image_id': image_id, 'status': status, 'detail': detail}
|
||||||
|
|
||||||
|
|
||||||
|
@shared_task(
|
||||||
|
name='app.tasks.maintenance.verify_unverified_images',
|
||||||
|
soft_time_limit=300,
|
||||||
|
time_limit=600,
|
||||||
|
)
|
||||||
|
def verify_unverified_images(only_unknown: bool = True) -> dict:
|
||||||
|
"""Sweep every ImageRecord and enqueue per-image verify tasks.
|
||||||
|
|
||||||
|
Skips paths that are currently the target of an in-flight import — same
|
||||||
|
contract `deep_scan_directory` uses, so a half-written file mid-import
|
||||||
|
doesn't get false-flagged. By default only revisits rows whose status is
|
||||||
|
still 'unknown' (post-migration default + brand-new rows that haven't
|
||||||
|
been picked up by the import-time hook yet); pass only_unknown=False to
|
||||||
|
force a full re-verify across the library.
|
||||||
|
"""
|
||||||
|
active_paths = {
|
||||||
|
row[0] for row in db.session.execute(text("""
|
||||||
|
SELECT source_path FROM import_task
|
||||||
|
WHERE status IN ('pending', 'queued', 'processing')
|
||||||
|
""")).fetchall()
|
||||||
|
}
|
||||||
|
|
||||||
|
q = ImageRecord.query.filter(ImageRecord.filepath.isnot(None))
|
||||||
|
if only_unknown:
|
||||||
|
q = q.filter(ImageRecord.integrity_status == 'unknown')
|
||||||
|
|
||||||
|
enqueued = 0
|
||||||
|
skipped_active = 0
|
||||||
|
last_id = 0
|
||||||
|
BATCH = 500
|
||||||
|
while True:
|
||||||
|
rows = (
|
||||||
|
q.filter(ImageRecord.id > last_id)
|
||||||
|
.order_by(ImageRecord.id.asc())
|
||||||
|
.limit(BATCH)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
if not rows:
|
||||||
|
break
|
||||||
|
for r in rows:
|
||||||
|
if r.filepath in active_paths:
|
||||||
|
skipped_active += 1
|
||||||
|
continue
|
||||||
|
verify_media_integrity.delay(r.id)
|
||||||
|
enqueued += 1
|
||||||
|
last_id = rows[-1].id
|
||||||
|
|
||||||
|
log.info(
|
||||||
|
"verify_unverified_images: enqueued=%d skipped_active=%d only_unknown=%s",
|
||||||
|
enqueued, skipped_active, only_unknown,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
'enqueued': enqueued,
|
||||||
|
'skipped_active': skipped_active,
|
||||||
|
'only_unknown': only_unknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@shared_task(
|
@shared_task(
|
||||||
name='app.tasks.maintenance.sweep_blocklisted_tag_from_images',
|
name='app.tasks.maintenance.sweep_blocklisted_tag_from_images',
|
||||||
soft_time_limit=60,
|
soft_time_limit=60,
|
||||||
|
|||||||
@@ -100,6 +100,15 @@ def tag_and_embed(self, image_id: int):
|
|||||||
log.warning(f"tag_and_embed: file missing at {image.filepath}")
|
log.warning(f"tag_and_embed: file missing at {image.filepath}")
|
||||||
return {'status': 'file_missing'}
|
return {'status': 'file_missing'}
|
||||||
|
|
||||||
|
# Skip flagged-corrupt rows. The verifier already saw the file fail
|
||||||
|
# structurally; running inference would either crash or produce noise.
|
||||||
|
# 'unknown' (pre-sweep / mid-import) still proceeds.
|
||||||
|
if image.integrity_status not in ('ok', 'unknown'):
|
||||||
|
log.info(
|
||||||
|
f"tag_and_embed: skipping image {image_id} — integrity={image.integrity_status}"
|
||||||
|
)
|
||||||
|
return {'status': 'skipped_integrity', 'integrity': image.integrity_status}
|
||||||
|
|
||||||
is_video = image.filepath.lower().endswith(VIDEO_EXTS)
|
is_video = image.filepath.lower().endswith(VIDEO_EXTS)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -187,6 +196,7 @@ def backfill(self, batch_size: int = 50, pause_seconds: float = 0.5):
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
.filter(ImageRecord.id > last_id)
|
.filter(ImageRecord.id > last_id)
|
||||||
|
.filter(ImageRecord.integrity_status.in_(('ok', 'unknown')))
|
||||||
.filter(
|
.filter(
|
||||||
(ImageTagPrediction.image_id.is_(None))
|
(ImageTagPrediction.image_id.is_(None))
|
||||||
| (ImageEmbedding.image_id.is_(None))
|
| (ImageEmbedding.image_id.is_(None))
|
||||||
|
|||||||
@@ -470,6 +470,30 @@
|
|||||||
|
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
<section class="settings-section">
|
||||||
|
<h3>Verify image integrity</h3>
|
||||||
|
<p class="settings-hint">
|
||||||
|
Walks every image and runs a structural check (PIL for images,
|
||||||
|
ffprobe for video, zipfile for archives). Flagged rows are excluded
|
||||||
|
from random/showcase, ML retag, and the suggestion modal until a
|
||||||
|
fresh copy lands in <code>/import/<artist>/<filename></code>
|
||||||
|
— at which point the import pipeline supersedes the corrupt row,
|
||||||
|
preserving its tags. The default mode only checks rows whose status
|
||||||
|
is still <code>unknown</code> (post-migration default + freshly
|
||||||
|
imported); use "force re-verify all" after disk events to recheck
|
||||||
|
everything. Flagged rows are listed at
|
||||||
|
<code>/api/integrity/failed</code>.
|
||||||
|
</p>
|
||||||
|
<form action="{{ url_for('main.trigger_verify_unverified_images') }}" method="post" style="display:inline-block;">
|
||||||
|
<button type="submit" class="btn secondary-btn">Verify unknown</button>
|
||||||
|
</form>
|
||||||
|
<form action="{{ url_for('main.trigger_verify_unverified_images') }}" method="post" style="display:inline-block; margin-left:0.5rem;"
|
||||||
|
onsubmit="return confirm('Re-verify every image in the library? Cheap per-file but enqueues one task per image.');">
|
||||||
|
<input type="hidden" name="mode" value="all">
|
||||||
|
<button type="submit" class="btn secondary-btn">Force re-verify all</button>
|
||||||
|
</form>
|
||||||
|
</section>
|
||||||
|
|
||||||
<section class="settings-section">
|
<section class="settings-section">
|
||||||
<h3>Auto-accept threshold (general tags)</h3>
|
<h3>Auto-accept threshold (general tags)</h3>
|
||||||
<p class="settings-hint">
|
<p class="settings-hint">
|
||||||
|
|||||||
@@ -252,6 +252,11 @@ def supersede_image(old_record: "ImageRecord", new_filepath: str, new_thumb_path
|
|||||||
old_record.width = new_metadata["width"]
|
old_record.width = new_metadata["width"]
|
||||||
old_record.height = new_metadata["height"]
|
old_record.height = new_metadata["height"]
|
||||||
old_record.format = new_metadata["format"]
|
old_record.format = new_metadata["format"]
|
||||||
|
# The bytes on disk just changed — the prior verification result is no
|
||||||
|
# longer authoritative. Reset to 'unknown' so the post-supersede verify
|
||||||
|
# hook (or the next sweep) re-records the truth.
|
||||||
|
old_record.integrity_status = 'unknown'
|
||||||
|
old_record.integrity_checked_at = None
|
||||||
# Keep the earlier taken_at date (prefer original date)
|
# Keep the earlier taken_at date (prefer original date)
|
||||||
if new_metadata.get("taken_at") and old_record.taken_at:
|
if new_metadata.get("taken_at") and old_record.taken_at:
|
||||||
if new_metadata["taken_at"] < old_record.taken_at:
|
if new_metadata["taken_at"] < old_record.taken_at:
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
"""Add integrity_status + integrity_checked_at to image_record.
|
||||||
|
|
||||||
|
Tracks per-image structural verification state so corrupt files (truncated
|
||||||
|
downloads, broken containers, partial archives) can be flagged and excluded
|
||||||
|
from random/showcase/ML/suggestion paths instead of blowing up downstream
|
||||||
|
preprocessing.
|
||||||
|
|
||||||
|
Status values:
|
||||||
|
- unknown: never verified (default for existing rows + freshly imported
|
||||||
|
until the verifier runs)
|
||||||
|
- ok: passed marker check + format-level verify
|
||||||
|
- truncated: missing trailing bytes (PIL EOI/IEND, ffprobe end-of-stream)
|
||||||
|
- unreadable: header/structure invalid; can't even open the file
|
||||||
|
- missing: filepath doesn't exist on disk
|
||||||
|
|
||||||
|
Revision ID: l26042501
|
||||||
|
Revises: k26042201
|
||||||
|
Create Date: 2026-04-25
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = 'l26042501'
|
||||||
|
down_revision = 'k26042201'
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
op.add_column(
|
||||||
|
'image_record',
|
||||||
|
sa.Column(
|
||||||
|
'integrity_status',
|
||||||
|
sa.String(16),
|
||||||
|
nullable=False,
|
||||||
|
server_default='unknown',
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
'image_record',
|
||||||
|
sa.Column(
|
||||||
|
'integrity_checked_at',
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
# Partial index so the "list flagged rows" report and the
|
||||||
|
# exclude-from-random/ML filters stay cheap on a million-row table.
|
||||||
|
op.create_index(
|
||||||
|
'image_record_integrity_status_idx',
|
||||||
|
'image_record',
|
||||||
|
['integrity_status'],
|
||||||
|
postgresql_where=sa.text("integrity_status <> 'ok'"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
op.drop_index('image_record_integrity_status_idx', table_name='image_record')
|
||||||
|
op.drop_column('image_record', 'integrity_checked_at')
|
||||||
|
op.drop_column('image_record', 'integrity_status')
|
||||||
Reference in New Issue
Block a user