fix(external): path-safe unlink + per-link staging + orphan repair (#859)
External downloads import IN PLACE, so the post-attach dedup-skip unlink could delete a file that IS an ImageRecord's backing file — orphaning the record and 404-ing on playback. Two sources of that: - Two links on the same post (same film from mega + gdrive) emitted the same filename into one external/<post_id>/ dir; the second overwrote the first. Stage per-LINK now (external/<post_id>/<link_id>/) so each file keeps its path. - The duplicate_hash/duplicate_phash branch unlinked `f` unconditionally. Make it path-safe: only unlink when `f` is NOT the existing record's canonical file. Plus an operator-triggered orphan-repair maintenance task (prune_missing_file_records_task) to clean up records already orphaned by the bug: scans ImageRecords, deletes those whose file is gone (cascade), with an NFS-stall guard that aborts without deleting if a large sample is mostly missing. Wired through POST /api/admin/maintenance/prune-missing-files and a MissingFileRepairCard in the Maintenance panel. Tests: refetch-same-link keeps the canonical file; orphan repair deletes only real orphans and aborts on the mostly-missing guard. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -14,9 +14,11 @@ from __future__ import annotations
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..models import ImageRecord
|
||||
from ..services import cleanup_service
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
@@ -41,6 +43,85 @@ def delete_artist_cascade_task(self, *, artist_id: int) -> dict:
|
||||
)
|
||||
|
||||
|
||||
# Orphan repair (#859). Safety guard: an NFS/filesystem stall makes EVERY file
|
||||
# look missing — never delete records en masse on that basis. If a non-trivial
|
||||
# sample comes back mostly missing, ABORT without deleting (assume the FS is
|
||||
# unhealthy, not that the library evaporated). Operator-triggered ONLY — NOT a
|
||||
# periodic sweep, precisely to avoid an unattended run firing during an NFS blip.
|
||||
_ORPHAN_MIN_SAMPLE = 50
|
||||
_ORPHAN_MAX_MISSING_FRAC = 0.10
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.admin.prune_missing_file_records_task",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
|
||||
)
|
||||
def prune_missing_file_records_task(self) -> dict:
|
||||
"""Delete ImageRecords whose backing file is gone from disk (orphans — e.g.
|
||||
the external-attach unlink bug #859). Every FK to image_record is CASCADE /
|
||||
SET NULL, so a Core DELETE cleans provenance, series pages, predictions and
|
||||
tag links; leftover thumbnails are unlinked best-effort. Returns a summary.
|
||||
|
||||
Aborts WITHOUT deleting if a non-trivial sample is mostly missing (a
|
||||
filesystem/NFS stall, not real orphans) — see the guard constants above.
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
checked = 0
|
||||
missing_ids: list[int] = []
|
||||
thumbs: list[str] = []
|
||||
last_id = 0
|
||||
with SessionLocal() as session:
|
||||
while True:
|
||||
rows = session.execute(
|
||||
select(ImageRecord.id, ImageRecord.path, ImageRecord.thumbnail_path)
|
||||
.where(ImageRecord.id > last_id)
|
||||
.order_by(ImageRecord.id)
|
||||
.limit(1000)
|
||||
).all()
|
||||
if not rows:
|
||||
break
|
||||
for rid, path, thumb in rows:
|
||||
last_id = rid
|
||||
checked += 1
|
||||
if not (IMAGES_ROOT / path).exists():
|
||||
missing_ids.append(rid)
|
||||
if thumb:
|
||||
thumbs.append(thumb)
|
||||
|
||||
if not missing_ids:
|
||||
return {"checked": checked, "missing": 0, "deleted": 0}
|
||||
|
||||
frac = len(missing_ids) / checked if checked else 0.0
|
||||
if checked >= _ORPHAN_MIN_SAMPLE and frac > _ORPHAN_MAX_MISSING_FRAC:
|
||||
log.warning(
|
||||
"orphan-repair ABORTED: %d/%d (%.0f%%) records missing on disk — "
|
||||
"likely a filesystem/NFS problem, not real orphans. No deletions.",
|
||||
len(missing_ids), checked, frac * 100,
|
||||
)
|
||||
return {
|
||||
"checked": checked, "missing": len(missing_ids), "deleted": 0,
|
||||
"aborted": "too many missing — filesystem problem suspected",
|
||||
}
|
||||
|
||||
deleted = 0
|
||||
for i in range(0, len(missing_ids), 500): # keep well under psycopg's param ceiling
|
||||
chunk = missing_ids[i:i + 500]
|
||||
session.execute(delete(ImageRecord).where(ImageRecord.id.in_(chunk)))
|
||||
deleted += len(chunk)
|
||||
session.commit()
|
||||
|
||||
for t in thumbs: # cosmetic — outside the txn, never fail the repair on these
|
||||
try:
|
||||
(IMAGES_ROOT / t).unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
log.info("orphan-repair: checked=%d missing=%d deleted=%d", checked, len(missing_ids), deleted)
|
||||
return {"checked": checked, "missing": len(missing_ids), "deleted": deleted}
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.admin.bulk_delete_images_task",
|
||||
bind=True,
|
||||
|
||||
Reference in New Issue
Block a user