fix(external): path-safe unlink + per-link staging + orphan repair (#859)
External downloads import IN PLACE, so the post-attach dedup-skip unlink could delete a file that IS an ImageRecord's backing file — orphaning the record and 404-ing on playback. Two sources of that: - Two links on the same post (same film from mega + gdrive) emitted the same filename into one external/<post_id>/ dir; the second overwrote the first. Stage per-LINK now (external/<post_id>/<link_id>/) so each file keeps its path. - The duplicate_hash/duplicate_phash branch unlinked `f` unconditionally. Make it path-safe: only unlink when `f` is NOT the existing record's canonical file. Plus an operator-triggered orphan-repair maintenance task (prune_missing_file_records_task) to clean up records already orphaned by the bug: scans ImageRecords, deletes those whose file is gone (cascade), with an NFS-stall guard that aborts without deleting if a large sample is mostly missing. Wired through POST /api/admin/maintenance/prune-missing-files and a MissingFileRepairCard in the Maintenance panel. Tests: refetch-same-link keeps the canonical file; orphan repair deletes only real orphans and aborts on the mostly-missing guard. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -28,7 +28,7 @@ from sqlalchemy import delete, select, update
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..config import get_config
|
||||
from ..models import Artist, ExternalLink, ImportSettings, Post, Source
|
||||
from ..models import Artist, ExternalLink, ImageRecord, ImportSettings, Post, Source
|
||||
from ..services.external_fetch import fetch_external
|
||||
from ..services.importer import Importer
|
||||
from ..services.thumbnailer import Thumbnailer
|
||||
@@ -167,7 +167,15 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict:
|
||||
return {"link_id": link_id, "requeued": "host busy"}
|
||||
|
||||
started = time.monotonic()
|
||||
post_dir = IMAGES_ROOT / artist.slug / platform / "external" / str(post.external_post_id)
|
||||
# Per-LINK staging dir (not just per-post): two links on the same post (e.g.
|
||||
# the same film from mega + gdrive) can emit the same filename — sharing one
|
||||
# external/<post_id>/ dir let the second overwrite the first's file, then the
|
||||
# dedup-skip unlink orphaned a record → 404 on playback. Isolating per link
|
||||
# keeps each link's file at its own path. (#859)
|
||||
post_dir = (
|
||||
IMAGES_ROOT / artist.slug / platform / "external"
|
||||
/ str(post.external_post_id) / str(link_id)
|
||||
)
|
||||
try:
|
||||
result = fetch_external(host, url, post_dir, timeout=_FETCH_TIMEOUT)
|
||||
with SessionLocal() as session:
|
||||
@@ -292,7 +300,23 @@ def _route_files(session, files, post, platform, artist, source) -> list[int]:
|
||||
"extdl skipped: post=%s file=%s reason=%s", post.id, f.name, reason,
|
||||
)
|
||||
if reason in ("duplicate_hash", "duplicate_phash"):
|
||||
f.unlink(missing_ok=True) # canonical copy already in the library
|
||||
# Path-safe unlink: external files are imported IN PLACE, so `f`
|
||||
# can BE the canonical record's backing file (same content
|
||||
# re-fetched / a colliding name). Only delete `f` when it is NOT
|
||||
# the existing record's file — otherwise we orphan the record and
|
||||
# playback 404s. (#859)
|
||||
canonical = None
|
||||
if result.image_id is not None:
|
||||
rec = session.get(ImageRecord, result.image_id)
|
||||
if rec is not None:
|
||||
canonical = (IMAGES_ROOT / rec.path).resolve()
|
||||
if canonical != f.resolve():
|
||||
f.unlink(missing_ok=True) # a redundant copy — safe to drop
|
||||
else:
|
||||
log.info(
|
||||
"extdl keep: %s IS record %s's canonical file — not unlinking",
|
||||
f.name, result.image_id,
|
||||
)
|
||||
elif result.status == "failed":
|
||||
log.warning(
|
||||
"extdl import FAILED: post=%s file=%s error=%s",
|
||||
|
||||
Reference in New Issue
Block a user