fix(external): path-safe unlink + per-link staging + orphan repair (#859)
External downloads import IN PLACE, so the post-attach dedup-skip unlink could delete a file that IS an ImageRecord's backing file — orphaning the record and 404-ing on playback. Two sources of that: - Two links on the same post (same film from mega + gdrive) emitted the same filename into one external/<post_id>/ dir; the second overwrote the first. Stage per-LINK now (external/<post_id>/<link_id>/) so each file keeps its path. - The duplicate_hash/duplicate_phash branch unlinked `f` unconditionally. Make it path-safe: only unlink when `f` is NOT the existing record's canonical file. Plus an operator-triggered orphan-repair maintenance task (prune_missing_file_records_task) to clean up records already orphaned by the bug: scans ImageRecords, deletes those whose file is gone (cascade), with an NFS-stall guard that aborts without deleting if a large sample is mostly missing. Wired through POST /api/admin/maintenance/prune-missing-files and a MissingFileRepairCard in the Maintenance panel. Tests: refetch-same-link keeps the canonical file; orphan repair deletes only real orphans and aborts on the mostly-missing guard. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -141,6 +141,80 @@ async def test_bulk_delete_images_task_removes_listed_images(
|
||||
assert surviving == 0
|
||||
|
||||
|
||||
# --- prune_missing_file_records_task (#859 orphan repair) -----------
|
||||
|
||||
|
||||
def _img(db_sync, artist, *, path, sha):
|
||||
img = ImageRecord(
|
||||
artist_id=artist.id, path=str(path),
|
||||
sha256=sha, size_bytes=10, mime="image/jpeg",
|
||||
origin="downloaded",
|
||||
)
|
||||
db_sync.add(img)
|
||||
db_sync.flush()
|
||||
return img.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prune_missing_file_records_deletes_only_orphans(db_sync, tmp_path):
|
||||
"""Records whose backing file is gone get deleted; records whose file is
|
||||
present survive. Below the abort sample size, fraction is irrelevant."""
|
||||
from backend.app.tasks.admin import prune_missing_file_records_task
|
||||
|
||||
a = Artist(name="Orph", slug="orph")
|
||||
db_sync.add(a)
|
||||
db_sync.flush()
|
||||
|
||||
present = []
|
||||
for i in range(3):
|
||||
f = tmp_path / f"present{i}.jpg"
|
||||
f.write_bytes(b"real")
|
||||
present.append(_img(db_sync, a, path=f, sha=f"{i:064x}"))
|
||||
missing = [
|
||||
_img(db_sync, a, path=tmp_path / f"gone{i}.jpg", sha=f"f{i:063x}")
|
||||
for i in range(2)
|
||||
]
|
||||
db_sync.commit()
|
||||
|
||||
result = prune_missing_file_records_task.delay().get()
|
||||
assert result["checked"] == 5
|
||||
assert result["missing"] == 2
|
||||
assert result["deleted"] == 2
|
||||
|
||||
db_sync.expire_all()
|
||||
surviving = db_sync.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.id.in_(present + missing))
|
||||
).scalars().all()
|
||||
assert set(surviving) == set(present)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prune_missing_file_records_aborts_when_mostly_missing(db_sync, tmp_path):
|
||||
"""NFS-stall guard: a large sample that's mostly missing means the
|
||||
filesystem is unhealthy, not that the library evaporated — abort, delete
|
||||
nothing."""
|
||||
from backend.app.tasks.admin import prune_missing_file_records_task
|
||||
|
||||
a = Artist(name="Stall", slug="stall")
|
||||
db_sync.add(a)
|
||||
db_sync.flush()
|
||||
ids = [
|
||||
_img(db_sync, a, path=tmp_path / f"phantom{i}.jpg", sha=f"{i:064x}")
|
||||
for i in range(55) # >= _ORPHAN_MIN_SAMPLE, all missing -> frac 1.0
|
||||
]
|
||||
db_sync.commit()
|
||||
|
||||
result = prune_missing_file_records_task.delay().get()
|
||||
assert result["deleted"] == 0
|
||||
assert "aborted" in result
|
||||
|
||||
db_sync.expire_all()
|
||||
surviving = db_sync.execute(
|
||||
select(func.count(ImageRecord.id)).where(ImageRecord.id.in_(ids))
|
||||
).scalar_one()
|
||||
assert surviving == 55
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_delete_images_task_idempotent_on_empty_list(db_sync):
|
||||
from backend.app.tasks.admin import bulk_delete_images_task
|
||||
|
||||
Reference in New Issue
Block a user