From f897e2534b751405b8466d32554a60bdcddae9d0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 15 Jun 2026 00:42:14 -0400 Subject: [PATCH 1/2] feat(posts): full-width body for image-less posts (drop dead 'no images' box) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Text-only Patreon posts (WIP/announcement/poll — the bulk of a creator's feed) rendered a big empty 'No images attached to this post' placeholder taking half the card. Render the media column only when the post HAS images; image-less posts let the title + body span the full width. Removes the now-dead PostEmptyThumbs. Co-Authored-By: Claude Opus 4.8 --- frontend/src/components/posts/PostCard.vue | 55 +++++++++---------- .../src/components/posts/PostEmptyThumbs.vue | 35 ------------ 2 files changed, 27 insertions(+), 63 deletions(-) delete mode 100644 frontend/src/components/posts/PostEmptyThumbs.vue diff --git a/frontend/src/components/posts/PostCard.vue b/frontend/src/components/posts/PostCard.vue index 29f15f9..9b2388d 100644 --- a/frontend/src/components/posts/PostCard.vue +++ b/frontend/src/components/posts/PostCard.vue @@ -25,36 +25,36 @@
-
- - + +
@@ -128,7 +128,6 @@ import { RouterLink } from 'vue-router' import { useModalStore } from '../../stores/modal.js' import { usePostsStore } from '../../stores/posts.js' import { toPlainText } from '../../utils/htmlSanitize.js' -import PostEmptyThumbs from './PostEmptyThumbs.vue' import PostSeriesMenu from './PostSeriesMenu.vue' const props = defineProps({ diff --git a/frontend/src/components/posts/PostEmptyThumbs.vue b/frontend/src/components/posts/PostEmptyThumbs.vue deleted file mode 100644 index 9927424..0000000 --- a/frontend/src/components/posts/PostEmptyThumbs.vue +++ /dev/null @@ -1,35 +0,0 @@ - - - - - From 949c9abcc6a129b8438df9b901487358e5900804 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 15 Jun 2026 01:48:38 -0400 Subject: [PATCH 2/2] fix(external): path-safe unlink + per-link staging + orphan repair (#859) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit External downloads import IN PLACE, so the post-attach dedup-skip unlink could delete a file that IS an ImageRecord's backing file — orphaning the record and 404-ing on playback. Two sources of that: - Two links on the same post (same film from mega + gdrive) emitted the same filename into one external// dir; the second overwrote the first. Stage per-LINK now (external///) so each file keeps its path. - The duplicate_hash/duplicate_phash branch unlinked `f` unconditionally. Make it path-safe: only unlink when `f` is NOT the existing record's canonical file. Plus an operator-triggered orphan-repair maintenance task (prune_missing_file_records_task) to clean up records already orphaned by the bug: scans ImageRecords, deletes those whose file is gone (cascade), with an NFS-stall guard that aborts without deleting if a large sample is mostly missing. Wired through POST /api/admin/maintenance/prune-missing-files and a MissingFileRepairCard in the Maintenance panel. Tests: refetch-same-link keeps the canonical file; orphan repair deletes only real orphans and aborts on the mostly-missing guard. Co-Authored-By: Claude Opus 4.8 --- backend/app/api/admin.py | 13 +++ backend/app/tasks/admin.py | 81 +++++++++++++++++++ backend/app/tasks/external.py | 30 ++++++- .../components/settings/MaintenancePanel.vue | 2 + .../settings/MissingFileRepairCard.vue | 47 +++++++++++ tests/test_external_worker.py | 52 +++++++++++- tests/test_tasks_admin.py | 74 +++++++++++++++++ 7 files changed, 295 insertions(+), 4 deletions(-) create mode 100644 frontend/src/components/settings/MissingFileRepairCard.vue diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index 0503331..f366394 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -348,3 +348,16 @@ async def trigger_reextract_archives(): async_result = reextract_archive_attachments_task.delay() return jsonify({"task_id": async_result.id, "status": "queued"}), 202 + + +@admin_bp.route("/maintenance/prune-missing-files", methods=["POST"]) +async def trigger_prune_missing_files(): + """Operator-triggered orphan repair (#859): delete ImageRecords whose backing + file is gone from disk (e.g. left by the external-attach unlink bug), so they + stop 404-ing on playback. The task aborts WITHOUT deleting if a large fraction + of files look missing (a filesystem/NFS stall). Maintenance queue; + operator-triggered only — never an unattended sweep.""" + from ..tasks.admin import prune_missing_file_records_task + + async_result = prune_missing_file_records_task.delay() + return jsonify({"task_id": async_result.id, "status": "queued"}), 202 diff --git a/backend/app/tasks/admin.py b/backend/app/tasks/admin.py index c6a1b08..c9b7ca8 100644 --- a/backend/app/tasks/admin.py +++ b/backend/app/tasks/admin.py @@ -14,9 +14,11 @@ from __future__ import annotations import logging from pathlib import Path +from sqlalchemy import delete, select from sqlalchemy.exc import DBAPIError, OperationalError from ..celery_app import celery +from ..models import ImageRecord from ..services import cleanup_service from ._sync_engine import sync_session_factory as _sync_session_factory @@ -41,6 +43,85 @@ def delete_artist_cascade_task(self, *, artist_id: int) -> dict: ) +# Orphan repair (#859). Safety guard: an NFS/filesystem stall makes EVERY file +# look missing — never delete records en masse on that basis. If a non-trivial +# sample comes back mostly missing, ABORT without deleting (assume the FS is +# unhealthy, not that the library evaporated). Operator-triggered ONLY — NOT a +# periodic sweep, precisely to avoid an unattended run firing during an NFS blip. +_ORPHAN_MIN_SAMPLE = 50 +_ORPHAN_MAX_MISSING_FRAC = 0.10 + + +@celery.task( + name="backend.app.tasks.admin.prune_missing_file_records_task", + bind=True, + autoretry_for=(OperationalError, DBAPIError), + retry_backoff=15, retry_backoff_max=180, max_retries=1, + soft_time_limit=1800, time_limit=2400, # 30 min / 40 min +) +def prune_missing_file_records_task(self) -> dict: + """Delete ImageRecords whose backing file is gone from disk (orphans — e.g. + the external-attach unlink bug #859). Every FK to image_record is CASCADE / + SET NULL, so a Core DELETE cleans provenance, series pages, predictions and + tag links; leftover thumbnails are unlinked best-effort. Returns a summary. + + Aborts WITHOUT deleting if a non-trivial sample is mostly missing (a + filesystem/NFS stall, not real orphans) — see the guard constants above. + """ + SessionLocal = _sync_session_factory() + checked = 0 + missing_ids: list[int] = [] + thumbs: list[str] = [] + last_id = 0 + with SessionLocal() as session: + while True: + rows = session.execute( + select(ImageRecord.id, ImageRecord.path, ImageRecord.thumbnail_path) + .where(ImageRecord.id > last_id) + .order_by(ImageRecord.id) + .limit(1000) + ).all() + if not rows: + break + for rid, path, thumb in rows: + last_id = rid + checked += 1 + if not (IMAGES_ROOT / path).exists(): + missing_ids.append(rid) + if thumb: + thumbs.append(thumb) + + if not missing_ids: + return {"checked": checked, "missing": 0, "deleted": 0} + + frac = len(missing_ids) / checked if checked else 0.0 + if checked >= _ORPHAN_MIN_SAMPLE and frac > _ORPHAN_MAX_MISSING_FRAC: + log.warning( + "orphan-repair ABORTED: %d/%d (%.0f%%) records missing on disk — " + "likely a filesystem/NFS problem, not real orphans. No deletions.", + len(missing_ids), checked, frac * 100, + ) + return { + "checked": checked, "missing": len(missing_ids), "deleted": 0, + "aborted": "too many missing — filesystem problem suspected", + } + + deleted = 0 + for i in range(0, len(missing_ids), 500): # keep well under psycopg's param ceiling + chunk = missing_ids[i:i + 500] + session.execute(delete(ImageRecord).where(ImageRecord.id.in_(chunk))) + deleted += len(chunk) + session.commit() + + for t in thumbs: # cosmetic — outside the txn, never fail the repair on these + try: + (IMAGES_ROOT / t).unlink(missing_ok=True) + except OSError: + pass + log.info("orphan-repair: checked=%d missing=%d deleted=%d", checked, len(missing_ids), deleted) + return {"checked": checked, "missing": len(missing_ids), "deleted": deleted} + + @celery.task( name="backend.app.tasks.admin.bulk_delete_images_task", bind=True, diff --git a/backend/app/tasks/external.py b/backend/app/tasks/external.py index e5f243d..efbb028 100644 --- a/backend/app/tasks/external.py +++ b/backend/app/tasks/external.py @@ -28,7 +28,7 @@ from sqlalchemy import delete, select, update from ..celery_app import celery from ..config import get_config -from ..models import Artist, ExternalLink, ImportSettings, Post, Source +from ..models import Artist, ExternalLink, ImageRecord, ImportSettings, Post, Source from ..services.external_fetch import fetch_external from ..services.importer import Importer from ..services.thumbnailer import Thumbnailer @@ -167,7 +167,15 @@ def fetch_external_link(self, link_id: int, _serialize_waits: int = 0) -> dict: return {"link_id": link_id, "requeued": "host busy"} started = time.monotonic() - post_dir = IMAGES_ROOT / artist.slug / platform / "external" / str(post.external_post_id) + # Per-LINK staging dir (not just per-post): two links on the same post (e.g. + # the same film from mega + gdrive) can emit the same filename — sharing one + # external// dir let the second overwrite the first's file, then the + # dedup-skip unlink orphaned a record → 404 on playback. Isolating per link + # keeps each link's file at its own path. (#859) + post_dir = ( + IMAGES_ROOT / artist.slug / platform / "external" + / str(post.external_post_id) / str(link_id) + ) try: result = fetch_external(host, url, post_dir, timeout=_FETCH_TIMEOUT) with SessionLocal() as session: @@ -292,7 +300,23 @@ def _route_files(session, files, post, platform, artist, source) -> list[int]: "extdl skipped: post=%s file=%s reason=%s", post.id, f.name, reason, ) if reason in ("duplicate_hash", "duplicate_phash"): - f.unlink(missing_ok=True) # canonical copy already in the library + # Path-safe unlink: external files are imported IN PLACE, so `f` + # can BE the canonical record's backing file (same content + # re-fetched / a colliding name). Only delete `f` when it is NOT + # the existing record's file — otherwise we orphan the record and + # playback 404s. (#859) + canonical = None + if result.image_id is not None: + rec = session.get(ImageRecord, result.image_id) + if rec is not None: + canonical = (IMAGES_ROOT / rec.path).resolve() + if canonical != f.resolve(): + f.unlink(missing_ok=True) # a redundant copy — safe to drop + else: + log.info( + "extdl keep: %s IS record %s's canonical file — not unlinking", + f.name, result.image_id, + ) elif result.status == "failed": log.warning( "extdl import FAILED: post=%s file=%s error=%s", diff --git a/frontend/src/components/settings/MaintenancePanel.vue b/frontend/src/components/settings/MaintenancePanel.vue index 76b85c5..32c85fd 100644 --- a/frontend/src/components/settings/MaintenancePanel.vue +++ b/frontend/src/components/settings/MaintenancePanel.vue @@ -16,6 +16,7 @@ + + + Repair missing-file records + +

+ Scans every image/video record and removes any whose underlying file is + gone from disk — orphaned records that otherwise 404 when you open the + post. Safe: it aborts without deleting if a large share of + files look missing (which means a storage/NFS hiccup, not real orphans). + Run it only when storage is healthy. +

+ + mdi-file-remove-outline Repair missing-file records + + Queued ✓ + + + + + + diff --git a/tests/test_external_worker.py b/tests/test_external_worker.py index 7bde94e..0db0029 100644 --- a/tests/test_external_worker.py +++ b/tests/test_external_worker.py @@ -5,13 +5,14 @@ import zipfile import pytest from PIL import Image -from sqlalchemy import func, select +from sqlalchemy import func, select, update import backend.app.tasks.external as ext from backend.app.models import ( Artist, ExternalLink, ImageProvenance, + ImageRecord, Post, PostAttachment, Source, @@ -102,6 +103,55 @@ def test_fetch_external_link_downloads_and_attaches(db_sync, tmp_path, monkeypat assert len(atts) == 1 +def test_refetch_same_link_keeps_canonical_file(db_sync, tmp_path, monkeypatch): + """#859: re-fetching the SAME link (identical bytes into the same per-link + staging dir) must NOT unlink the canonical file out from under its record. + The first fetch imports the image in place; the second dedups on sha256, and + the path-safe unlink recognises the file IS the record's backing file and + keeps it — so the post never 404s on playback.""" + _, link = _seed(db_sync, host="pixeldrain") + link_id = link.id + + monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path) + monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis()) + + def fake_fetch(host, url, dest_dir, *, timeout, should_stop=lambda: False): + dest_dir.mkdir(parents=True, exist_ok=True) + f = dest_dir / "clip.jpg" # art → ImageRecord (imported in place) + f.write_bytes(_structured_jpeg_bytes()) + return FetchResult(files=[f], bytes=f.stat().st_size) + + monkeypatch.setattr(ext, "fetch_external", fake_fetch) + + import backend.app.tasks.ml as ml_mod + import backend.app.tasks.thumbnail as thumb_mod + monkeypatch.setattr(ml_mod.tag_and_embed, "delay", lambda i: None) + monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda i: None) + + out = ext.fetch_external_link(link_id) + assert out["images"] == 1 + + db_sync.expire_all() + rec = db_sync.execute(select(ImageRecord)).scalars().one() + canonical = tmp_path / rec.path # absolute path collapses the join + assert canonical.exists() + + # Re-arm the SAME link and fetch again — same staging dir, identical bytes. + db_sync.execute( + update(ExternalLink).where(ExternalLink.id == link_id) + .values(status="pending", attempts=0, completed_at=None) + ) + db_sync.commit() + + out2 = ext.fetch_external_link(link_id) + assert out2["images"] == 0 # deduped, no new record + + db_sync.expire_all() + # Exactly one record still, and its file survived the dedup unlink (the bug). + assert db_sync.execute(select(func.count(ImageRecord.id))).scalar() == 1 + assert canonical.exists() + + def test_fetch_external_link_records_failure(db_sync, tmp_path, monkeypatch): _, link = _seed(db_sync) link_id = link.id diff --git a/tests/test_tasks_admin.py b/tests/test_tasks_admin.py index 948dd39..4d7ac77 100644 --- a/tests/test_tasks_admin.py +++ b/tests/test_tasks_admin.py @@ -141,6 +141,80 @@ async def test_bulk_delete_images_task_removes_listed_images( assert surviving == 0 +# --- prune_missing_file_records_task (#859 orphan repair) ----------- + + +def _img(db_sync, artist, *, path, sha): + img = ImageRecord( + artist_id=artist.id, path=str(path), + sha256=sha, size_bytes=10, mime="image/jpeg", + origin="downloaded", + ) + db_sync.add(img) + db_sync.flush() + return img.id + + +@pytest.mark.asyncio +async def test_prune_missing_file_records_deletes_only_orphans(db_sync, tmp_path): + """Records whose backing file is gone get deleted; records whose file is + present survive. Below the abort sample size, fraction is irrelevant.""" + from backend.app.tasks.admin import prune_missing_file_records_task + + a = Artist(name="Orph", slug="orph") + db_sync.add(a) + db_sync.flush() + + present = [] + for i in range(3): + f = tmp_path / f"present{i}.jpg" + f.write_bytes(b"real") + present.append(_img(db_sync, a, path=f, sha=f"{i:064x}")) + missing = [ + _img(db_sync, a, path=tmp_path / f"gone{i}.jpg", sha=f"f{i:063x}") + for i in range(2) + ] + db_sync.commit() + + result = prune_missing_file_records_task.delay().get() + assert result["checked"] == 5 + assert result["missing"] == 2 + assert result["deleted"] == 2 + + db_sync.expire_all() + surviving = db_sync.execute( + select(ImageRecord.id).where(ImageRecord.id.in_(present + missing)) + ).scalars().all() + assert set(surviving) == set(present) + + +@pytest.mark.asyncio +async def test_prune_missing_file_records_aborts_when_mostly_missing(db_sync, tmp_path): + """NFS-stall guard: a large sample that's mostly missing means the + filesystem is unhealthy, not that the library evaporated — abort, delete + nothing.""" + from backend.app.tasks.admin import prune_missing_file_records_task + + a = Artist(name="Stall", slug="stall") + db_sync.add(a) + db_sync.flush() + ids = [ + _img(db_sync, a, path=tmp_path / f"phantom{i}.jpg", sha=f"{i:064x}") + for i in range(55) # >= _ORPHAN_MIN_SAMPLE, all missing -> frac 1.0 + ] + db_sync.commit() + + result = prune_missing_file_records_task.delay().get() + assert result["deleted"] == 0 + assert "aborted" in result + + db_sync.expire_all() + surviving = db_sync.execute( + select(func.count(ImageRecord.id)).where(ImageRecord.id.in_(ids)) + ).scalar_one() + assert surviving == 55 + + @pytest.mark.asyncio async def test_bulk_delete_images_task_idempotent_on_empty_list(db_sync): from backend.app.tasks.admin import bulk_delete_images_task