From 8f6547f8d7d8ecfefeee9c6e873310106a97c2b2 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 6 Jul 2026 14:15:19 -0400 Subject: [PATCH] refactor(subscriptions): remove the dead backfill dry-run preview (#1281) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PreviewDialog.vue was orphaned — nothing mounted it — so the dry-run preview was unreachable. Rather than wire it up, remove the whole chain: its unique value (a capped 3-page count of what a backfill would grab) is low, and its adjacent needs are already covered — auth validation by verify_source_credential, and actually fetching recent posts by "Check now". Operator decision 2026-07-06. Removed: - frontend: PreviewDialog.vue + sources store previewSource() - backend: POST /api/sources//preview route, download_backends.preview_source, IngestCore.preview() + its now-unused NativeIngestError import - tests: the 3 ingester preview tests Nothing else referenced the chain (verified). Shared campaign-resolution and ledger helpers stay — they're used by run()/verify_source_credential. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- backend/app/api/sources.py | 46 ------- backend/app/services/download_backends.py | 44 ------- backend/app/services/ingest_core.py | 67 ---------- .../subscriptions/PreviewDialog.vue | 114 ------------------ frontend/src/stores/sources.js | 7 -- tests/test_patreon_ingester.py | 51 -------- 6 files changed, 329 deletions(-) delete mode 100644 frontend/src/components/subscriptions/PreviewDialog.vue diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index 44227cb..2463ea2 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -230,52 +230,6 @@ async def set_backfill(source_id: int): return jsonify(record.to_dict()) -@sources_bp.route("//preview", methods=["POST"]) -async def preview_source_endpoint(source_id: int): - """Plan #708 B4: dry-run — count what a backfill WOULD download for a native - platform (Patreon today), without downloading. Walks the first few feed pages - and counts media not already in the seen/dead ledgers. Returns - {total_new, posts_scanned, pages_scanned, has_more, sample[]} or 409 + reason - (unresolvable campaign id / auth / drift). 400 for gallery-dl platforms (no - cheap dry-run — their verify is a slow --simulate).""" - from pathlib import Path - - from ..services.credential_service import CredentialService - from ..services.download_backends import preview_source, uses_native_ingester - from ..tasks._sync_engine import sync_session_factory - from .credentials import _get_crypto - - async with get_session() as session: - rec = await SourceService(session).get(source_id) - if rec is None: - return _bad("not_found", status=404) - if not uses_native_ingester(rec.platform): - return _bad( - "unsupported", - detail="Preview is only available for native-ingester platforms.", - status=400, - ) - cred = CredentialService(session, _get_crypto()) - cookies_path = await cred.get_cookies_path(rec.platform) - auth_token = await cred.get_token(rec.platform) - - # The walk + ledger reads are sync (run off the request loop); the process - # sync engine is the same one the download task uses. - result = await preview_source( - platform=rec.platform, - url=rec.url, - source_id=source_id, - config_overrides=rec.config_overrides or {}, - cookies_path=str(cookies_path) if cookies_path else None, - images_root=Path("/images"), - sync_session_factory=sync_session_factory(), - auth_token=auth_token, - ) - if "error" in result: - return _bad("preview_failed", detail=result["error"], status=409) - return jsonify(result) - - @sources_bp.route("//check", methods=["POST"]) async def check_source(source_id: int): """FC-3c: enqueue a download for this source. diff --git a/backend/app/services/download_backends.py b/backend/app/services/download_backends.py index 9d799ef..c6e1d2e 100644 --- a/backend/app/services/download_backends.py +++ b/backend/app/services/download_backends.py @@ -24,7 +24,6 @@ import asyncio from pathlib import Path from .gallery_dl import DownloadResult, ErrorType -from .native_ingest_common import NativeIngestError from .patreon_ingester import PatreonIngester from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source from .pixiv_client import user_id_from_url @@ -204,49 +203,6 @@ async def _run_native_ingester( return dl_result, resolved_campaign_id -async def preview_source( - *, - platform: str, - url: str, - source_id: int, - config_overrides: dict | None, - cookies_path: str | None, - images_root: Path, - sync_session_factory, - auth_token: str | None = None, - page_limit: int = 3, -) -> dict: - """Dry-run preview for a native platform (plan #708 B4): resolve the campaign - id, then walk a few pages counting media not already seen/dead — no download. - - Returns the preview dict (total_new / posts_scanned / pages_scanned / - has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure. - Native-only — the caller gates on `uses_native_ingester`. - """ - import asyncio - - campaign_id, _ = await _resolve_native_campaign_id( - platform, url, cookies_path, config_overrides or {} - ) - if not campaign_id: - return {"error": _campaign_resolution_error(platform, url)} - ingester = _native_ingester_cls(platform)( - images_root=images_root, - cookies_path=cookies_path, - session_factory=sync_session_factory, - auth_token=auth_token, - ) - loop = asyncio.get_running_loop() - try: - result = await loop.run_in_executor( - None, - lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit), - ) - except NativeIngestError as exc: - return {"error": f"Couldn't preview: {exc}"} - return result - - async def verify_source_credential( *, platform: str, diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 9b5d391..38f7eb7 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -577,73 +577,6 @@ class Ingester: error_type=None, error_message=None, ) - # -- preview (dry-run) ------------------------------------------------- - - def preview( - self, - source_id: int, - campaign_id: str, - *, - page_limit: int = 3, - sample_size: int = 10, - ) -> dict: - """Dry-run (plan #708 B4): walk up to `page_limit` pages and count media - NOT already in the seen/dead ledgers, WITHOUT downloading anything. - - Read-only — only the seen/dead SELECTs touch the DB (short sessions). Lets - an operator gauge "is this source worth a backfill?" cheaply. Returns: - {total_new, posts_scanned, pages_scanned, has_more, - sample: [{title, date, new}, ...]} # sample = posts with new media - A client-level failure (auth/drift) propagates to the caller. - """ - total_new = 0 - posts_scanned = 0 - pages_scanned = 0 - has_more = False - sample: list[dict] = [] - unset = object() - last_page: object = unset - # #874: same gated-post gate as run() — the preview must not count - # blurred locked-preview media as "new", or it would overstate a gated - # source's backlog (preview/apply parity, rule 93). - post_is_gated = getattr(self.client, "post_is_gated", None) - for post, included, page_cursor in self.client.iter_posts( - campaign_id, cursor=None - ): - if page_cursor != last_page: - last_page = page_cursor - pages_scanned += 1 - if pages_scanned > page_limit: - has_more = True - pages_scanned = page_limit - break - posts_scanned += 1 - if post_is_gated and post_is_gated(post): - continue - media = self.client.extract_media(post, included) - if not media: - continue - keys = [self._ledger_key(m) for m in media] - skip = self._seen_keys(source_id, keys) | self._dead_keys(source_id, keys) - new_count = sum(1 for m in media if self._ledger_key(m) not in skip) - total_new += new_count - if new_count > 0 and len(sample) < sample_size: - meta = self.client.post_meta(post) - sample.append( - { - "title": meta.get("title") or "(untitled)", - "date": meta.get("date"), - "new": new_count, - } - ) - return { - "total_new": total_new, - "posts_scanned": posts_scanned, - "pages_scanned": pages_scanned, - "has_more": has_more, - "sample": sample, - } - # -- failure mapping (adapter overrides) ------------------------------- def _failure_result(self, exc: Exception, _result) -> DownloadResult: diff --git a/frontend/src/components/subscriptions/PreviewDialog.vue b/frontend/src/components/subscriptions/PreviewDialog.vue deleted file mode 100644 index 0dac268..0000000 --- a/frontend/src/components/subscriptions/PreviewDialog.vue +++ /dev/null @@ -1,114 +0,0 @@ - - - - - diff --git a/frontend/src/stores/sources.js b/frontend/src/stores/sources.js index d169873..b1f760e 100644 --- a/frontend/src/stores/sources.js +++ b/frontend/src/stores/sources.js @@ -149,12 +149,6 @@ export const useSourcesStore = defineStore('sources', () => { return body } - // Plan #708 B4: dry-run preview — count what a backfill WOULD download - // (native platforms), without downloading. Read-only, so no cache invalidate. - async function previewSource(id) { - return await api.post(`/api/sources/${id}/preview`) - } - function sourcesByArtistGrouped() { // returns [{artist: {id,name,slug}, sources: [...]}, ...] const arr = byArtist.value.get(null) ?? [] @@ -185,7 +179,6 @@ export const useSourcesStore = defineStore('sources', () => { stopBackfill, recoverSource, recaptureSource, - previewSource, findOrCreateArtist, autocompleteArtist, reassign, loadScheduleStatus, sourcesByArtistGrouped, diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index 9e75462..b711c02 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -448,42 +448,6 @@ async def test_backfill_budget_cut_returns_partial_with_progress( assert result.cursor == "CUR2" -@pytest.mark.asyncio -async def test_preview_counts_new_media_without_downloading( - source_id, sync_engine, tmp_path, -): - """plan #708 B4: preview walks + counts media not already seen/dead, downloads - nothing, and samples only posts that have new media.""" - m1, m2, m3 = _media("p1", 1), _media("p2", 1), _media("p2", 2) - # Seed m1 as already-seen → only p2's two media are "new". - factory = sessionmaker(sync_engine, expire_on_commit=False) - with factory() as s: - s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1")) - s.commit() - client = _FakeClient([(None, [("p1", [m1]), ("p2", [m2, m3])])]) - downloader = _FakeDownloader(tmp_path) - ing = _ingester(sync_engine, tmp_path, client, downloader) - - result = ing.preview(source_id, "c1") - assert result["total_new"] == 2 # p2's m2 + m3 (m1 already seen) - assert result["posts_scanned"] == 2 - assert result["has_more"] is False - assert downloader.download_calls == 0 # dry-run — nothing downloaded - # The sample lists only posts WITH new media (p2), not the all-seen p1. - assert [row["new"] for row in result["sample"]] == [2] - - -@pytest.mark.asyncio -async def test_preview_page_limit_caps_walk(source_id, sync_engine, tmp_path): - """plan #708 B4: preview stops after page_limit pages and flags has_more.""" - pages = [(f"CUR{i}", [(f"p{i}", [_media(f"p{i}", 1)])]) for i in range(6)] - ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path)) - result = ing.preview(source_id, "c1", page_limit=2) - assert result["pages_scanned"] == 2 - assert result["posts_scanned"] == 2 # one post per page, 2 pages - assert result["has_more"] is True - - @pytest.mark.asyncio async def test_write_live_progress_updates_running_event( source_id, sync_engine, tmp_path, db, @@ -714,21 +678,6 @@ async def test_gated_post_skipped_entirely_no_media_no_record( assert _count_ledger(sync_engine, source_id) == 2 -@pytest.mark.asyncio -async def test_preview_excludes_gated_posts(source_id, sync_engine, tmp_path): - """#874 preview/apply parity (rule 93): the dry-run must not count a gated - post's blurred preview media as new, or it overstates a gated source's - backlog.""" - gm = _media("gated", 1) - am = _media("open", 1) - client = _FakeClient([(None, [("gated", [gm]), ("open", [am])])], gated={"gated"}) - ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path)) - - preview = ing.preview(source_id, "c1") - assert preview["total_new"] == 1 # only the open post - assert [s for s in preview["sample"] if s["title"] == "gated"] == [] - - @pytest.mark.asyncio async def test_recapture_does_not_refetch_seen_media_missing_from_disk( source_id, sync_engine, tmp_path,