diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index 19d47ce..d5871e1 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -191,6 +191,50 @@ async def set_backfill(source_id: int): return jsonify(record.to_dict()) +@sources_bp.route("//preview", methods=["POST"]) +async def preview_source_endpoint(source_id: int): + """Plan #708 B4: dry-run — count what a backfill WOULD download for a native + platform (Patreon today), without downloading. Walks the first few feed pages + and counts media not already in the seen/dead ledgers. Returns + {total_new, posts_scanned, pages_scanned, has_more, sample[]} or 409 + reason + (unresolvable campaign id / auth / drift). 400 for gallery-dl platforms (no + cheap dry-run — their verify is a slow --simulate).""" + from pathlib import Path + + from ..services.credential_service import CredentialService + from ..services.download_backends import preview_source, uses_native_ingester + from ..tasks._sync_engine import sync_session_factory + from .credentials import _get_crypto + + async with get_session() as session: + rec = await SourceService(session).get(source_id) + if rec is None: + return _bad("not_found", status=404) + if not uses_native_ingester(rec.platform): + return _bad( + "unsupported", + detail="Preview is only available for native-ingester platforms.", + status=400, + ) + cred = CredentialService(session, _get_crypto()) + cookies_path = await cred.get_cookies_path(rec.platform) + + # The walk + ledger reads are sync (run off the request loop); the process + # sync engine is the same one the download task uses. + result = await preview_source( + platform=rec.platform, + url=rec.url, + source_id=source_id, + config_overrides=rec.config_overrides or {}, + cookies_path=str(cookies_path) if cookies_path else None, + images_root=Path("/images"), + sync_session_factory=sync_session_factory(), + ) + if "error" in result: + return _bad("preview_failed", detail=result["error"], status=409) + return jsonify(result) + + @sources_bp.route("//check", methods=["POST"]) async def check_source(source_id: int): """FC-3c: enqueue a download for this source. diff --git a/backend/app/services/download_backends.py b/backend/app/services/download_backends.py index 89b295a..02095f8 100644 --- a/backend/app/services/download_backends.py +++ b/backend/app/services/download_backends.py @@ -144,6 +144,54 @@ async def _run_native_ingester( return dl_result, resolved_campaign_id +async def preview_source( + *, + platform: str, + url: str, + source_id: int, + config_overrides: dict | None, + cookies_path: str | None, + images_root: Path, + sync_session_factory, + page_limit: int = 3, +) -> dict: + """Dry-run preview for a native platform (plan #708 B4): resolve the campaign + id, then walk a few pages counting media not already seen/dead — no download. + + Returns the preview dict (total_new / posts_scanned / pages_scanned / + has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure. + Native-only — the caller gates on `uses_native_ingester`. + """ + import asyncio + + from .patreon_client import PatreonAPIError + + campaign_id, _ = await resolve_campaign_id_for_source( + url, cookies_path, config_overrides or {} + ) + if not campaign_id: + return { + "error": ( + "Couldn't resolve the campaign id from the source URL " + "(cookies expired, or the creator moved/renamed?)." + ) + } + ingester = PatreonIngester( + images_root=images_root, + cookies_path=cookies_path, + session_factory=sync_session_factory, + ) + loop = asyncio.get_running_loop() + try: + result = await loop.run_in_executor( + None, + lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit), + ) + except PatreonAPIError as exc: + return {"error": f"Couldn't preview: {exc}"} + return result + + async def verify_source_credential( *, platform: str, diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 7f7dcab..2b5286c 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -361,6 +361,67 @@ class Ingester: error_type=None, error_message=None, ) + # -- preview (dry-run) ------------------------------------------------- + + def preview( + self, + source_id: int, + campaign_id: str, + *, + page_limit: int = 3, + sample_size: int = 10, + ) -> dict: + """Dry-run (plan #708 B4): walk up to `page_limit` pages and count media + NOT already in the seen/dead ledgers, WITHOUT downloading anything. + + Read-only — only the seen/dead SELECTs touch the DB (short sessions). Lets + an operator gauge "is this source worth a backfill?" cheaply. Returns: + {total_new, posts_scanned, pages_scanned, has_more, + sample: [{title, date, new}, ...]} # sample = posts with new media + A client-level failure (auth/drift) propagates to the caller. + """ + total_new = 0 + posts_scanned = 0 + pages_scanned = 0 + has_more = False + sample: list[dict] = [] + unset = object() + last_page: object = unset + for post, included, page_cursor in self.client.iter_posts( + campaign_id, cursor=None + ): + if page_cursor != last_page: + last_page = page_cursor + pages_scanned += 1 + if pages_scanned > page_limit: + has_more = True + pages_scanned = page_limit + break + posts_scanned += 1 + media = self.client.extract_media(post, included) + if not media: + continue + keys = [self._ledger_key(m) for m in media] + skip = self._seen_keys(source_id, keys) | self._dead_keys(source_id, keys) + new_count = sum(1 for m in media if self._ledger_key(m) not in skip) + total_new += new_count + if new_count > 0 and len(sample) < sample_size: + meta = self.client.post_meta(post) + sample.append( + { + "title": meta.get("title") or "(untitled)", + "date": meta.get("date"), + "new": new_count, + } + ) + return { + "total_new": total_new, + "posts_scanned": posts_scanned, + "pages_scanned": pages_scanned, + "has_more": has_more, + "sample": sample, + } + # -- failure mapping (adapter overrides) ------------------------------- def _failure_result(self, exc: Exception, _result) -> DownloadResult: diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index 69846db..98d8134 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -520,6 +520,18 @@ class PatreonClient: return _dedup_by_filehash(items) + @staticmethod + def post_meta(post: dict) -> dict: + """Title + published date for a post — for the preview sample (plan #708 + B4). Part of the client contract `ingest_core.Ingester.preview` calls.""" + attrs = post.get("attributes") or {} + title = attrs.get("title") + published = attrs.get("published_at") + return { + "title": title if isinstance(title, str) else None, + "date": published if isinstance(published, str) else None, + } + # -- iteration --------------------------------------------------------- def iter_posts( diff --git a/frontend/src/components/subscriptions/PreviewDialog.vue b/frontend/src/components/subscriptions/PreviewDialog.vue new file mode 100644 index 0000000..7639526 --- /dev/null +++ b/frontend/src/components/subscriptions/PreviewDialog.vue @@ -0,0 +1,113 @@ + + + + + diff --git a/frontend/src/components/subscriptions/SourceCard.vue b/frontend/src/components/subscriptions/SourceCard.vue index c0c092f..0e0617b 100644 --- a/frontend/src/components/subscriptions/SourceCard.vue +++ b/frontend/src/components/subscriptions/SourceCard.vue @@ -61,6 +61,16 @@ : 'Backfill full history' }} + + mdi-eye-outline + + Preview — count what a backfill would download (no download) + + + + mdi-eye-outline + + Preview — count what a backfill would download (no download) + + @@ -253,6 +254,7 @@ @check="onCheck" @backfill="onBackfill" @recover="onRecover" + @preview="onPreview" />
No sources yet. Tap + to add one. @@ -268,6 +270,11 @@ @saved="onSourceSaved" /> +
@@ -284,6 +291,7 @@ import SourceCard from './SourceCard.vue' import SourceHealthDot from './SourceHealthDot.vue' import SourceFormDialog from './SourceFormDialog.vue' import ArtistCreateDialog from './ArtistCreateDialog.vue' +import PreviewDialog from './PreviewDialog.vue' import PlatformChip from './PlatformChip.vue' import SchedulerStatusBar from './SchedulerStatusBar.vue' import { formatRelative } from '../../utils/date.js' @@ -334,6 +342,8 @@ const showSourceDialog = ref(false) const editingSource = ref(null) const editingArtist = ref(null) const showArtistDialog = ref(false) +const showPreviewDialog = ref(false) +const previewTarget = ref(null) const artistFilter = computed(() => { const raw = route.query.artist_id @@ -572,6 +582,18 @@ async function onRecover(source) { } } +// Plan #708 B4: open the dry-run preview dialog (it self-fetches on open). +function onPreview(source) { + previewTarget.value = source + showPreviewDialog.value = true +} + +// "Start backfill" from inside the preview dialog → arm it + close. +async function onPreviewBackfill(source) { + showPreviewDialog.value = false + await onBackfill(source) +} + async function checkAll(group) { let ok = 0 let conflict = 0 diff --git a/frontend/src/stores/sources.js b/frontend/src/stores/sources.js index e7916bc..f16ad8e 100644 --- a/frontend/src/stores/sources.js +++ b/frontend/src/stores/sources.js @@ -107,6 +107,12 @@ export const useSourcesStore = defineStore('sources', () => { return body } + // Plan #708 B4: dry-run preview — count what a backfill WOULD download + // (native platforms), without downloading. Read-only, so no cache invalidate. + async function previewSource(id) { + return await api.post(`/api/sources/${id}/preview`) + } + function sourcesByArtistGrouped() { // returns [{artist: {id,name,slug}, sources: [...]}, ...] const arr = byArtist.value.get(null) ?? [] @@ -136,6 +142,7 @@ export const useSourcesStore = defineStore('sources', () => { startBackfill, stopBackfill, recoverSource, + previewSource, findOrCreateArtist, autocompleteArtist, loadScheduleStatus, sourcesByArtistGrouped, diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index 57a491b..95da4bd 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -65,6 +65,9 @@ class _FakeClient: def extract_media(self, post, included_index): return post["_media"] + def post_meta(self, post): + return {"title": post.get("id"), "date": None} + class _FakeDownloader: """Stub PatreonDownloader. Honors the injected is_seen (tier-1); media in @@ -382,6 +385,42 @@ async def test_backfill_budget_cut_returns_partial_with_progress( assert result.cursor == "CUR2" +@pytest.mark.asyncio +async def test_preview_counts_new_media_without_downloading( + source_id, sync_engine, tmp_path, +): + """plan #708 B4: preview walks + counts media not already seen/dead, downloads + nothing, and samples only posts that have new media.""" + m1, m2, m3 = _media("p1", 1), _media("p2", 1), _media("p2", 2) + # Seed m1 as already-seen → only p2's two media are "new". + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1")) + s.commit() + client = _FakeClient([(None, [("p1", [m1]), ("p2", [m2, m3])])]) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.preview(source_id, "c1") + assert result["total_new"] == 2 # p2's m2 + m3 (m1 already seen) + assert result["posts_scanned"] == 2 + assert result["has_more"] is False + assert downloader.download_calls == 0 # dry-run — nothing downloaded + # The sample lists only posts WITH new media (p2), not the all-seen p1. + assert [row["new"] for row in result["sample"]] == [2] + + +@pytest.mark.asyncio +async def test_preview_page_limit_caps_walk(source_id, sync_engine, tmp_path): + """plan #708 B4: preview stops after page_limit pages and flags has_more.""" + pages = [(f"CUR{i}", [(f"p{i}", [_media(f"p{i}", 1)])]) for i in range(6)] + ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path)) + result = ing.preview(source_id, "c1", page_limit=2) + assert result["pages_scanned"] == 2 + assert result["posts_scanned"] == 2 # one post per page, 2 pages + assert result["has_more"] is True + + @pytest.mark.asyncio async def test_still_running_reads_backfill_state(source_id, sync_engine, tmp_path, db): """plan #708 B4: _still_running reflects the source's `_backfill_state` — the