diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index 55c8a3d..4e202dc 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -138,14 +138,16 @@ async def delete_source(source_id: int): @sources_bp.route("//backfill", methods=["POST"]) async def set_backfill(source_id: int): - """Plan #693/#697: start/stop a run-until-done backfill, or start a recovery. - Body: `{"action": "start" | "stop" | "recover"}` (default "start"). 'start' - walks the full post history in time-boxed chunks until it reaches the bottom - (then the source shows 'complete'); 'recover' is the same walk but bypasses - the Patreon seen-ledger to re-fetch dropped-and-deleted near-dups under the - current pHash threshold; 'stop' cancels either back to tick mode. Returns the - updated source dict (incl. backfill_state / backfill_chunks / - backfill_bypass_seen).""" + """Plan #693/#697 + #830: start/stop a backfill, or start a recovery / + recapture. Body: `{"action": "start" | "stop" | "recover" | "recapture"}` + (default "start"). 'start' walks the full post history in time-boxed chunks + until it reaches the bottom (then the source shows 'complete'); 'recover' is + the same walk but bypasses the Patreon seen-ledger to re-fetch + dropped-and-deleted near-dups under the current pHash threshold; 'recapture' + re-grabs EVERY post's body + external links and localizes on-disk inline + images WITHOUT re-downloading media; 'stop' cancels any back to tick mode. + Returns the updated source dict (incl. backfill_state / backfill_chunks / + backfill_bypass_seen / backfill_recapture).""" from pathlib import Path from ..services.credential_service import CredentialService @@ -157,10 +159,10 @@ async def set_backfill(source_id: int): payload = await request.get_json(silent=True) or {} action = payload.get("action", "start") - if action not in ("start", "stop", "recover"): + if action not in ("start", "stop", "recover", "recapture"): return _bad( "invalid_action", - detail="action must be 'start', 'stop', or 'recover'", + detail="action must be 'start', 'stop', 'recover', or 'recapture'", ) # Pre-flight (plan #703 #2): before arming a deep walk on a native-ingester @@ -170,7 +172,7 @@ async def set_backfill(source_id: int): # platforms: gallery-dl verify is a slow --simulate subprocess, too heavy for # an arm action. The credential read happens in a session that's CLOSED # before the verify network call (don't hold a DB conn across the request). - if action in ("start", "recover"): + if action in ("start", "recover", "recapture"): async with get_session() as session: rec = await SourceService(session).get(source_id) if rec is None: @@ -200,6 +202,8 @@ async def set_backfill(source_id: int): record = await svc.start_backfill(source_id) elif action == "recover": record = await svc.start_recovery(source_id) + elif action == "recapture": + record = await svc.start_recapture(source_id) else: record = await svc.stop_backfill(source_id) except LookupError: diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 479013f..bbf48fb 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -111,6 +111,10 @@ class DownloadService: # `_backfill_bypass_seen` flag rides alongside the running backfill state; # download mode is "recovery" when both are set. bypass_seen = bool(overrides.get("_backfill_bypass_seen")) + # #830 recapture: re-grab post bodies/links + localize on-disk inline + # images, WITHOUT re-downloading media. Rides alongside the backfill + # state like _backfill_bypass_seen; mutually exclusive with it. + recapture = bool(overrides.get("_backfill_recapture")) if in_backfill: skip_value: bool | str = BACKFILL_SKIP_VALUE source_config.timeout = BACKFILL_CHUNK_SECONDS @@ -130,6 +134,8 @@ class DownloadService: if uses_native_ingester(ctx["platform"]): if in_backfill and bypass_seen: mode = "recovery" + elif in_backfill and recapture: + mode = "recapture" elif in_backfill: mode = "backfill" else: @@ -396,6 +402,20 @@ class DownloadService: await loop.run_in_executor(None, _upsert) + # #830 recapture: backfill source_filehash on EXISTING on-disk images so + # their post-body inline remaps to the local copy. A + # SEPARATE non-deleting channel (NOT the import list — that would unlink + # the duplicate file). Empty outside recapture mode. + for rel_str, rel_url in getattr(dl_result, "relink_source_paths", None) or []: + rel_path = Path(rel_str) + if not rel_path.exists(): # noqa: ASYNC240 + continue + + def _relink(p=rel_path, u=rel_url): + return self.importer.relink_source_filehash(p, u, artist=artist) + + await loop.run_in_executor(None, _relink) + # Kick the off-platform file-host downloader for any links this run # recorded (mega/gdrive/…). Global + idempotent (only claims pending/ # retryable rows); the beat sweep is the backstop. Lazy import dodges a @@ -508,6 +528,8 @@ class DownloadService: # plan #697: a recovery walk shares this lifecycle; clear its bypass # flag on completion so the next routine tick honors the seen-ledger. new_overrides.pop("_backfill_bypass_seen", None) + # #830: same for the recapture flag. + new_overrides.pop("_backfill_recapture", None) src.config_overrides = new_overrides src.backfill_runs_remaining = 0 return diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 274ba86..6797b89 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -153,6 +153,11 @@ class DownloadResult: # Importer.upsert_post_record (keyed on external_post_id → updates, never # doubles, the same Post a media import would create). post_record_paths: list[str] = field(default_factory=list) + # Native ingester, recapture mode only (#830): (on-disk path, CDN source_url) + # pairs for already-present media whose ImageRecord.source_filehash should be + # backfilled (inline-image localization) WITHOUT re-download or unlink. Empty + # on the gallery-dl path and outside recapture. + relink_source_paths: list[tuple[str, str]] = field(default_factory=list) stdout: str = "" stderr: str = "" return_code: int = 0 diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index c9b5abe..8cc3bd3 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -1005,6 +1005,35 @@ class Importer: return sv.strip() return None + def relink_source_filehash( + self, path: Path, source_url: str, *, artist: Artist | None = None, + ) -> bool: + """#830 recapture: backfill source_url + source_filehash on an EXISTING + on-disk image's ImageRecord (matched by sha256) so its post-body inline + `` remaps to the local copy at render time — WITHOUT + re-downloading and WITHOUT unlinking the file (unlike the import path's + duplicate-hash branch). NULL-only, mirroring _apply_sidecar: never + clobbers an already-set filehash. Returns True only when a row was + updated. No-op when the file is gone, the url has no filehash, no record + matches the bytes, or the record already carries a filehash. + """ + fh = filehash_from_url(source_url) + if not fh: + return False + try: + sha = _sha256_of(path) + except OSError: + return False + record = self.session.execute( + select(ImageRecord).where(ImageRecord.sha256 == sha) + ).scalar_one_or_none() + if record is None or record.source_filehash is not None: + return False + record.source_url = source_url + record.source_filehash = fh + self.session.commit() + return True + def _apply_sidecar( self, record: ImageRecord, diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index d83525f..8cd293b 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -105,18 +105,28 @@ class Ingester: ) -> DownloadResult: """Walk + download for one source, returning a gallery-dl-shaped result. - `mode` is "tick" | "backfill" | "recovery". Recovery bypasses the tier-1 - seen-ledger AND the dead-letter ledger (tier-2 disk still skips kept - files). The walk stops on: + `mode` is "tick" | "backfill" | "recovery" | "recapture". Recovery + bypasses the tier-1 seen-ledger AND the dead-letter ledger (tier-2 disk + still skips kept files). Recapture (#830) is the cheap "re-grab post + text" walk: it bypasses the post-record gate (so EVERY post's body + + external links are re-captured / detail-fetched) but KEEPS the media + seen-ledger — on-disk media is NOT re-downloaded, only surfaced so its + ImageRecord's source_filehash can be backfilled for inline-image + localization. The walk stops on: - budget exhaustion (time_budget_seconds) → TIMEOUT / PARTIAL - tick early-out (seen_threshold contiguous seen) → success - reaching the bottom of the feed → success (rc 0) A client-level failure (drift / auth / network) fails the whole run loud. """ bypass_seen = mode == "recovery" + recapture = mode == "recapture" + # Both recovery and recapture re-capture EVERY post's body + links — they + # bypass the post-record seen-gate (recovery via bypass_seen, recapture + # explicitly). A plain backfill stays gated (capture once per post). + recapture_records = bypass_seen or recapture # Only deep walks checkpoint their cursor mid-flight (plan #705 #6); a # tick has no resumable backfill state. - checkpoint = mode in ("backfill", "recovery") + checkpoint = mode in ("backfill", "recovery", "recapture") ledger_key = self._ledger_key # Optional seams (Patreon native ingester): capture pure-text posts that # have NO downloadable media so the artist archive is complete. Absent on @@ -129,6 +139,10 @@ class Ingester: written: list[str] = [] post_records: list[str] = [] quarantined_paths: list[str] = [] + # #830 recapture: (on-disk path, CDN source_url) pairs for already-present + # media, so phase 3 can backfill the ImageRecord's source_filehash WITHOUT + # re-downloading or unlinking the file. Empty outside recapture mode. + relink: list[tuple[str, str]] = [] downloaded = 0 errors = 0 quarantined = 0 @@ -165,6 +179,7 @@ class Ingester: quarantined_paths=list(quarantined_paths), written_paths=written, post_record_paths=list(post_records), + relink_source_paths=list(relink), stdout="\n".join(log_lines), stderr="", return_code=return_code, @@ -241,7 +256,7 @@ class Ingester: if rk is not None: pkey, ppid = rk already = ( - set() if bypass_seen + set() if recapture_records else self._seen_keys(source_id, [pkey]) ) if pkey not in already: @@ -275,6 +290,7 @@ class Ingester: outcomes = self.downloader.download_post( post, media, artist_slug, is_seen=_is_skip, should_stop=lambda: time.monotonic() - start >= time_budget_seconds, + recapture=recapture, ) to_mark: list[tuple[str, str]] = [] @@ -300,6 +316,12 @@ class Ingester: to_clear.append(key) skipped_count += 1 consecutive_seen += 1 + # #830 recapture: surface (on-disk path, CDN url) so phase + # 3 can backfill source_filehash for inline-image + # localization — a SEPARATE non-deleting channel, never the + # import list (which would unlink the file, per above). + if recapture and outcome.path is not None: + relink.append((str(outcome.path), media_item.url)) elif outcome.status == "skipped_seen": skipped_count += 1 consecutive_seen += 1 diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index 1f7443d..c142150 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -198,6 +198,7 @@ class PatreonDownloader: *, is_seen: Callable[[object], bool] = lambda m: False, should_stop: Callable[[], bool] = lambda: False, + recapture: bool = False, ) -> list[MediaOutcome]: """Download every media item of one post; return per-item outcomes. @@ -212,6 +213,11 @@ class PatreonDownloader: re-checks the budget between posts), so we honour the deadline mid-post and return the items done so far — the rest re-fetch next chunk (they were never marked seen). Bounds chunk overrun to one media download. + + `recapture` (#830): don't re-download already-present media, but DO surface + on-disk media as `skipped_disk` (with its path) even when the seen-ledger + would tier-1 skip it — so the engine can backfill source_filehash for + inline-image localization. Genuinely-missing seen media is NOT refetched. """ post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post) outcomes: list[MediaOutcome] = [] @@ -221,7 +227,10 @@ class PatreonDownloader: break try: outcomes.append( - self._download_one(post, media, post_dir, artist_slug, i, is_seen) + self._download_one( + post, media, post_dir, artist_slug, i, is_seen, + recapture=recapture, + ) ) except Exception as exc: # resilient: isolate one item's failure log.warning( @@ -243,9 +252,15 @@ class PatreonDownloader: artist_slug: str, index: int, is_seen: Callable[[object], bool], + *, + recapture: bool = False, ) -> MediaOutcome: - # tier-1: seen ledger (injected; no DB here). - if is_seen(media): + # tier-1: seen ledger (injected; no DB here). In recapture mode we DON'T + # short-circuit here — we fall through to the disk check so an on-disk + # seen file is surfaced as skipped_disk (with its path) for source_filehash + # backfill; a seen file that's NOT on disk is left alone (not refetched). + seen = is_seen(media) + if seen and not recapture: return MediaOutcome(media=media, status="skipped_seen", path=None, error=None) nn = f"{index:02d}" @@ -265,6 +280,12 @@ class PatreonDownloader: media=media, status="skipped_disk", path=existing, error=None ) + # recapture: a seen item that isn't on disk is NOT re-downloaded (that's + # recovery's job) — recapture only re-grabs post text + localizes existing + # files. Returns skipped_seen so the run-of-seen / counts stay consistent. + if seen: + return MediaOutcome(media=media, status="skipped_seen", path=None, error=None) + post_dir.mkdir(parents=True, exist_ok=True) # Pace real downloads only (the skips above already returned). plan #703. diff --git a/backend/app/services/source_service.py b/backend/app/services/source_service.py index 073b4d8..e16e2ae 100644 --- a/backend/app/services/source_service.py +++ b/backend/app/services/source_service.py @@ -70,6 +70,10 @@ class SourceRecord: # plan #697: a running deep-walk that bypasses the Patreon seen-ledger # (recovery) vs. a normal backfill. Lets the badge label it "Recovering". backfill_bypass_seen: bool + # #830: a running deep-walk in RECAPTURE mode (re-grab post bodies/links + + # localize on-disk inline images, no media re-download). Lets the badge label + # it "Recapturing". + backfill_recapture: bool # plan #704: cumulative posts processed across the walk's chunks — live # progress for the badge. backfill_posts: int @@ -94,6 +98,7 @@ class SourceRecord: "backfill_state": self.backfill_state, "backfill_chunks": self.backfill_chunks, "backfill_bypass_seen": self.backfill_bypass_seen, + "backfill_recapture": self.backfill_recapture, "backfill_posts": self.backfill_posts, } @@ -171,6 +176,7 @@ class SourceService: backfill_state=co.get("_backfill_state"), backfill_chunks=int(co.get("_backfill_chunks", 0)), backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")), + backfill_recapture=bool(co.get("_backfill_recapture")), backfill_posts=int(co.get("_backfill_posts", 0)), ) @@ -359,6 +365,34 @@ class SourceService: await self.session.commit() return await self._row_to_record(source) + async def start_recapture(self, source_id: int) -> SourceRecord: + """#830: arm a RECAPTURE walk — a backfill that re-grabs EVERY post's body + + external links (detail-fetching empty bodies) and localizes already-on- + disk inline images, WITHOUT re-downloading media. Reuses the entire #693 + backfill state machine plus a `_backfill_recapture` flag that flips + download mode to recapture. Distinct from recovery (which re-downloads the + whole source); the two flags are mutually exclusive, so arming recapture + clears bypass_seen. Clears prior cursor/chunk/stall state so it walks + fresh from the top. The flag is cleared on completion (download_service) + and on stop. Recapture is Patreon-only (the native ingester's post-record + capture); inert elsewhere. The UI gates the action to Patreon sources.""" + source = (await self.session.execute( + select(Source).where(Source.id == source_id) + )).scalar_one_or_none() + if source is None: + raise LookupError(f"source id={source_id} not found") + co = dict(source.config_overrides or {}) + co["_backfill_state"] = "running" + co["_backfill_recapture"] = True + co.pop("_backfill_bypass_seen", None) # mutually exclusive with recovery + for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks", + "_backfill_posts"): + co.pop(k, None) + source.config_overrides = co + source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS + await self.session.commit() + return await self._row_to_record(source) + async def stop_backfill(self, source_id: int) -> SourceRecord: """Plan #693: cancel an in-progress backfill — back to idle/tick mode. Clears the running state + cursor/chunk/stall bookkeeping.""" @@ -369,7 +403,8 @@ class SourceService: raise LookupError(f"source id={source_id} not found") co = dict(source.config_overrides or {}) for k in ("_backfill_state", "_backfill_cursor", "_backfill_cursor_stalls", - "_backfill_chunks", "_backfill_bypass_seen", "_backfill_posts"): + "_backfill_chunks", "_backfill_bypass_seen", "_backfill_recapture", + "_backfill_posts"): co.pop(k, None) source.config_overrides = co source.backfill_runs_remaining = 0 diff --git a/frontend/src/components/subscriptions/SourceActions.vue b/frontend/src/components/subscriptions/SourceActions.vue index bdf01df..7d23dc9 100644 --- a/frontend/src/components/subscriptions/SourceActions.vue +++ b/frontend/src/components/subscriptions/SourceActions.vue @@ -21,7 +21,7 @@ {{ running ? 'mdi-stop' : 'mdi-magnify-scan' }} {{ running - ? (recovering ? 'Stop recovery' : 'Stop backfill') + ? (recovering ? 'Stop recovery' : (recapturing ? 'Stop recapture' : 'Stop backfill')) : 'Backfill — one-time walk of the FULL post history until complete' }} @@ -38,6 +38,17 @@ under the current similarity threshold + + Recapture post text & links + + Re-grab every post's body + external links and localize inline images + already on disk — without re-downloading media + + props.source.backfill_state === 'running') const recovering = computed(() => !!props.source.backfill_bypass_seen) +const recapturing = computed(() => !!props.source.backfill_recapture) const isPatreon = computed(() => props.source.platform === 'patreon') diff --git a/frontend/src/components/subscriptions/SourceCard.vue b/frontend/src/components/subscriptions/SourceCard.vue index 0ba7f86..c4526b5 100644 --- a/frontend/src/components/subscriptions/SourceCard.vue +++ b/frontend/src/components/subscriptions/SourceCard.vue @@ -40,7 +40,7 @@ {{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling' + >{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling') }}{{ source.backfill_posts ? ` · ${source.backfill_posts} posts` : (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }} @@ -60,6 +60,7 @@ @check="$emit('check', $event)" @backfill="$emit('backfill', $event)" @recover="$emit('recover', $event)" + @recapture="$emit('recapture', $event)" @remove="$emit('remove', $event)" /> @@ -79,7 +80,7 @@ const props = defineProps({ checking: { type: Boolean, default: false }, warningThreshold: { type: Number, default: 5 }, }) -const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover']) +const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover', 'recapture']) function onToggleEnabled(value) { emit('toggle', { source: props.source, enabled: value }) diff --git a/frontend/src/components/subscriptions/SourceRow.vue b/frontend/src/components/subscriptions/SourceRow.vue index 8f737d5..10de1b0 100644 --- a/frontend/src/components/subscriptions/SourceRow.vue +++ b/frontend/src/components/subscriptions/SourceRow.vue @@ -51,7 +51,7 @@ {{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling' + >{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling') }}{{ source.backfill_posts ? ` · ${source.backfill_posts} posts` : (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }} @@ -71,6 +71,7 @@ @check="$emit('check', $event)" @backfill="$emit('backfill', $event)" @recover="$emit('recover', $event)" + @recapture="$emit('recapture', $event)" @remove="$emit('remove', $event)" /> @@ -87,7 +88,7 @@ const props = defineProps({ checking: { type: Boolean, default: false }, warningThreshold: { type: Number, default: 5 }, }) -const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover']) +const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover', 'recapture']) function onToggleEnabled(value) { emit('toggle', { source: props.source, enabled: value }) diff --git a/frontend/src/components/subscriptions/SubscriptionsTab.vue b/frontend/src/components/subscriptions/SubscriptionsTab.vue index 83bbb8c..bcadbfb 100644 --- a/frontend/src/components/subscriptions/SubscriptionsTab.vue +++ b/frontend/src/components/subscriptions/SubscriptionsTab.vue @@ -138,7 +138,7 @@ :source="item.singleSource" :checking="store.checkingIds.has(item.singleSource.id)" @check="onCheck" @backfill="onBackfill" - @recover="onRecover" @remove="removeSource" + @recover="onRecover" @recapture="onRecapture" @remove="removeSource" /> mdi-pencil @@ -214,6 +214,7 @@ @check="onCheck" @backfill="onBackfill" @recover="onRecover" + @recapture="onRecapture" /> @@ -291,6 +292,7 @@ @check="onCheck" @backfill="onBackfill" @recover="onRecover" + @recapture="onRecapture" />
No sources yet. Tap + to add one. @@ -646,6 +648,22 @@ async function onRecover(source) { } } +// #830: arm a recapture walk (Patreon-only) — re-grab every post's body + +// external links and localize on-disk inline images, without re-downloading +// media. Reuses the backfill lifecycle/badge; stop via the same Stop control. +async function onRecapture(source) { + try { + await store.recaptureSource(source.id, source.artist_id) + toast({ text: `Recapture started for ${source.artist_name}`, type: 'success' }) + // recaptureSource patches the source in place — no full reload. + } catch (e) { + toast({ + text: `Recapture start failed: ${e?.body?.detail || e?.message || e}`, + type: 'error', + }) + } +} + async function checkAll(group) { let ok = 0 let conflict = 0 diff --git a/frontend/src/stores/sources.js b/frontend/src/stores/sources.js index 0b4d132..10c16b5 100644 --- a/frontend/src/stores/sources.js +++ b/frontend/src/stores/sources.js @@ -131,6 +131,14 @@ export const useSourcesStore = defineStore('sources', () => { _patchSource(body) return body } + // #830: arm a recapture walk (Patreon-only) — re-grab every post's body + + // external links and localize on-disk inline images, WITHOUT re-downloading + // media. Shares the backfill lifecycle/badge; stop via stopBackfill. + async function recaptureSource(id, artistIdHint = null) { + const body = await api.post(`/api/sources/${id}/backfill`, { body: { action: 'recapture' } }) + _patchSource(body) + return body + } // Plan #708 B4: dry-run preview — count what a backfill WOULD download // (native platforms), without downloading. Read-only, so no cache invalidate. @@ -167,6 +175,7 @@ export const useSourcesStore = defineStore('sources', () => { startBackfill, stopBackfill, recoverSource, + recaptureSource, previewSource, findOrCreateArtist, autocompleteArtist, loadScheduleStatus, diff --git a/tests/test_api_sources.py b/tests/test_api_sources.py index e136ac2..838fbac 100644 --- a/tests/test_api_sources.py +++ b/tests/test_api_sources.py @@ -324,6 +324,32 @@ async def test_backfill_endpoint_recover_arms_bypass(client, artist, db): assert stopped_body["backfill_bypass_seen"] is False +@pytest.mark.asyncio +async def test_backfill_endpoint_recapture_arms_flag(client, artist, db): + """#830: action='recapture' arms a backfill that re-grabs post bodies/links + + localizes on-disk inline images; the response exposes backfill_recapture=True + (badge 'Recapturing') and NOT backfill_bypass_seen. Stop clears it.""" + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-recapture", enabled=True, + ) + db.add(src) + await db.commit() + sid = src.id + + resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "recapture"}) + assert resp.status_code == 200 + body = await resp.get_json() + assert body["backfill_state"] == "running" + assert body["backfill_recapture"] is True + assert body["backfill_bypass_seen"] is False + + stopped = await client.post(f"/api/sources/{sid}/backfill", json={"action": "stop"}) + stopped_body = await stopped.get_json() + assert stopped_body["backfill_state"] is None + assert stopped_body["backfill_recapture"] is False + + # --- Plan #703 #2: pre-flight credential verify on backfill/recover arm ----- diff --git a/tests/test_download_service.py b/tests/test_download_service.py index a4b4779..6f2b13f 100644 --- a/tests/test_download_service.py +++ b/tests/test_download_service.py @@ -60,8 +60,10 @@ def _make_fake_dl_result( *, success=True, written_paths=None, quarantined_paths=None, files_downloaded=0, error_type=None, error_message=None, stdout="", stderr="", cursor=None, run_stats=None, posts_processed=0, + relink_source_paths=None, ): return SimpleNamespace( + relink_source_paths=relink_source_paths or [], success=success, url="https://patreon.com/alice", artist_slug="alice", @@ -793,6 +795,31 @@ async def test_recovery_mode_selected_and_flag_cleared_on_complete( assert "_backfill_bypass_seen" not in co +@pytest.mark.asyncio +async def test_recapture_mode_selected_and_flag_cleared_on_complete( + db, db_sync, tmp_path, seed_artist_and_source, +): + """#830: `_backfill_recapture` alongside a running backfill selects recapture + mode. A clean rc=0 walk completes the shared lifecycle AND clears the + recapture flag so the next tick is a plain tick again.""" + _artist, source = seed_artist_and_source + source.config_overrides = {"_backfill_state": "running", "_backfill_recapture": True} + source.backfill_runs_remaining = 5 + await db.commit() + + svc, calls = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result( + success=True, written_paths=[], files_downloaded=0, + )) + await svc.download_source(source.id) + + assert calls[0]["mode"] == "recapture" + co = (await db.execute( + select(Source.config_overrides).where(Source.id == source.id) + )).scalar_one() + assert co.get("_backfill_state") == "complete" + assert "_backfill_recapture" not in co + + @pytest.mark.asyncio async def test_partial_error_type_maps_to_ok_status( db, db_sync, tmp_path, seed_artist_and_source, diff --git a/tests/test_patreon_downloader.py b/tests/test_patreon_downloader.py index 06d4700..e6dc8c5 100644 --- a/tests/test_patreon_downloader.py +++ b/tests/test_patreon_downloader.py @@ -206,6 +206,54 @@ def test_skip_disk(tmp_path): assert existing.read_bytes() == b"already here" # untouched +def test_recapture_surfaces_seen_on_disk_without_download(tmp_path): + """#830 recapture: a seen item that IS on disk is surfaced as skipped_disk + (with its path, for source_filehash backfill) — NOT tier-1 short-circuited, + and never re-downloaded.""" + session = _FakeSession() + dl = _downloader(tmp_path, session=session) + post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title" + post_dir.mkdir(parents=True) + existing = post_dir / "01_media1.png" + existing.write_bytes(b"already here") + + outcomes = dl.download_post( + _post(), [_img("media1.png")], "artist-x", + is_seen=lambda m: True, recapture=True, + ) + assert outcomes[0].status == "skipped_disk" + assert outcomes[0].path == existing + assert session.calls == [] # no re-download + assert existing.read_bytes() == b"already here" # untouched + + +def test_recapture_does_not_refetch_seen_missing(tmp_path): + """#830 recapture: a seen item that's NOT on disk is left alone (skipped_seen, + no download) — refetching is recovery's job, not recapture's.""" + session = _FakeSession() + dl = _downloader(tmp_path, session=session) + outcomes = dl.download_post( + _post(), [_img("media1.png")], "artist-x", + is_seen=lambda m: True, recapture=True, + ) + assert outcomes[0].status == "skipped_seen" + assert outcomes[0].path is None + assert session.calls == [] # not refetched + + +def test_recapture_still_downloads_genuinely_new(tmp_path): + """#830 recapture: media that is neither seen nor on disk still downloads — a + recapture walk is thorough, it just doesn't re-pull existing files.""" + session = _FakeSession() + dl = _downloader(tmp_path, session=session) + outcomes = dl.download_post( + _post(), [_img("media1.png")], "artist-x", + is_seen=lambda m: False, recapture=True, + ) + assert outcomes[0].status == "downloaded" + assert outcomes[0].path is not None + + def test_video_routes_to_ytdlp(tmp_path, monkeypatch): session = _FakeSession() dl = _downloader(tmp_path, session=session) diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index ed7cddb..358a7cc 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -88,16 +88,23 @@ class _FakeDownloader: self.post_records = 0 def download_post(self, post, media_items, artist_slug, *, is_seen, - should_stop=lambda: False): + should_stop=lambda: False, recapture=False): outcomes = [] for m in media_items: if should_stop(): break - if is_seen(m): + seen = is_seen(m) + # Recapture mirrors the real downloader: a seen item isn't tier-1 + # short-circuited — it falls through so an on-disk file surfaces as + # skipped_disk (for source_filehash backfill); a seen file NOT on + # disk is left alone (skipped_seen, never refetched). + if seen and not recapture: outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None)) elif _ledger_key(m) in self.on_disk: p = self.tmp_path / f"{m.post_id}_{m.filename}" outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None)) + elif seen: + outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None)) elif _ledger_key(m) in self.error: self.download_calls += 1 outcomes.append(MediaOutcome(media=m, status="error", path=None, error="boom")) @@ -581,6 +588,78 @@ async def test_backfill_recaptures_body_for_already_downloaded_post( assert downloader.post_records == 1 +def _seed_seen(sync_engine, source_id, key, post_id=None): + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + s.add(PatreonSeenMedia(source_id=source_id, filehash=key, post_id=post_id)) + s.commit() + + +@pytest.mark.asyncio +async def test_backfill_skips_already_captured_post_but_recapture_forces_it( + source_id, sync_engine, tmp_path, +): + """#830: a plain backfill GATES post-record capture on the seen-ledger, so a + post whose `post:` key is already seen is NOT recaptured (the operator's + 'Industrial Lust body missing' bug). RECAPTURE mode bypasses that gate and + re-captures the body/links even though the post key is seen — without + re-downloading the on-disk media.""" + m1 = _media("p1", 1) + # Pre-seed BOTH the media key and the synthetic post key as already seen. + _seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1") + _seed_seen(sync_engine, source_id, "post:p1", post_id="p1") + + # 1) Plain backfill: post key is seen → gate skips body recapture. + client = _FakeClient([(None, [("p1", [m1])])]) + downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)}) + ing = _ingester(sync_engine, tmp_path, client, downloader) + res_backfill = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="backfill", + ) + assert res_backfill.post_record_paths == [] # gated → not recaptured + assert downloader.post_records == 0 + + # 2) Recapture: gate bypassed → body recaptured; media NOT re-downloaded; + # the on-disk file is surfaced for source_filehash relink. + client2 = _FakeClient([(None, [("p1", [m1])])]) + downloader2 = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)}) + ing2 = _ingester(sync_engine, tmp_path, client2, downloader2) + res_recap = ing2.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="recapture", + ) + assert len(res_recap.post_record_paths) == 1 # gate bypassed → recaptured + assert downloader2.post_records == 1 + assert res_recap.files_downloaded == 0 # no media re-download + assert downloader2.download_calls == 0 + # The on-disk media is surfaced as a (path, CDN url) relink pair. + assert len(res_recap.relink_source_paths) == 1 + rel_path, rel_url = res_recap.relink_source_paths[0] + assert rel_url == m1.url + + +@pytest.mark.asyncio +async def test_recapture_does_not_refetch_seen_media_missing_from_disk( + source_id, sync_engine, tmp_path, +): + """Recapture must NOT re-download a seen item that's absent from disk (that's + recovery's job) — it returns skipped_seen and contributes no relink pair.""" + m1 = _media("p1", 1) + _seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1") + client = _FakeClient([(None, [("p1", [m1])])]) + # NOT in on_disk → seen + missing. + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="recapture", + ) + assert result.files_downloaded == 0 + assert downloader.download_calls == 0 + assert result.relink_source_paths == [] + + # --- dead-letter ledger (plan #705 #7) ------------------------------------ diff --git a/tests/test_sidecar_import.py b/tests/test_sidecar_import.py index 7a6e150..7ec05d4 100644 --- a/tests/test_sidecar_import.py +++ b/tests/test_sidecar_import.py @@ -120,6 +120,44 @@ def test_sidecar_source_url_persists_filehash(importer, import_layout): assert rec.source_filehash == "0123456789abcdef0123456789abcdef" +def test_relink_source_filehash_backfills_existing(importer, import_layout): + """#830 recapture (Part B): relink_source_filehash backfills source_url + + source_filehash on an existing on-disk image's ImageRecord (matched by + sha256) WITHOUT re-importing or unlinking the file. NULL-only — never + clobbers a filehash already set.""" + import_root, _ = import_layout + m = import_root / "Alice" / "img.jpg" + _split(m, "v") + r = importer.import_one(m) + assert r.status == "imported" + rec = importer.session.get(ImageRecord, r.image_id) + assert rec.source_filehash is None + + url = "https://cdn.test/p/0123456789ABCDEF0123456789abcdef/img.jpg" + assert importer.relink_source_filehash(m, url) is True + importer.session.refresh(rec) + assert rec.source_url == url + assert rec.source_filehash == "0123456789abcdef0123456789abcdef" + assert m.exists() # never unlinked + + # NULL-only: a second relink with a different url is a no-op. + other = "https://cdn.test/p/ffffffffffffffffffffffffffffffff/x.jpg" + assert importer.relink_source_filehash(m, other) is False + importer.session.refresh(rec) + assert rec.source_filehash == "0123456789abcdef0123456789abcdef" + + +def test_relink_source_filehash_no_match_is_noop(importer, import_layout): + """A path whose bytes match no ImageRecord (never imported) is a clean no-op, + not an error.""" + import_root, _ = import_layout + m = import_root / "Alice" / "ghost.jpg" + _split(m, "h") + assert importer.relink_source_filehash( + m, "https://cdn.test/p/0123456789abcdef0123456789abcdef/ghost.jpg" + ) is False + + def test_reimport_same_post_idempotent(importer, import_layout): import_root, _ = import_layout # Threshold 0: only an exact phash match collapses. Orthogonal splits diff --git a/tests/test_source_service.py b/tests/test_source_service.py index 7bceea8..83dfc68 100644 --- a/tests/test_source_service.py +++ b/tests/test_source_service.py @@ -267,6 +267,47 @@ async def test_start_recovery_arms_bypass_flag(db): assert "_backfill_bypass_seen" not in (co2 or {}) +@pytest.mark.asyncio +async def test_start_recapture_arms_recapture_flag(db): + """#830: start_recapture arms the backfill state machine PLUS the + _backfill_recapture flag, surfaced on the record so the badge can label it + 'Recapturing'. Mutually exclusive with recovery (clears bypass_seen); stop + clears it.""" + from backend.app.services.source_service import BACKFILL_MAX_CHUNKS + + artist = await _artist(db, "Alice") + svc = SourceService(db) + rec = await svc.create( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice", + ) + # Pre-arm recovery, then recapture must clear bypass_seen (mutual exclusion). + await svc.start_recovery(rec.id) + + updated = await svc.start_recapture(rec.id) + assert updated.backfill_state == "running" + assert updated.backfill_recapture is True + assert updated.backfill_bypass_seen is False + assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS + + co = (await db.execute( + select(Source.config_overrides).where(Source.id == rec.id) + )).scalar_one() + assert co.get("_backfill_recapture") is True + assert "_backfill_bypass_seen" not in co + + # to_dict carries the new field for the API/UI. + assert updated.to_dict()["backfill_recapture"] is True + + # Stop clears the recapture flag too. + stopped = await svc.stop_backfill(rec.id) + assert stopped.backfill_recapture is False + co2 = (await db.execute( + select(Source.config_overrides).where(Source.id == rec.id) + )).scalar_one() + assert "_backfill_recapture" not in (co2 or {}) + + @pytest.mark.asyncio async def test_start_backfill_raises_when_source_missing(db): svc = SourceService(db)