feat(patreon): phase-2 ingester integration — build step 3 (plan #697)

Branch download_service phase 2 by platform: Patreon now routes to the
native PatreonIngester (zero per-file HEADs, native cursor/resume, loud
drift detection) instead of gallery-dl; the other 5 platforms are
unchanged. The ingester returns a DownloadResult-shaped object so phase 1
(DB setup) and phase 3 (import → pHash → thumbs → ML) are untouched.

Three modes wired from config_overrides state:
  - tick: skip seen (tier-1 ledger + tier-2 disk), early-out after N
    contiguous already-have-it items.
  - backfill: full-history time-boxed chunk, cursor checkpoint via
    gallery-dl-style "Cursor: <token>" lines in stdout (reuses the #693
    lifecycle + parse_last_cursor verbatim).
  - recovery: backfill that BYPASSES the tier-1 seen-ledger so
    dropped-and-deleted near-dups get re-fetched and re-evaluated under
    the current pHash threshold. Rides the #693 state machine via a
    _backfill_bypass_seen flag, cleared on completion / stop.

The seen-ledger uses short-lived sync sessions (injected sessionmaker),
never held across the walk (avoids the connection-reaping trap). Campaign
id resolves from override, an id: URL, or a vanity lookup; unresolvable =
loud NOT_FOUND, never a silent empty success.

Tests: new test_patreon_ingester.py (modes, ledger skip/idempotency,
budget→PARTIAL, recovery bypass, tier-2 disk, drift). The patreon-oriented
download_service tests now drive the ingester branch via a stub; the
gallery-dl campaign-retry test is replaced by resolution/caching coverage.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/app/services/download_service.py

@@ -29,12 +29,14 @@ from .gallery_dl import (
     BACKFILL_CHUNK_SECONDS,
     BACKFILL_SKIP_VALUE,
     TICK_SKIP_VALUE,
+    DownloadResult,
     ErrorType,
     GalleryDLService,
     SourceConfig,
     parse_last_cursor,
 )
 from .importer import Importer
+from .patreon_ingester import PatreonIngester
 from .patreon_resolver import resolve_campaign_id
 from .platforms import auth_type_for
 from .scheduler_service import set_platform_cooldown
@@ -78,12 +80,18 @@ class DownloadService:
         gdl: GalleryDLService,
         importer: Importer,
         cred_service: CredentialService,
+        sync_session_factory=None,
     ):
         self.async_session = async_session
         self.sync_session = sync_session
         self.gdl = gdl
         self.importer = importer
         self.cred_service = cred_service
+        # Sync sessionmaker the native Patreon ingester opens SHORT-LIVED
+        # sessions from for its seen-ledger reads/writes (never one held across
+        # the multi-minute walk — see PatreonIngester). Only the patreon branch
+        # of phase 2 uses it; gallery-dl sources leave it None.
+        self.sync_session_factory = sync_session_factory
 
     async def download_source(self, source_id: int) -> int:
         """Returns DownloadEvent.id. Idempotent: in-flight events are returned as-is."""
@@ -118,6 +126,14 @@ class DownloadService:
         # Operator drives this via POST /api/sources/{id}/backfill {action}.
         overrides = ctx["config_overrides"] or {}
         in_backfill = overrides.get("_backfill_state") == "running"
+        # Recovery (plan #697) reuses the entire #693 backfill state machine —
+        # cursor checkpoint, time-boxed chunks, complete/stall lifecycle — and
+        # differs only in bypassing the tier-1 seen-ledger so dropped-and-deleted
+        # near-dups get re-fetched and re-evaluated under the CURRENT pHash
+        # threshold (tier-2 disk still spares files we kept). The
+        # `_backfill_bypass_seen` flag rides alongside the running backfill state;
+        # download mode is "recovery" when both are set.
+        bypass_seen = bool(overrides.get("_backfill_bypass_seen"))
         if in_backfill:
             skip_value: bool | str = BACKFILL_SKIP_VALUE
             source_config.timeout = BACKFILL_CHUNK_SECONDS
@@ -127,46 +143,35 @@ class DownloadService:
         else:
             skip_value = TICK_SKIP_VALUE
 
-        effective_url = _effective_url(
-            ctx["platform"], ctx["url"], ctx["config_overrides"] or {}
-        )
-
-        dl_result = await self.gdl.download(
-            url=effective_url,
-            artist_slug=ctx["artist_slug"],
-            platform=ctx["platform"],
-            source_config=source_config,
-            cookies_path=ctx["cookies_path"],
-            auth_token=ctx["auth_token"],
-            skip_value=skip_value,
-        )
-
         resolved_campaign_id: str | None = None
-        if (
-            ctx["platform"] == "patreon"
-            and not dl_result.success
-            and "patreon_campaign_id" not in (ctx["config_overrides"] or {})
-            and _looks_like_campaign_id_failure(dl_result.stdout, dl_result.stderr)
-        ):
-            vanity = _extract_patreon_vanity(ctx["url"])
-            if vanity:
-                log.info(
-                    "Attempting campaign-ID resolution for %s (%s)",
-                    ctx["artist_slug"], vanity,
-                )
-                resolved_campaign_id = await resolve_campaign_id(
-                    vanity, ctx["cookies_path"]
-                )
-                if resolved_campaign_id:
-                    dl_result = await self.gdl.download(
-                        url=f"https://www.patreon.com/id:{resolved_campaign_id}",
-                        artist_slug=ctx["artist_slug"],
-                        platform=ctx["platform"],
-                        source_config=source_config,
-                        cookies_path=ctx["cookies_path"],
-                        auth_token=ctx["auth_token"],
-                        skip_value=skip_value,
-                    )
+        if ctx["platform"] == "patreon":
+            # Native ingester (plan #697) fully replaces gallery-dl for Patreon
+            # in phase 2 — zero per-file HEADs, native cursor/resume, loud drift
+            # detection. Returns a DownloadResult-shaped object so phase 3 is
+            # untouched. The gallery-dl Patreon path + its campaign-id retry are
+            # removed at the step-5 cutover.
+            if in_backfill and bypass_seen:
+                mode = "recovery"
+            elif in_backfill:
+                mode = "backfill"
+            else:
+                mode = "tick"
+            dl_result, resolved_campaign_id = await self._run_patreon_ingester(
+                ctx, source_config, mode,
+            )
+        else:
+            effective_url = _effective_url(
+                ctx["platform"], ctx["url"], ctx["config_overrides"] or {}
+            )
+            dl_result = await self.gdl.download(
+                url=effective_url,
+                artist_slug=ctx["artist_slug"],
+                platform=ctx["platform"],
+                source_config=source_config,
+                cookies_path=ctx["cookies_path"],
+                auth_token=ctx["auth_token"],
+                skip_value=skip_value,
+            )
 
         # A backfill chunk that hit its time-box but made forward progress is
         # NORMAL, not a failure — reclassify TIMEOUT → PARTIAL so it reads as
@@ -190,6 +195,71 @@ class DownloadService:
             ctx["event_id"], ctx, dl_result, resolved_campaign_id,
         )
 
+    async def _run_patreon_ingester(
+        self, ctx: dict, source_config, mode: str,
+    ) -> tuple[DownloadResult, str | None]:
+        """Phase-2 Patreon branch: resolve the campaign id, then run the native
+        ingester in a worker thread (it is sync requests/subprocess).
+
+        Returns (DownloadResult, resolved_campaign_id). `resolved_campaign_id` is
+        non-None only when we had to look it up from the vanity URL this run, so
+        phase 3 caches it on the source the same way the old gallery-dl retry did.
+        A campaign id we cannot resolve is a loud failure (NOT_FOUND) — never a
+        silent empty success.
+        """
+        overrides = ctx["config_overrides"] or {}
+        campaign_id = overrides.get("patreon_campaign_id")
+        resolved_campaign_id: str | None = None
+        if not campaign_id:
+            # A `.../id:<digits>` URL already carries the campaign id — no lookup
+            # needed (and the vanity regex deliberately excludes the id: form).
+            id_match = re.search(r"/id:(\d+)", ctx["url"])
+            if id_match:
+                campaign_id = id_match.group(1)
+            else:
+                vanity = _extract_patreon_vanity(ctx["url"])
+                if vanity:
+                    campaign_id = await resolve_campaign_id(vanity, ctx["cookies_path"])
+                    resolved_campaign_id = campaign_id
+
+        if not campaign_id:
+            return (
+                DownloadResult(
+                    success=False,
+                    url=ctx["url"],
+                    artist_slug=ctx["artist_slug"],
+                    platform="patreon",
+                    error_type=ErrorType.NOT_FOUND,
+                    error_message=(
+                        "Could not resolve Patreon campaign id from the source "
+                        "URL (vanity lookup failed — cookies expired or creator "
+                        "moved?)"
+                    ),
+                ),
+                None,
+            )
+
+        ingester = PatreonIngester(
+            images_root=self.gdl.images_root,
+            cookies_path=ctx["cookies_path"],
+            session_factory=self.sync_session_factory,
+            validate=self.gdl._validate_files,
+        )
+        loop = asyncio.get_running_loop()
+        dl_result = await loop.run_in_executor(
+            None,
+            lambda: ingester.run(
+                source_id=ctx["source_id"],
+                campaign_id=campaign_id,
+                artist_slug=ctx["artist_slug"],
+                url=ctx["url"],
+                mode=mode,
+                resume_cursor=source_config.resume_cursor,
+                time_budget_seconds=source_config.timeout,
+            ),
+        )
+        return dl_result, resolved_campaign_id
+
     async def _phase1_setup(self, source_id: int) -> dict[str, Any]:
         source = (await self.async_session.execute(
             select(Source).options(joinedload(Source.artist)).where(Source.id == source_id)
@@ -468,6 +538,9 @@ class DownloadService:
             new_overrides["_backfill_state"] = "complete"
             new_overrides.pop("_backfill_cursor", None)
             new_overrides.pop("_backfill_cursor_stalls", None)
+            # plan #697: a recovery walk shares this lifecycle; clear its bypass
+            # flag on completion so the next routine tick honors the seen-ledger.
+            new_overrides.pop("_backfill_bypass_seen", None)
             src.config_overrides = new_overrides
             src.backfill_runs_remaining = 0
             return