Merge pull request 'Native Patreon ingester + download-engine ownership (plans #697, #703–#708)' (#72) from dev into main

alembic/versions/0037_patreon_seen_media.py

@@ -0,0 +1,53 @@
+"""patreon_seen_media: per-source ledger of already-ingested Patreon media
+
+Revision ID: 0037
+Revises: 0036
+Create Date: 2026-06-05
+
+Native Patreon ingester (build step 2a). Replaces gallery-dl's
+archive.sqlite3 with our own queryable table. The downloader upserts one
+row per (source, media) so routine walks skip media we've already
+processed; a future "recovery" mode bypasses the ledger to re-walk.
+
+`filehash` is a 32-hex Patreon CDN MD5, OR a video sentinel of the form
+``video:<post_id>:<media_id>`` — hence String(128). The unique
+constraint on (source_id, filehash) is the dedup upsert key.
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0037"
+down_revision: Union[str, None] = "0036"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "patreon_seen_media",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column(
+            "source_id",
+            sa.Integer,
+            sa.ForeignKey("source.id", ondelete="CASCADE"),
+            nullable=False,
+            index=True,
+        ),
+        sa.Column("filehash", sa.String(128), nullable=False),
+        sa.Column("post_id", sa.String(64), nullable=True),
+        sa.Column(
+            "seen_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("NOW()"),
+        ),
+        sa.UniqueConstraint(
+            "source_id", "filehash", name="uq_patreon_seen_media_source_id"
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("patreon_seen_media")

alembic/versions/0038_patreon_failed_media.py

@@ -0,0 +1,58 @@
+"""patreon_failed_media: per-source dead-letter ledger for failing Patreon media
+
+Revision ID: 0038
+Revises: 0037
+Create Date: 2026-06-06
+
+Plan #705 (#7). Media that keeps failing to download/validate (404'd CDN,
+deleted post, geo-blocked Mux, persistently-corrupt bytes) gets recorded here
+with an attempt counter; once it crosses the dead-letter threshold the ingester
+skips it on routine walks (recovery still re-attempts). A clean download clears
+the row. UNIQUE (source_id, filehash) is the upsert key (same media key the
+seen-ledger uses).
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0038"
+down_revision: Union[str, None] = "0037"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "patreon_failed_media",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column(
+            "source_id",
+            sa.Integer,
+            sa.ForeignKey("source.id", ondelete="CASCADE"),
+            nullable=False,
+            index=True,
+        ),
+        sa.Column("filehash", sa.String(128), nullable=False),
+        sa.Column("attempts", sa.Integer, nullable=False, server_default="1"),
+        sa.Column("last_error", sa.Text, nullable=True),
+        sa.Column(
+            "first_failed_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("NOW()"),
+        ),
+        sa.Column(
+            "last_failed_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("NOW()"),
+        ),
+        sa.UniqueConstraint(
+            "source_id", "filehash", name="uq_patreon_failed_media_source_id"
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("patreon_failed_media")

backend/app/api/credentials.py

@@ -121,13 +121,15 @@ async def delete_credential(platform: str):
 
 @credentials_bp.route("/<platform>/verify", methods=["POST"])
 async def verify_credential(platform: str):
-    """Test the stored credential by running gallery-dl --simulate
-    against one of the platform's enabled sources. On success stamps
-    last_verified. Returns {valid: bool|null, reason, last_verified?}.
-    valid=null means "couldn't test" (no credential, or no enabled
-    source to point at)."""
+    """Test the stored credential against one of the platform's enabled sources,
+    WITHOUT downloading. Routes through the platform's backend
+    (download_backends.verify_credential) — native ingester for Patreon, an
+    authenticated API page; gallery-dl --simulate for the rest. On success
+    stamps last_verified. Returns {valid: bool|null, reason, last_verified?};
+    valid=null means "couldn't test" (no credential, no enabled source, or an
+    inconclusive network/drift result)."""
     from ..models import Artist, Source
-    from ..services.gallery_dl import GalleryDLService, SourceConfig
+    from ..services.download_backends import verify_source_credential
 
     async with get_session() as session:
         if not await _ext_key_ok(session):
@@ -154,14 +156,14 @@ async def verify_credential(platform: str):
         cookies_path = await svc.get_cookies_path(platform)
         auth_token = await svc.get_token(platform)
 
-    gdl = GalleryDLService(images_root=Path("/images"))
-    ok, message = await gdl.verify(
+    ok, message = await verify_source_credential(
+        platform=platform,
         url=source.url,
         artist_slug=artist.slug,
-        platform=platform,
-        source_config=SourceConfig.from_dict(source.config_overrides or {}),
+        config_overrides=source.config_overrides or {},
         cookies_path=str(cookies_path) if cookies_path else None,
         auth_token=auth_token,
+        images_root=Path("/images"),
     )
 
     last_verified = None

backend/app/api/sources.py

@@ -122,28 +122,119 @@ async def delete_source(source_id: int):
 
 @sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
 async def set_backfill(source_id: int):
-    """Plan #693: start or stop a run-until-done backfill. Body:
-    `{"action": "start" | "stop"}` (default "start"). 'start' walks the full
-    post history in time-boxed chunks until it reaches the bottom (then the
-    source shows 'complete'); 'stop' cancels back to tick mode. Returns the
-    updated source dict (incl. backfill_state / backfill_chunks)."""
+    """Plan #693/#697: start/stop a run-until-done backfill, or start a recovery.
+    Body: `{"action": "start" | "stop" | "recover"}` (default "start"). 'start'
+    walks the full post history in time-boxed chunks until it reaches the bottom
+    (then the source shows 'complete'); 'recover' is the same walk but bypasses
+    the Patreon seen-ledger to re-fetch dropped-and-deleted near-dups under the
+    current pHash threshold; 'stop' cancels either back to tick mode. Returns the
+    updated source dict (incl. backfill_state / backfill_chunks /
+    backfill_bypass_seen)."""
+    from pathlib import Path
+
+    from ..services.credential_service import CredentialService
+    from ..services.download_backends import (
+        uses_native_ingester,
+        verify_source_credential,
+    )
+    from .credentials import _get_crypto
+
     payload = await request.get_json(silent=True) or {}
     action = payload.get("action", "start")
-    if action not in ("start", "stop"):
-        return _bad("invalid_action", detail="action must be 'start' or 'stop'")
+    if action not in ("start", "stop", "recover"):
+        return _bad(
+            "invalid_action",
+            detail="action must be 'start', 'stop', or 'recover'",
+        )
+
+    # Pre-flight (plan #703 #2): before arming a deep walk on a native-ingester
+    # platform (where verify is one cheap API page), refuse if the credential is
+    # DEFINITIVELY rejected — don't burn chunks against expired cookies. Proceed
+    # on valid OR inconclusive (a network blip shouldn't block). Gated to native
+    # platforms: gallery-dl verify is a slow --simulate subprocess, too heavy for
+    # an arm action. The credential read happens in a session that's CLOSED
+    # before the verify network call (don't hold a DB conn across the request).
+    if action in ("start", "recover"):
+        async with get_session() as session:
+            rec = await SourceService(session).get(source_id)
+            if rec is None:
+                return _bad("not_found", status=404)
+            native = uses_native_ingester(rec.platform)
+            if native:
+                cred = CredentialService(session, _get_crypto())
+                cookies_path = await cred.get_cookies_path(rec.platform)
+                auth_token = await cred.get_token(rec.platform)
+        if native:
+            ok, message = await verify_source_credential(
+                platform=rec.platform,
+                url=rec.url,
+                artist_slug=rec.artist_slug,
+                config_overrides=rec.config_overrides or {},
+                cookies_path=str(cookies_path) if cookies_path else None,
+                auth_token=auth_token,
+                images_root=Path("/images"),
+            )
+            if ok is False:
+                return _bad("credential_rejected", detail=message, status=409)
+
     async with get_session() as session:
         try:
             svc = SourceService(session)
-            record = (
-                await svc.start_backfill(source_id)
-                if action == "start"
-                else await svc.stop_backfill(source_id)
-            )
+            if action == "start":
+                record = await svc.start_backfill(source_id)
+            elif action == "recover":
+                record = await svc.start_recovery(source_id)
+            else:
+                record = await svc.stop_backfill(source_id)
         except LookupError:
             return _bad("not_found", status=404)
     return jsonify(record.to_dict())
 
 
+@sources_bp.route("/<int:source_id>/preview", methods=["POST"])
+async def preview_source_endpoint(source_id: int):
+    """Plan #708 B4: dry-run — count what a backfill WOULD download for a native
+    platform (Patreon today), without downloading. Walks the first few feed pages
+    and counts media not already in the seen/dead ledgers. Returns
+    {total_new, posts_scanned, pages_scanned, has_more, sample[]} or 409 + reason
+    (unresolvable campaign id / auth / drift). 400 for gallery-dl platforms (no
+    cheap dry-run — their verify is a slow --simulate)."""
+    from pathlib import Path
+
+    from ..services.credential_service import CredentialService
+    from ..services.download_backends import preview_source, uses_native_ingester
+    from ..tasks._sync_engine import sync_session_factory
+    from .credentials import _get_crypto
+
+    async with get_session() as session:
+        rec = await SourceService(session).get(source_id)
+        if rec is None:
+            return _bad("not_found", status=404)
+        if not uses_native_ingester(rec.platform):
+            return _bad(
+                "unsupported",
+                detail="Preview is only available for native-ingester platforms.",
+                status=400,
+            )
+        cred = CredentialService(session, _get_crypto())
+        cookies_path = await cred.get_cookies_path(rec.platform)
+
+    # The walk + ledger reads are sync (run off the request loop); the process
+    # sync engine is the same one the download task uses.
+    result = await preview_source(
+        platform=rec.platform,
+        url=rec.url,
+        source_id=source_id,
+        config_overrides=rec.config_overrides or {},
+        cookies_path=str(cookies_path) if cookies_path else None,
+        images_root=Path("/images"),
+        sync_session_factory=sync_session_factory(),
+    )
+    if "error" in result:
+        return _bad("preview_failed", detail=result["error"], status=409)
+    return jsonify(result)
+
+
 @sources_bp.route("/<int:source_id>/check", methods=["POST"])
 async def check_source(source_id: int):
     """FC-3c: enqueue a download for this source.

backend/app/models/__init__.py

@@ -14,6 +14,8 @@ from .import_settings import ImportSettings
 from .import_task import ImportTask
 from .library_audit_run import LibraryAuditRun
 from .ml_settings import MLSettings
+from .patreon_failed_media import PatreonFailedMedia
+from .patreon_seen_media import PatreonSeenMedia
 from .post import Post
 from .post_attachment import PostAttachment
 from .series_page import SeriesPage
@@ -33,6 +35,8 @@ __all__ = [
     "BackupRun",
     "Source",
     "Credential",
+    "PatreonFailedMedia",
+    "PatreonSeenMedia",
     "Post",
     "PostAttachment",
     "SeriesPage",

backend/app/models/patreon_failed_media.py

@@ -0,0 +1,45 @@
+"""PatreonFailedMedia — per-source dead-letter ledger of Patreon media that
+keeps failing to download/validate.
+
+Plan #705 (#7). A media that fails every walk (404'd CDN URL, deleted post,
+geo-blocked Mux stream, persistently-corrupt bytes) would otherwise re-error
+forever and re-burn backfill chunks. After ``attempts`` reaches the dead-letter
+threshold the ingester skips it on routine tick/backfill walks (recovery still
+re-attempts it — the operator's "try everything again"). A later clean download
+clears the row (the media recovered).
+
+`filehash` is the same per-media key the seen-ledger uses (32-hex CDN MD5, or a
+``video:`` / ``post:filename`` synthesized key) — hence String(128). UNIQUE
+(source_id, filehash) is the upsert key.
+"""
+
+from datetime import datetime
+
+from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func
+from sqlalchemy.orm import Mapped, mapped_column
+from sqlalchemy.types import DateTime
+
+from .base import Base
+
+
+class PatreonFailedMedia(Base):
+    __tablename__ = "patreon_failed_media"
+    __table_args__ = (
+        UniqueConstraint(
+            "source_id", "filehash", name="uq_patreon_failed_media_source_id"
+        ),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    source_id: Mapped[int] = mapped_column(
+        ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    filehash: Mapped[str] = mapped_column(String(128), nullable=False)
+    attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    first_failed_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    last_failed_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )

backend/app/models/patreon_seen_media.py

@@ -0,0 +1,38 @@
+"""PatreonSeenMedia — per-source ledger of Patreon media already
+downloaded+processed.
+
+Replaces gallery-dl's archive.sqlite3 with our own queryable table so
+routine walks can skip media we've already ingested (and a future
+"recovery" mode can deliberately bypass the ledger to re-walk).
+
+`filehash` is normally a Patreon CDN MD5 (32 hex chars), but videos —
+which have no stable content hash at discovery time — use a sentinel of
+the form ``video:<post_id>:<media_id>``, hence String(128) rather than 32.
+"""
+
+from datetime import datetime
+
+from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
+from sqlalchemy.orm import Mapped, mapped_column
+from sqlalchemy.types import DateTime
+
+from .base import Base
+
+
+class PatreonSeenMedia(Base):
+    __tablename__ = "patreon_seen_media"
+    __table_args__ = (
+        # Dedup key the downloader upserts against: one ledger row per
+        # (source, media). A second sighting of the same media is a no-op.
+        UniqueConstraint("source_id", "filehash", name="uq_patreon_seen_media_source_id"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    source_id: Mapped[int] = mapped_column(
+        ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    filehash: Mapped[str] = mapped_column(String(128), nullable=False)
+    post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    seen_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )

backend/app/services/download_backends.py

@@ -0,0 +1,229 @@
+"""Platform → download-backend dispatch (one place that knows which platforms
+are served by the native FC ingester vs. the gallery-dl subprocess).
+
+gallery-dl wasn't built to be driven by an automated scheduler — no native
+checkpoint/resume, no structured logs, per-file HEADs that dominate wall-clock.
+The native ingester (services/patreon_ingester.py, plan #697) replaces it for
+Patreon and is the path we grow as more platforms migrate. To keep that
+migration DRY, every caller that has to behave differently per backend —
+download routing, the credential-verify probe, cursor handling — asks THIS
+module instead of testing ``platform == "patreon"`` inline. When a platform gets
+a native ingester, it moves into ``NATIVE_INGESTER_PLATFORMS`` here and both the
+download path and verify switch over together.
+
+The backend surfaces share a UNIFORM signature so a caller invokes the same
+function regardless of platform:
+  - verify_credential(...)  → (ok: bool|None, message: str)
+  - (download stays in download_service for now; uses_native_ingester() is the
+    shared predicate it routes on, so the decision lives here too.)
+"""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+
+from .gallery_dl import DownloadResult, ErrorType
+from .patreon_ingester import PatreonIngester
+from .patreon_resolver import resolve_campaign_id_for_source
+
+# Platforms whose download + verify go through the native ingester rather than
+# gallery-dl. gallery-dl still serves every other platform (subscribestar,
+# hentaifoundry, discord, pixiv, deviantart) unchanged.
+NATIVE_INGESTER_PLATFORMS = frozenset({"patreon"})
+
+
+def uses_native_ingester(platform: str) -> bool:
+    """True when `platform` is served by the native ingester (not gallery-dl).
+    The single predicate the download path and verify both route on."""
+    return platform in NATIVE_INGESTER_PLATFORMS
+
+
+async def run_download(
+    *,
+    ctx: dict,
+    source_config,
+    skip_value: bool | str,
+    mode: str | None,
+    gdl,
+    sync_session_factory,
+) -> tuple[DownloadResult, str | None]:
+    """Uniform download across backends — the download counterpart to
+    `verify_source_credential`, so this module is the ONE place that knows how
+    each platform both downloads AND verifies (the seam that makes adding a
+    platform a bounded job).
+
+    Returns `(DownloadResult, resolved_campaign_id)`; `resolved_campaign_id` is
+    non-None only when a native vanity lookup ran this call (so phase 3 caches
+    it). Native platforms route through their ingester in `mode`
+    (tick/backfill/recovery); gallery-dl platforms run the subprocess. The caller
+    (download_service) prepares `source_config`/`skip_value`/`mode` from the
+    backfill state machine and owns phase 3.
+    """
+    platform = ctx["platform"]
+    if uses_native_ingester(platform):
+        return await _run_native_ingester(
+            ctx, source_config, mode, gdl, sync_session_factory
+        )
+    result = await gdl.download(
+        url=ctx["url"],
+        artist_slug=ctx["artist_slug"],
+        platform=platform,
+        source_config=source_config,
+        cookies_path=ctx["cookies_path"],
+        auth_token=ctx["auth_token"],
+        skip_value=skip_value,
+    )
+    return result, None
+
+
+async def _run_native_ingester(
+    ctx: dict, source_config, mode: str | None, gdl, sync_session_factory,
+) -> tuple[DownloadResult, str | None]:
+    """Patreon (today the only native platform): resolve the campaign id, then run
+    the native ingester in a worker thread (it is sync requests/subprocess).
+
+    `resolved_campaign_id` is non-None only when we had to look it up from the
+    vanity URL this run, so phase 3 caches it the way the old gallery-dl retry
+    did. A campaign id we cannot resolve is a loud NOT_FOUND — never a silent
+    empty success.
+    """
+    overrides = ctx["config_overrides"] or {}
+    campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source(
+        ctx["url"], ctx["cookies_path"], overrides
+    )
+    if not campaign_id:
+        return (
+            DownloadResult(
+                success=False,
+                url=ctx["url"],
+                artist_slug=ctx["artist_slug"],
+                platform="patreon",
+                error_type=ErrorType.NOT_FOUND,
+                error_message=(
+                    "Could not resolve Patreon campaign id from the source URL "
+                    "(vanity lookup failed — cookies expired or creator moved?)"
+                ),
+            ),
+            None,
+        )
+
+    # Honor the operator's existing rate-limit knobs on the native path (plan
+    # #703): the global download_rate_limit_seconds (gallery-dl's `rate_limit`,
+    # here on the gdl service) paces media downloads; page fetches use the
+    # per-source sleep_request override, else `max(0.5, rate_limit/4)` — the same
+    # API-pacing default gallery-dl applied as its `sleep-request`.
+    rate_limit = gdl._rate_limit
+    request_sleep = (
+        source_config.sleep_request
+        if source_config.sleep_request is not None
+        else max(0.5, rate_limit / 4)
+    )
+    ingester = PatreonIngester(
+        images_root=gdl.images_root,
+        cookies_path=ctx["cookies_path"],
+        session_factory=sync_session_factory,
+        validate=gdl._validate_files,
+        rate_limit=rate_limit,
+        request_sleep=request_sleep,
+    )
+    loop = asyncio.get_running_loop()
+    dl_result = await loop.run_in_executor(
+        None,
+        lambda: ingester.run(
+            source_id=ctx["source_id"],
+            campaign_id=campaign_id,
+            artist_slug=ctx["artist_slug"],
+            url=ctx["url"],
+            mode=mode,
+            resume_cursor=source_config.resume_cursor,
+            time_budget_seconds=source_config.timeout,
+            posts_base=int(overrides.get("_backfill_posts", 0)),
+        ),
+    )
+    return dl_result, resolved_campaign_id
+
+
+async def preview_source(
+    *,
+    platform: str,
+    url: str,
+    source_id: int,
+    config_overrides: dict | None,
+    cookies_path: str | None,
+    images_root: Path,
+    sync_session_factory,
+    page_limit: int = 3,
+) -> dict:
+    """Dry-run preview for a native platform (plan #708 B4): resolve the campaign
+    id, then walk a few pages counting media not already seen/dead — no download.
+
+    Returns the preview dict (total_new / posts_scanned / pages_scanned /
+    has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure.
+    Native-only — the caller gates on `uses_native_ingester`.
+    """
+    import asyncio
+
+    from .patreon_client import PatreonAPIError
+
+    campaign_id, _ = await resolve_campaign_id_for_source(
+        url, cookies_path, config_overrides or {}
+    )
+    if not campaign_id:
+        return {
+            "error": (
+                "Couldn't resolve the campaign id from the source URL "
+                "(cookies expired, or the creator moved/renamed?)."
+            )
+        }
+    ingester = PatreonIngester(
+        images_root=images_root,
+        cookies_path=cookies_path,
+        session_factory=sync_session_factory,
+    )
+    loop = asyncio.get_running_loop()
+    try:
+        result = await loop.run_in_executor(
+            None,
+            lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit),
+        )
+    except PatreonAPIError as exc:
+        return {"error": f"Couldn't preview: {exc}"}
+    return result
+
+
+async def verify_source_credential(
+    *,
+    platform: str,
+    url: str,
+    artist_slug: str,
+    config_overrides: dict | None,
+    cookies_path: str | None,
+    auth_token: str | None,
+    images_root: Path,
+) -> tuple[bool | None, str]:
+    """Uniform credential probe across backends. Returns `(ok, message)`:
+    True = authenticated, False = rejected, None = inconclusive (drift /
+    network / nothing to test). Callers don't branch on platform — they call
+    this and render the result.
+    """
+    if uses_native_ingester(platform):
+        # Native ingester platforms verify via their own lightweight auth probe
+        # (resolve campaign id + one authenticated API page). Patreon today.
+        from .patreon_ingester import verify_patreon_credential
+
+        return await verify_patreon_credential(url, cookies_path, config_overrides)
+
+    # gallery-dl platforms: --simulate one item; the extractor errors before it
+    # can list if auth is bad.
+    from .gallery_dl import GalleryDLService, SourceConfig
+
+    gdl = GalleryDLService(images_root=images_root)
+    return await gdl.verify(
+        url=url,
+        artist_slug=artist_slug,
+        platform=platform,
+        source_config=SourceConfig.from_dict(config_overrides or {}),
+        cookies_path=cookies_path,
+        auth_token=auth_token,
+    )

backend/app/services/download_service.py

@@ -13,7 +13,6 @@ from __future__ import annotations
 
 import asyncio
 import logging
-import re
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
@@ -25,45 +24,25 @@ from sqlalchemy.orm import joinedload
 
 from ..models import Artist, DownloadEvent, Source
 from .credential_service import CredentialService
+from .download_backends import run_download, uses_native_ingester
 from .gallery_dl import (
     BACKFILL_CHUNK_SECONDS,
     BACKFILL_SKIP_VALUE,
     TICK_SKIP_VALUE,
+    DownloadResult,
     ErrorType,
     GalleryDLService,
     SourceConfig,
-    parse_last_cursor,
+    extract_errors_warnings,
+    truncate_log,
 )
 from .importer import Importer
-from .patreon_resolver import resolve_campaign_id
 from .platforms import auth_type_for
 from .scheduler_service import set_platform_cooldown
 
 log = logging.getLogger(__name__)
 
 
-_PATREON_VANITY_RE = re.compile(
-    r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)",
-    re.IGNORECASE,
-)
-_CAMPAIGN_ID_FAILURE_PATTERN = "failed to extract campaign id"
-
-
-def _extract_patreon_vanity(url: str) -> str | None:
-    m = _PATREON_VANITY_RE.match(url)
-    return m.group(1) if m else None
-
-
-def _looks_like_campaign_id_failure(stdout: str, stderr: str) -> bool:
-    return _CAMPAIGN_ID_FAILURE_PATTERN in f"{stdout}\n{stderr}".lower()
-
-
-def _effective_url(platform: str, source_url: str, overrides: dict) -> str:
-    if platform == "patreon" and overrides.get("patreon_campaign_id"):
-        return f"https://www.patreon.com/id:{overrides['patreon_campaign_id']}"
-    return source_url
-
-
 class DownloadService:
     """Async orchestrator. The Celery task runs `asyncio.run(svc.download_source(N))`.
 
@@ -78,12 +57,18 @@ class DownloadService:
         gdl: GalleryDLService,
         importer: Importer,
         cred_service: CredentialService,
+        sync_session_factory=None,
     ):
         self.async_session = async_session
         self.sync_session = sync_session
         self.gdl = gdl
         self.importer = importer
         self.cred_service = cred_service
+        # Sync sessionmaker the native Patreon ingester opens SHORT-LIVED
+        # sessions from for its seen-ledger reads/writes (never one held across
+        # the multi-minute walk — see PatreonIngester). Only the patreon branch
+        # of phase 2 uses it; gallery-dl sources leave it None.
+        self.sync_session_factory = sync_session_factory
 
     async def download_source(self, source_id: int) -> int:
         """Returns DownloadEvent.id. Idempotent: in-flight events are returned as-is."""
@@ -118,56 +103,41 @@ class DownloadService:
         # Operator drives this via POST /api/sources/{id}/backfill {action}.
         overrides = ctx["config_overrides"] or {}
         in_backfill = overrides.get("_backfill_state") == "running"
+        # Recovery (plan #697) reuses the entire #693 backfill state machine —
+        # cursor checkpoint, time-boxed chunks, complete/stall lifecycle — and
+        # differs only in bypassing the tier-1 seen-ledger so dropped-and-deleted
+        # near-dups get re-fetched and re-evaluated under the CURRENT pHash
+        # threshold (tier-2 disk still spares files we kept). The
+        # `_backfill_bypass_seen` flag rides alongside the running backfill state;
+        # download mode is "recovery" when both are set.
+        bypass_seen = bool(overrides.get("_backfill_bypass_seen"))
         if in_backfill:
             skip_value: bool | str = BACKFILL_SKIP_VALUE
             source_config.timeout = BACKFILL_CHUNK_SECONDS
             pending_cursor = overrides.get("_backfill_cursor")
-            if ctx["platform"] == "patreon" and pending_cursor:
+            if uses_native_ingester(ctx["platform"]) and pending_cursor:
                 source_config.resume_cursor = pending_cursor
         else:
             skip_value = TICK_SKIP_VALUE
 
-        effective_url = _effective_url(
-            ctx["platform"], ctx["url"], ctx["config_overrides"] or {}
+        # Phase 2 dispatch is uniform across backends (download_backends.
+        # run_download — the download counterpart to verify_source_credential):
+        # native platforms run their ingester in `mode` (zero per-file HEADs,
+        # native cursor/resume, loud drift detection); gallery-dl platforms run
+        # the subprocess. Either returns a DownloadResult-shaped object so phase 3
+        # is untouched. `mode` is None for gallery-dl (run_download ignores it).
+        mode: str | None = None
+        if uses_native_ingester(ctx["platform"]):
+            if in_backfill and bypass_seen:
+                mode = "recovery"
+            elif in_backfill:
+                mode = "backfill"
+            else:
+                mode = "tick"
+        dl_result, resolved_campaign_id = await self._run_download(
+            ctx=ctx, source_config=source_config, skip_value=skip_value, mode=mode,
         )
 
-        dl_result = await self.gdl.download(
-            url=effective_url,
-            artist_slug=ctx["artist_slug"],
-            platform=ctx["platform"],
-            source_config=source_config,
-            cookies_path=ctx["cookies_path"],
-            auth_token=ctx["auth_token"],
-            skip_value=skip_value,
-        )
-
-        resolved_campaign_id: str | None = None
-        if (
-            ctx["platform"] == "patreon"
-            and not dl_result.success
-            and "patreon_campaign_id" not in (ctx["config_overrides"] or {})
-            and _looks_like_campaign_id_failure(dl_result.stdout, dl_result.stderr)
-        ):
-            vanity = _extract_patreon_vanity(ctx["url"])
-            if vanity:
-                log.info(
-                    "Attempting campaign-ID resolution for %s (%s)",
-                    ctx["artist_slug"], vanity,
-                )
-                resolved_campaign_id = await resolve_campaign_id(
-                    vanity, ctx["cookies_path"]
-                )
-                if resolved_campaign_id:
-                    dl_result = await self.gdl.download(
-                        url=f"https://www.patreon.com/id:{resolved_campaign_id}",
-                        artist_slug=ctx["artist_slug"],
-                        platform=ctx["platform"],
-                        source_config=source_config,
-                        cookies_path=ctx["cookies_path"],
-                        auth_token=ctx["auth_token"],
-                        skip_value=skip_value,
-                    )
-
         # A backfill chunk that hit its time-box but made forward progress is
         # NORMAL, not a failure — reclassify TIMEOUT → PARTIAL so it reads as
         # "ok/progress" (PARTIAL maps to status "ok"), not a red error, and the
@@ -175,7 +145,7 @@ class DownloadService:
         # timed out with NO progress stays TIMEOUT and feeds phase 3's
         # stall-guard. Leave RATE_LIMITED alone so the platform-cooldown fires.
         if in_backfill and dl_result.error_type == ErrorType.TIMEOUT:
-            new_cursor = parse_last_cursor(dl_result.stdout, dl_result.stderr)
+            new_cursor = dl_result.cursor  # plan #704: structured, not scraped
             advanced = bool(
                 (new_cursor and new_cursor != overrides.get("_backfill_cursor"))
                 or dl_result.files_downloaded > 0
@@ -190,6 +160,18 @@ class DownloadService:
             ctx["event_id"], ctx, dl_result, resolved_campaign_id,
         )
 
+    async def _run_download(
+        self, *, ctx: dict, source_config, skip_value, mode: str | None,
+    ) -> tuple[DownloadResult, str | None]:
+        """Phase-2 dispatch → `download_backends.run_download`, passing this
+        service's gdl + sync sessionmaker. Kept as a thin instance method so tests
+        can stub the whole phase-2 dispatch on the service, and so the per-backend
+        construction lives in download_backends (the backend registry)."""
+        return await run_download(
+            ctx=ctx, source_config=source_config, skip_value=skip_value, mode=mode,
+            gdl=self.gdl, sync_session_factory=self.sync_session_factory,
+        )
+
     async def _phase1_setup(self, source_id: int) -> dict[str, Any]:
         source = (await self.async_session.execute(
             select(Source).options(joinedload(Source.artist)).where(Source.id == source_id)
@@ -386,11 +368,16 @@ class DownloadService:
             select(DownloadEvent).where(DownloadEvent.id == event_id)
         )).scalar_one()
 
-        run_stats = self.gdl._compute_run_stats(
-            dl_result.return_code, dl_result.stdout, dl_result.stderr
-        )
+        # plan #704: the native ingester returns structured run_stats; only the
+        # gallery-dl path needs the regex-over-stdout reconstruction.
+        if dl_result.run_stats is not None:
+            run_stats = dict(dl_result.run_stats)
+        else:
+            run_stats = self.gdl._compute_run_stats(
+                dl_result.return_code, dl_result.stdout, dl_result.stderr
+            )
         run_stats["quarantined_count"] = dl_result.files_quarantined
-        stderr_summary = self.gdl._extract_errors_warnings(dl_result.stderr)
+        stderr_summary = extract_errors_warnings(dl_result.stderr)
 
         # Plan #544: PARTIAL means the run downloaded ≥1 file but the
         # subprocess didn't finish in budget (typically wall-clock timeout
@@ -410,8 +397,8 @@ class DownloadService:
         ev.metadata_ = {
             "run_stats": run_stats,
             "error_type": dl_result.error_type.value if dl_result.error_type else None,
-            "stdout": self.gdl._truncate_log(dl_result.stdout) or None,
-            "stderr": self.gdl._truncate_log(dl_result.stderr) or None,
+            "stdout": truncate_log(dl_result.stdout) or None,
+            "stderr": truncate_log(dl_result.stderr) or None,
             "stderr_errors_warnings": stderr_summary or None,
             "duration_seconds": dl_result.duration_seconds,
             "quarantined_paths": dl_result.quarantined_paths or None,
@@ -420,6 +407,7 @@ class DownloadService:
         await self._update_source_health(
             source_id=ctx["source_id"], status=status, error_message=ev.error,
             error_type=dl_result.error_type.value if dl_result.error_type else None,
+            retry_after_seconds=getattr(dl_result, "retry_after_seconds", None),
         )
         await self._apply_backfill_lifecycle(ctx, dl_result)
         await self.async_session.commit()
@@ -458,6 +446,12 @@ class DownloadService:
         new_overrides = dict(src.config_overrides or {})
         chunks = int(new_overrides.get("_backfill_chunks", 0)) + 1
         new_overrides["_backfill_chunks"] = chunks
+        # plan #704 (#5): _backfill_posts (the live progress badge) is OWNED by the
+        # ingester now — it writes a monotonic absolute mid-walk at each page
+        # boundary (ingest_core._checkpoint_posts), so the badge climbs DURING a
+        # chunk instead of jumping once per chunk here, and the re-walked resume
+        # page is no longer double-counted. new_overrides (read fresh above)
+        # carries the ingester's committed value forward untouched.
 
         completed = (
             dl_result.success
@@ -468,16 +462,19 @@ class DownloadService:
             new_overrides["_backfill_state"] = "complete"
             new_overrides.pop("_backfill_cursor", None)
             new_overrides.pop("_backfill_cursor_stalls", None)
+            # plan #697: a recovery walk shares this lifecycle; clear its bypass
+            # flag on completion so the next routine tick honors the seen-ledger.
+            new_overrides.pop("_backfill_bypass_seen", None)
             src.config_overrides = new_overrides
             src.backfill_runs_remaining = 0
             return
 
-        # Did not finish. Patreon checkpoints + resumes via cursor; other
-        # platforms have no resumable cursor (every chunk re-walks from the
-        # top), so they advance only by the download archive growing.
+        # Did not finish. The native ingester checkpoints + resumes via cursor
+        # (carried structurally on the result, plan #704); gallery-dl platforms
+        # have no resumable cursor (every chunk re-walks from the top), so they
+        # advance only by the download archive growing.
         new_cursor = (
-            parse_last_cursor(dl_result.stdout, dl_result.stderr)
-            if ctx["platform"] == "patreon" else None
+            dl_result.cursor if uses_native_ingester(ctx["platform"]) else None
         )
         advanced = bool(
             (new_cursor and new_cursor != old_cursor)
@@ -505,7 +502,7 @@ class DownloadService:
 
     async def _update_source_health(
         self, *, source_id: int, status: str, error_message: str | None,
-        error_type: str | None = None,
+        error_type: str | None = None, retry_after_seconds: float | None = None,
     ) -> None:
         """FC-3d: update Source.{consecutive_failures, last_error, last_checked_at}.
 
@@ -537,7 +534,17 @@ class DownloadService:
             # by error class without opening Logs per row.
             source.error_type = error_type
             if error_type == "rate_limited":
-                await set_platform_cooldown(self.async_session, source.platform)
+                # plan #708 B1: honor the server's Retry-After when the native
+                # client surfaced one (clamped to a sane [60, 3600] window so a
+                # tiny hint can't leave the platform effectively un-cooled and a
+                # huge one can't strand it for hours); else the flat default.
+                if retry_after_seconds is not None:
+                    seconds = int(min(max(retry_after_seconds, 60), 3600))
+                    await set_platform_cooldown(
+                        self.async_session, source.platform, seconds=seconds,
+                    )
+                else:
+                    await set_platform_cooldown(self.async_session, source.platform)
         elif status == "skipped":
             source.last_error = None
         source.last_checked_at = now

backend/app/services/file_validator.py

@@ -11,9 +11,15 @@ expected to write.
 
 from __future__ import annotations
 
+import json
+import logging
+import shutil
 from dataclasses import dataclass
+from datetime import UTC, datetime
 from pathlib import Path
 
+log = logging.getLogger(__name__)
+
 JPEG_HEAD = b"\xff\xd8\xff"
 JPEG_TAIL = b"\xff\xd9"
 
@@ -108,3 +114,52 @@ def validate_file(path: Path) -> ValidationResult:
         return ValidationResult(ok=True, format="webp", size=size)
 
     return ValidationResult(ok=True, format=None, size=size)
+
+
+def quarantine_file(
+    images_root: Path,
+    path: Path,
+    artist_slug: str,
+    platform: str,
+    *,
+    url: str | None,
+    result: ValidationResult,
+) -> Path | None:
+    """Move a validation-failed file to `_quarantine/<slug>/<platform>` and write
+    a `.quarantine.json` provenance sidecar next to it.
+
+    Returns the destination path, or None if the move itself failed (file left
+    in place — the caller decides what to report). The caller has already run
+    `validate_file` and seen `result.ok is False`. ONE implementation for both
+    download backends — gallery-dl's batch post-process and the native ingester's
+    per-media path — so the quarantine layout + provenance sidecar can't drift.
+    """
+    quarantine_root = images_root / "_quarantine" / artist_slug / platform
+    try:
+        quarantine_root.mkdir(parents=True, exist_ok=True)
+        dest = quarantine_root / path.name
+        counter = 1
+        while dest.exists():
+            dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}"
+            counter += 1
+        shutil.move(str(path), str(dest))
+        sidecar = dest.with_suffix(dest.suffix + ".quarantine.json")
+        sidecar.write_text(
+            json.dumps(
+                {
+                    "original_path": str(path),
+                    "source_url": url,
+                    "artist_slug": artist_slug,
+                    "platform": platform,
+                    "format": result.format,
+                    "reason": result.reason,
+                    "size": result.size,
+                    "quarantined_at": datetime.now(UTC).isoformat(),
+                },
+                indent=2,
+            )
+        )
+    except OSError as exc:
+        log.error("Failed to quarantine %s: %s. File left in place.", path, exc)
+        return None
+    return dest

backend/app/services/gallery_dl.py

@@ -13,7 +13,6 @@ from __future__ import annotations
 import asyncio
 import json
 import logging
-import re
 import subprocess
 import sys
 import tempfile
@@ -23,7 +22,7 @@ from datetime import UTC, datetime
 from enum import StrEnum
 from pathlib import Path
 
-from .file_validator import is_validatable, validate_file
+from .file_validator import is_validatable, quarantine_file, validate_file
 
 log = logging.getLogger(__name__)
 
@@ -40,6 +39,12 @@ class ErrorType(StrEnum):
     HTTP_ERROR = "http_error"
     UNSUPPORTED_URL = "unsupported_url"
     VALIDATION_FAILED = "validation_failed"
+    # Native Patreon ingester only (plan #697): a response parsed as JSON but
+    # didn't match the JSON:API shape the ingester depends on (missing `data`,
+    # media lacking `file_name`/`url`, etc.). Distinct from AUTH_ERROR — the fix
+    # is updating the ingester's field-set/parser, not rotating credentials. The
+    # contract test guards against silently shipping this.
+    API_DRIFT = "api_drift"
     # Run made real progress (downloaded ≥1 file) but did not finish in the
     # subprocess budget. Distinct from UNKNOWN_ERROR — the downstream status
     # mapping classifies this as "ok" because the next tick continues.
@@ -101,12 +106,14 @@ class SourceConfig:
 
     `resume_cursor` is RUNTIME state, not persisted operator config: the
     cursor-paged backfill checkpoint (see parse_last_cursor + the
-    download_service backfill lifecycle). When set on a patreon backfill
-    run, gallery-dl resumes its newest→oldest walk from that pagination
-    cursor instead of restarting from the top. It is threaded by
-    download_service from config_overrides["_backfill_cursor"]; SourceConfig
-    deliberately does NOT read it in from_dict (config_overrides also holds
-    operator config, and resume_cursor must only apply in backfill mode).
+    download_service backfill lifecycle). It is consumed only by the native
+    Patreon ingester (the sole cursor-paged platform; gallery-dl's Patreon path
+    was removed at the #697 cutover): on a backfill/recovery run the ingester
+    resumes its newest→oldest walk from that pagination cursor instead of
+    restarting from the top. download_service threads it from
+    config_overrides["_backfill_cursor"]; SourceConfig deliberately does NOT read
+    it in from_dict (config_overrides also holds operator config, and
+    resume_cursor must only apply in backfill/recovery mode).
     """
     content_types: list[str] = field(default_factory=lambda: ["all"])
     sleep: float | None = None
@@ -148,6 +155,51 @@ class DownloadResult:
     duration_seconds: float = 0.0
     started_at: str | None = None
     completed_at: str | None = None
+    # Plan #704 — structured fields the NATIVE ingester populates directly (None
+    # on the gallery-dl path, which keeps the regex-over-stdout route). When set,
+    # phase 3 reads these instead of scraping the text it would otherwise have to
+    # reconstruct: `run_stats` mirrors _compute_run_stats' shape; `cursor` is the
+    # backfill checkpoint the ingester knows exactly (no parse_last_cursor).
+    run_stats: dict | None = None
+    cursor: str | None = None
+    posts_processed: int = 0
+    # Plan #708 B1 — the server's Retry-After seconds on a RATE_LIMITED result, so
+    # the platform cooldown matches the hint instead of a flat default. None when
+    # unknown (no header, or not a rate-limit failure).
+    retry_after_seconds: float | None = None
+
+
+def extract_errors_warnings(stderr: str) -> str:
+    """Keep only the `[error]`/`[warning]` lines from a captured stderr blob.
+
+    A generic download-log helper (module-level so the native-ingester result
+    path can shape its logs without reaching through a GalleryDLService instance).
+    """
+    if not stderr:
+        return ""
+    kept = [
+        line for line in stderr.splitlines()
+        if "][error]" in line.lower() or "][warning]" in line.lower()
+    ]
+    return "\n".join(kept)
+
+
+def truncate_log(text: str, max_bytes: int = 500_000) -> str:
+    """Cap a captured log to `max_bytes`, eliding the middle (head + tail kept)."""
+    if not text:
+        return text
+    encoded = text.encode("utf-8")
+    if len(encoded) <= max_bytes:
+        return text
+    half = max_bytes // 2
+    head = encoded[:half].decode("utf-8", errors="ignore")
+    tail = encoded[-half:].decode("utf-8", errors="ignore")
+    head_lines = head.count("\n")
+    tail_lines = tail.count("\n")
+    total_lines = text.count("\n")
+    elided = max(0, total_lines - head_lines - tail_lines)
+    marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
+    return head + marker + tail
 
 
 def _summarize_validation_failures(failures: list[dict]) -> str:
@@ -164,19 +216,38 @@ def _summarize_validation_failures(failures: list[dict]) -> str:
     return f"{n} files quarantined ({top_count}× {top_reason}, mixed)"
 
 
-# gallery-dl logs its pagination cursor (when extractor.*.cursor is truthy)
-# as `Cursor: <token>` — once per fetched page, at debug verbosity. The LAST
-# such line is the furthest-progressed page, i.e. the resume point. We scan
-# both streams (the debug log lands on stderr, but be defensive) and take the
-# final match; passing it back as extractor.patreon.cursor resumes the walk.
-# Survives a timed-out run too: the TimeoutExpired path returns the partial
-# stdout/stderr, so an interrupted backfill still yields its last cursor.
-_CURSOR_RE = re.compile(r"Cursor:\s*(\S+)")
+# (parse_last_cursor was removed in plan #704: the native ingester now carries
+# its checkpoint cursor as a structured DownloadResult.cursor field, so there is
+# no log text to scrape — and gallery-dl platforms never had a cursor.)
 
 
-def parse_last_cursor(stdout: str, stderr: str) -> str | None:
-    matches = _CURSOR_RE.findall(f"{stdout or ''}\n{stderr or ''}")
-    return matches[-1] if matches else None
+def make_run_stats(
+    *,
+    exit_code: int = 0,
+    downloaded_count: int = 0,
+    skipped_count: int = 0,
+    per_item_failures: int = 0,
+    warning_count: int = 0,
+    tier_gated_count: int = 0,
+    quarantined_count: int = 0,
+    dead_lettered_count: int = 0,
+) -> dict:
+    """The canonical `run_stats` dict shape, in ONE place.
+
+    Both result producers — gallery-dl's `_compute_run_stats` (log scrape) and the
+    native ingester's per-outcome tally (ingest_core) — build through this so the
+    key set can't drift between backends. Phase 3 + the Logs UI read these keys.
+    """
+    return {
+        "exit_code": exit_code,
+        "downloaded_count": downloaded_count,
+        "skipped_count": skipped_count,
+        "per_item_failures": per_item_failures,
+        "warning_count": warning_count,
+        "tier_gated_count": tier_gated_count,
+        "quarantined_count": quarantined_count,
+        "dead_lettered_count": dead_lettered_count,
+    }
 
 
 class GalleryDLService:
@@ -212,16 +283,10 @@ class GalleryDLService:
         "permission denied", "tier required", "pledge required",
     ]
 
-    # Per-platform defaults. Lifted from GS — same six platforms FC supports.
+    # Per-platform defaults for the gallery-dl-backed platforms. Patreon was
+    # removed at the plan-#697 cutover — it now uses the native ingester
+    # (services/patreon_ingester.py), not gallery-dl.
     PLATFORM_DEFAULTS = {
-        "patreon": {
-            "content_types": ["images", "image_large", "attachments", "postfile", "content"],
-            "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
-            "filename": "{num:>02}_{filename}.{extension}",
-            "videos": True,
-            "embeds": True,
-            "cursor": True,
-        },
         "subscribestar": {
             "content_types": ["all"],
             "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
@@ -306,25 +371,11 @@ class GalleryDLService:
                 # transfer cap — large GIFs that keep streaming are unaffected.
                 "retries": 2,
                 "timeout": 60.0,
-                # Forward Patreon as Referer/Origin to yt-dlp when it
-                # fetches video manifests. Operator-flagged 2026-06-01
-                # (DaferQ patreon): video posts hosted on Mux carry a JWT
-                # playback restriction that checks Referer/Origin on every
-                # request — not just the token signature. gallery-dl's
-                # HEAD probe to stream.mux.com returns 200 (the token is
-                # valid), but yt-dlp's actual GET-with-Range to fetch the
-                # m3u8 manifest 403s because yt-dlp sends its own default
-                # Referer, which Mux's policy rejects. Forcing the right
-                # headers fixes the headers-only case; Mux IP-range
-                # restrictions are unfixable from here.
-                "ytdl": {
-                    "raw-options": {
-                        "http_headers": {
-                            "Referer": "https://www.patreon.com/",
-                            "Origin": "https://www.patreon.com",
-                        },
-                    },
-                },
+                # NOTE: the Patreon/Mux yt-dlp Referer/Origin forwarding lived
+                # here until the plan-#697 cutover. It was Patreon-specific (and
+                # would have wrongly tagged the other platforms' yt-dlp fetches);
+                # Patreon video is now handled by the native ingester's
+                # downloader (patreon_downloader._VIDEO_HEADERS), so it's gone.
             },
             "output": {"progress": True},
         }
@@ -376,23 +427,7 @@ class GalleryDLService:
 
         platform_section = config["extractor"].setdefault(platform, {})
 
-        if platform == "patreon":
-            if "all" in source_config.content_types:
-                platform_section["files"] = [
-                    "images", "image_large", "attachments", "postfile", "content",
-                ]
-            else:
-                platform_section["files"] = source_config.content_types
-            # Cursor-paged backfill: resume the newest→oldest walk from the
-            # checkpoint saved by the previous run instead of restarting from
-            # the top. Without this, a large catalog (Anduo's weekly
-            # image-dense Reports) never reaches its frontier inside the
-            # subprocess budget — the HEAD walk alone times out, 0 files
-            # written, no progress (event #40411). The PLATFORM_DEFAULTS leave
-            # `cursor: True` (log-only) for the first/fresh run.
-            if source_config.resume_cursor:
-                platform_section["cursor"] = source_config.resume_cursor
-        elif platform == "hentaifoundry":
+        if platform == "hentaifoundry":
             if "pictures" in source_config.content_types or "all" in source_config.content_types:
                 platform_section["include"] = "all"
 
@@ -552,10 +587,6 @@ class GalleryDLService:
         if not written_paths:
             return quarantined_relpaths, failures
 
-        quarantine_root = (
-            self.images_root / "_quarantine" / artist_slug / platform
-        )
-
         for path in written_paths:
             if not is_validatable(path):
                 continue
@@ -566,34 +597,14 @@ class GalleryDLService:
                 continue
             if result.ok:
                 continue
-            try:
-                quarantine_root.mkdir(parents=True, exist_ok=True)
-                dest = quarantine_root / path.name
-                counter = 1
-                while dest.exists():
-                    dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}"
-                    counter += 1
-                path.rename(dest)
-                sidecar = dest.with_suffix(dest.suffix + ".quarantine.json")
-                sidecar.write_text(
-                    json.dumps(
-                        {
-                            "original_path": str(path),
-                            "source_url": url,
-                            "artist_slug": artist_slug,
-                            "platform": platform,
-                            "format": result.format,
-                            "reason": result.reason,
-                            "size": result.size,
-                            "quarantined_at": datetime.now(UTC).isoformat(),
-                        },
-                        indent=2,
-                    )
-                )
-            except OSError as exc:
-                log.error("Failed to quarantine %s: %s. File left in place.", path, exc)
-                continue
-
+            # Shared move+sidecar (file_validator.quarantine_file) — same impl the
+            # native ingester uses, so the layout + provenance sidecar can't drift.
+            dest = quarantine_file(
+                self.images_root, path, artist_slug, platform,
+                url=url, result=result,
+            )
+            if dest is None:
+                continue  # move failed → left in place, not counted
             log.warning(
                 "Quarantined corrupt file: %s → %s (%s: %s)",
                 path, dest, result.format, result.reason,
@@ -631,41 +642,24 @@ class GalleryDLService:
             if "][warning]" in line.lower() and "not allowed to view post" in line.lower()
         )
 
-        return {
-            "exit_code": return_code,
-            "downloaded_count": self._count_downloaded_files(stdout),
-            "skipped_count": skipped_stdout + skipped_stderr,
-            "per_item_failures": per_item_failures,
-            "warning_count": warning_count,
-            "tier_gated_count": tier_gated_count,
-        }
+        return make_run_stats(
+            exit_code=return_code,
+            downloaded_count=self._count_downloaded_files(stdout),
+            skipped_count=skipped_stdout + skipped_stderr,
+            per_item_failures=per_item_failures,
+            warning_count=warning_count,
+            tier_gated_count=tier_gated_count,
+        )
 
     @staticmethod
     def _extract_errors_warnings(stderr: str) -> str:
-        if not stderr:
-            return ""
-        kept = [
-            line for line in stderr.splitlines()
-            if "][error]" in line.lower() or "][warning]" in line.lower()
-        ]
-        return "\n".join(kept)
+        # Thin delegator to the module-level helper (kept for existing callers).
+        return extract_errors_warnings(stderr)
 
     @staticmethod
     def _truncate_log(text: str, max_bytes: int = 500_000) -> str:
-        if not text:
-            return text
-        encoded = text.encode("utf-8")
-        if len(encoded) <= max_bytes:
-            return text
-        half = max_bytes // 2
-        head = encoded[:half].decode("utf-8", errors="ignore")
-        tail = encoded[-half:].decode("utf-8", errors="ignore")
-        head_lines = head.count("\n")
-        tail_lines = tail.count("\n")
-        total_lines = text.count("\n")
-        elided = max(0, total_lines - head_lines - tail_lines)
-        marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
-        return head + marker + tail
+        # Thin delegator to the module-level helper (kept for existing callers).
+        return truncate_log(text, max_bytes)
 
     async def download(
         self,

backend/app/services/importer.py

@@ -31,7 +31,12 @@ from ..models import (
     Source,
 )
 from ..utils import safe_probe
-from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name
+from ..utils.paths import (
+    derive_subdir,
+    derive_top_level_artist,
+    hash_suffixed_name,
+    safe_ext,
+)
 from ..utils.phash import compute_phash, find_similar
 from ..utils.sidecar import find_sidecar, parse_sidecar
 from ..utils.slug import slugify
@@ -82,27 +87,9 @@ def is_video(path: Path) -> bool:
 
 def _safe_ext(path: Path) -> str:
     """Conservatively extract a file extension for PostAttachment.ext
-    (varchar(32)).
-
-    gallery-dl produces some filenames with URL-encoded query-string
-    artifacts embedded into the basename (e.g.
-    `79507046_media_..._https___www.patreon.com_media-u_Z0FBQUFBQm5q...`).
-    `Path.suffix` finds the LAST dot and returns everything after, which
-    in those cases yields a 50+ char "extension" of mostly base64-ish
-    junk. That blows the column. Operator-flagged 2026-05-25.
-
-    Real extensions are short and alphanumeric. We accept anything ≤ 16
-    chars where every post-dot character is alphanumeric; anything else
-    means the input wasn't a real extension and we return the empty
-    string. ext is nullable-ish (empty string still satisfies NOT NULL)
-    and consumers should treat "" as "no known extension".
-    """
-    suffix = path.suffix.lower()
-    if not suffix or len(suffix) > 16:
-        return ""
-    if not all(c.isalnum() for c in suffix[1:]):
-        return ""
-    return suffix
+    (varchar(32)). Thin wrapper over the shared `utils.paths.safe_ext` (kept for
+    the Path-typed call sites + the [[path_suffix_sanitize]] memory pointer)."""
+    return safe_ext(path)
 
 
 def _mime_for(path: Path) -> str:

backend/app/services/ingest_core.py

@@ -0,0 +1,600 @@
+"""Platform-agnostic native-ingest core (plan #706, build on #697/#703/#704/#705).
+
+The orchestration that drives a native subscription walk — page a feed →
+extract media → tiered skip (seen-ledger / on-disk / dead-letter) → download →
+mark-seen / record-failures / checkpoint-cursor → return a gallery-dl-shaped
+`DownloadResult`, across tick/backfill/recovery modes — is identical for every
+platform. Only four things are platform-specific, and they're INJECTED at
+construction by a thin adapter (e.g. `PatreonIngester`):
+
+  - `client`    — `.iter_posts(feed_id, cursor)` yielding `(post, included,
+                  page_cursor)` + `.extract_media(post, included) -> [media]`.
+  - `downloader`— `.download_post(post, media, artist_slug, is_seen) ->
+                  [MediaOutcome]` (status in downloaded/skipped_seen/skipped_disk
+                  /quarantined/error; `.path`/`.error`/`.post_id`).
+  - ledger      — `seen_model` + `failed_model` SQLAlchemy models (+ their
+                  on-conflict UNIQUE constraint names) and a `ledger_key(media)`.
+  - failure map — the adapter overrides `_failure_result` (platform exception
+                  → DownloadResult.error_type) and supplies `error_base` (the
+                  exception type the walk catches) + `platform` (result label).
+
+Everything DB touches a SHORT-LIVED sync session from the injected sessionmaker —
+never held across a network fetch ([[db-connection-held-across-subprocess]]).
+Plain-HTTP homelab: no secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from collections.abc import Callable
+
+from sqlalchemy import delete, func, select, text
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+from .gallery_dl import DownloadResult, ErrorType, make_run_stats
+
+log = logging.getLogger(__name__)
+
+# Stop a tick after this many CONTIGUOUS already-have-it media (seen-ledger or
+# on-disk) — the cheap native equivalent of gallery-dl's `exit:20`, now free of
+# per-file HEADs. Headroom against paywalled/undownloadable items interleaving.
+_TICK_SEEN_THRESHOLD = 20
+
+# plan #705 #7: after this many failed download/validate attempts a media is
+# "dead-lettered" and skipped on routine tick/backfill walks (recovery still
+# re-attempts it). Stops a permanently-broken media re-erroring forever.
+DEAD_LETTER_THRESHOLD = 3
+# last_error is Text but bound it so a giant traceback doesn't bloat the row.
+_ERROR_MAX = 1000
+
+
+class Ingester:
+    """Generic native-ingest orchestration. Subclass with a platform adapter
+    (see the module docstring) — or construct directly with the keyword seams."""
+
+    def __init__(
+        self,
+        *,
+        client,
+        downloader,
+        session_factory: Callable[[], object],
+        seen_model,
+        failed_model,
+        seen_constraint: str,
+        failed_constraint: str,
+        ledger_key: Callable[[object], str],
+        platform: str,
+        error_base: type[Exception],
+    ):
+        self.client = client
+        self.downloader = downloader
+        self.session_factory = session_factory
+        self._seen_model = seen_model
+        self._failed_model = failed_model
+        self._seen_constraint = seen_constraint
+        self._failed_constraint = failed_constraint
+        self._ledger_key = ledger_key
+        self._platform = platform
+        self._error_base = error_base
+
+    # -- public ------------------------------------------------------------
+
+    def run(
+        self,
+        *,
+        source_id: int,
+        campaign_id: str,
+        artist_slug: str,
+        url: str,
+        mode: str,
+        resume_cursor: str | None = None,
+        time_budget_seconds: float = 870.0,
+        seen_threshold: int = _TICK_SEEN_THRESHOLD,
+        posts_base: int = 0,
+    ) -> DownloadResult:
+        """Walk + download for one source, returning a gallery-dl-shaped result.
+
+        `mode` is "tick" | "backfill" | "recovery". Recovery bypasses the tier-1
+        seen-ledger AND the dead-letter ledger (tier-2 disk still skips kept
+        files). The walk stops on:
+          - budget exhaustion (time_budget_seconds)         → TIMEOUT / PARTIAL
+          - tick early-out (seen_threshold contiguous seen)  → success
+          - reaching the bottom of the feed                  → success (rc 0)
+        A client-level failure (drift / auth / network) fails the whole run loud.
+        """
+        bypass_seen = mode == "recovery"
+        # Only deep walks checkpoint their cursor mid-flight (plan #705 #6); a
+        # tick has no resumable backfill state.
+        checkpoint = mode in ("backfill", "recovery")
+        ledger_key = self._ledger_key
+        start = time.monotonic()
+        log_lines: list[str] = []
+        written: list[str] = []
+        quarantined_paths: list[str] = []
+        downloaded = 0
+        errors = 0
+        quarantined = 0
+        dead_lettered = 0
+        skipped_count = 0
+        posts_processed = 0
+        # Net-new posts THIS chunk for the live progress badge (plan #704 #5);
+        # excludes the re-walked resume page so _backfill_posts stays a monotonic
+        # absolute across chunks instead of an inflating sum. posts_processed
+        # stays the gross per-chunk count used for the run summary.
+        chunk_new_posts = 0
+        consecutive_seen = 0
+        emitted_cursor: str | None = None
+        reached_bottom = False
+        budget_hit = False
+        early_out = False
+        stopped = False  # plan #708 B4: operator hit Stop mid-walk
+        cancel_armed = False  # latched once we observe a live "running" state
+
+        def _result(
+            *, success: bool, return_code: int,
+            error_type: ErrorType | None, error_message: str | None,
+        ) -> DownloadResult:
+            # plan #704: return STRUCTURED data — phase 3 reads run_stats/cursor
+            # directly instead of regex-scraping a reconstructed stdout. stdout
+            # stays a human-readable summary (no fake `Cursor:` lines).
+            return DownloadResult(
+                success=success,
+                url=url,
+                artist_slug=artist_slug,
+                platform=self._platform,
+                files_downloaded=downloaded,
+                files_quarantined=quarantined,
+                quarantined_paths=list(quarantined_paths),
+                written_paths=written,
+                stdout="\n".join(log_lines),
+                stderr="",
+                return_code=return_code,
+                error_type=error_type,
+                error_message=error_message,
+                duration_seconds=time.monotonic() - start,
+                cursor=emitted_cursor,
+                posts_processed=posts_processed,
+                run_stats=make_run_stats(
+                    exit_code=return_code,
+                    downloaded_count=downloaded,
+                    skipped_count=skipped_count,
+                    per_item_failures=errors,
+                    quarantined_count=quarantined,
+                    dead_lettered_count=dead_lettered,
+                ),
+            )
+
+        try:
+            for post, included, page_cursor in self.client.iter_posts(
+                campaign_id, cursor=resume_cursor
+            ):
+                # Checkpoint the cursor that FETCHED this page the moment we
+                # START it — so a chunk cut mid-page resumes the page, not the one
+                # after it. Carried as DownloadResult.cursor (plan #704).
+                if page_cursor and page_cursor != emitted_cursor:
+                    emitted_cursor = page_cursor
+                    # plan #705 #6: persist the cursor at each page boundary so a
+                    # worker SIGKILL mid-chunk resumes near the crash, not the
+                    # chunk start. (phase 3 still writes the final cursor — same
+                    # value; this is the crash-safety net.) plan #704 #5: persist
+                    # the live posts count alongside it so the badge climbs DURING
+                    # the chunk, not only when it ends.
+                    if checkpoint:
+                        # plan #708 B4: an operator Stop pops `_backfill_state` —
+                        # bail at the page boundary (progress already checkpointed)
+                        # before more network work, so the live chunk halts
+                        # promptly instead of running to its time-box. LATCH on the
+                        # first observed "running" state, so a run invoked WITHOUT a
+                        # running state (a unit test, or a stale call) never
+                        # spuriously self-cancels. A short SELECT, never held.
+                        if self._still_running(source_id):
+                            cancel_armed = True
+                        elif cancel_armed:
+                            stopped = True
+                            break
+                        self._checkpoint_cursor(source_id, emitted_cursor)
+                        self._checkpoint_posts(source_id, posts_base + chunk_new_posts)
+
+                # Time-box check at the post boundary (coarse, like a gallery-dl
+                # chunk). Backfill/recovery resume from emitted_cursor next chunk.
+                if time.monotonic() - start >= time_budget_seconds:
+                    budget_hit = True
+                    break
+
+                posts_processed += 1
+                # The resume page (its cursor == resume_cursor) was already
+                # counted by the chunk that checkpointed it — don't re-count it
+                # into the persisted badge (plan #704 #5). First chunk has
+                # resume_cursor None, so everything counts.
+                if not (resume_cursor and page_cursor == resume_cursor):
+                    chunk_new_posts += 1
+                media = self.client.extract_media(post, included)
+                if not media:
+                    continue
+
+                keys = [ledger_key(m) for m in media]
+                # Recovery bypasses BOTH the seen-ledger AND the dead-letter
+                # ledger (the operator's "try everything again"); routine walks
+                # skip seen + dead media (tier-1 + tier-1.5, plan #705 #7).
+                dead = set() if bypass_seen else self._dead_keys(source_id, keys)
+                seen = (
+                    set()
+                    if bypass_seen
+                    else self._seen_keys(source_id, keys)
+                )
+                skip = seen | dead
+
+                def _is_skip(m, _skip=skip) -> bool:
+                    return ledger_key(m) in _skip
+
+                outcomes = self.downloader.download_post(
+                    post, media, artist_slug, is_seen=_is_skip
+                )
+
+                to_mark: list[tuple[str, str]] = []
+                to_clear: list[str] = []  # recovered → drop any dead-letter row
+                to_fail: list[tuple[str, str, str]] = []  # (key, post_id, error)
+                for media_item, outcome in zip(media, outcomes, strict=False):
+                    key = ledger_key(media_item)
+                    if key in dead:
+                        dead_lettered += 1  # skipped because previously dead
+                    if outcome.status == "downloaded":
+                        downloaded += 1
+                        if outcome.path is not None:
+                            written.append(str(outcome.path))
+                        to_mark.append((key, media_item.post_id))
+                        to_clear.append(key)
+                        consecutive_seen = 0
+                    elif outcome.status == "skipped_disk":
+                        # Already on disk (a prior run). Reconcile the ledger so a
+                        # later tick skips it at tier-1 without a disk stat, but
+                        # do NOT re-feed it to phase 3 — attach_in_place would see
+                        # the duplicate sha256 and unlink the on-disk copy.
+                        to_mark.append((key, media_item.post_id))
+                        to_clear.append(key)
+                        skipped_count += 1
+                        consecutive_seen += 1
+                    elif outcome.status == "skipped_seen":
+                        skipped_count += 1
+                        consecutive_seen += 1
+                    elif outcome.status == "quarantined":
+                        # New content that failed validation (corrupt) — counted
+                        # distinctly so the run surfaces a real quarantined total.
+                        # Not marked seen (a later walk may re-fetch a fixed file);
+                        # it IS new content, so it breaks the run-of-seen. Counts
+                        # toward the dead-letter ledger (plan #705 #7).
+                        quarantined += 1
+                        if outcome.path is not None:
+                            quarantined_paths.append(str(outcome.path))
+                        to_fail.append((key, media_item.post_id, outcome.error or "quarantined"))
+                        consecutive_seen = 0
+                    elif outcome.status == "error":
+                        errors += 1
+                        to_fail.append((key, media_item.post_id, outcome.error or "error"))
+                        # An error neither advances nor resets the run-of-seen.
+
+                    if mode == "tick" and consecutive_seen >= seen_threshold:
+                        early_out = True
+                        break
+
+                # Persist ledger changes AFTER the network fetch, on short
+                # sessions: mark downloaded/on-disk seen, clear any dead-letter
+                # for recovered media, and record failures (plan #705 #7).
+                if to_mark:
+                    self._mark_seen(source_id, to_mark)
+                if to_clear:
+                    self._clear_failures(source_id, to_clear)
+                if to_fail:
+                    self._record_failures(source_id, to_fail)
+
+                if early_out:
+                    break
+            else:
+                reached_bottom = True
+        except self._error_base as exc:
+            # The platform's client-error base — _failure_result (adapter)
+            # maps it to a typed error.
+            return self._failure_result(exc, _result)
+
+        # plan #708 B4: a Stop already popped the backfill state (incl. cursor +
+        # posts), so don't re-write them — return PARTIAL (reads as "ok/progress",
+        # the lifecycle no-ops since state is gone) instead of a false "complete".
+        if stopped:
+            return _result(
+                success=False, return_code=-1,
+                error_type=ErrorType.PARTIAL,
+                error_message=f"Stopped by operator: {downloaded} file(s) this chunk",
+            )
+
+        # Final authoritative posts count for the badge — captures the last page
+        # after the last boundary write and the time-box break (plan #704 #5).
+        if checkpoint:
+            self._checkpoint_posts(source_id, posts_base + chunk_new_posts)
+
+        if errors:
+            log_lines.append(f"{errors} media item(s) failed")
+        if quarantined:
+            log_lines.append(f"{quarantined} media item(s) quarantined (invalid)")
+        if dead_lettered:
+            log_lines.append(f"{dead_lettered} media item(s) skipped (dead-lettered)")
+        log_lines.append(
+            f"{self._platform} ingest ({mode}): {downloaded} downloaded, "
+            f"{skipped_count} skipped, {quarantined} quarantined, "
+            f"{dead_lettered} dead-lettered, {errors} error(s), "
+            f"{posts_processed} post(s)"
+            + (", reached end" if reached_bottom else "")
+            + (", time-boxed" if budget_hit else "")
+        )
+
+        if budget_hit:
+            # A chunk that hit its time-box but made forward progress is a
+            # NORMAL chunk boundary, not a failure (PARTIAL → status "ok"); the
+            # next chunk resumes from the emitted cursor. No progress → TIMEOUT,
+            # which feeds download_service's backfill stall-guard. rc<0 mirrors
+            # subprocess TimeoutExpired so completion detection stays false.
+            made_progress = downloaded > 0 or emitted_cursor != resume_cursor
+            if made_progress:
+                return _result(
+                    success=False, return_code=-1,
+                    error_type=ErrorType.PARTIAL,
+                    error_message=(
+                        f"Backfill chunk: {downloaded} file(s) — continuing"
+                    ),
+                )
+            return _result(
+                success=False, return_code=-1,
+                error_type=ErrorType.TIMEOUT,
+                error_message="Chunk timed out with no progress",
+            )
+
+        # Normal success: reached the bottom, or a tick that early-outed. rc 0 +
+        # error_type None is REQUIRED for a backfill/recovery walk that reached
+        # the bottom to be marked COMPLETE by
+        # download_service._apply_backfill_lifecycle — so we return None even
+        # when downloaded == 0 (a re-confirming walk that found nothing new still
+        # completed). success=True maps to status "ok" regardless. A tick that
+        # early-outed also returns here; ticks never set backfill state so the
+        # lifecycle is a no-op for them.
+        return _result(
+            success=True, return_code=0,
+            error_type=None, error_message=None,
+        )
+
+    # -- preview (dry-run) -------------------------------------------------
+
+    def preview(
+        self,
+        source_id: int,
+        campaign_id: str,
+        *,
+        page_limit: int = 3,
+        sample_size: int = 10,
+    ) -> dict:
+        """Dry-run (plan #708 B4): walk up to `page_limit` pages and count media
+        NOT already in the seen/dead ledgers, WITHOUT downloading anything.
+
+        Read-only — only the seen/dead SELECTs touch the DB (short sessions). Lets
+        an operator gauge "is this source worth a backfill?" cheaply. Returns:
+            {total_new, posts_scanned, pages_scanned, has_more,
+             sample: [{title, date, new}, ...]}  # sample = posts with new media
+        A client-level failure (auth/drift) propagates to the caller.
+        """
+        total_new = 0
+        posts_scanned = 0
+        pages_scanned = 0
+        has_more = False
+        sample: list[dict] = []
+        unset = object()
+        last_page: object = unset
+        for post, included, page_cursor in self.client.iter_posts(
+            campaign_id, cursor=None
+        ):
+            if page_cursor != last_page:
+                last_page = page_cursor
+                pages_scanned += 1
+                if pages_scanned > page_limit:
+                    has_more = True
+                    pages_scanned = page_limit
+                    break
+            posts_scanned += 1
+            media = self.client.extract_media(post, included)
+            if not media:
+                continue
+            keys = [self._ledger_key(m) for m in media]
+            skip = self._seen_keys(source_id, keys) | self._dead_keys(source_id, keys)
+            new_count = sum(1 for m in media if self._ledger_key(m) not in skip)
+            total_new += new_count
+            if new_count > 0 and len(sample) < sample_size:
+                meta = self.client.post_meta(post)
+                sample.append(
+                    {
+                        "title": meta.get("title") or "(untitled)",
+                        "date": meta.get("date"),
+                        "new": new_count,
+                    }
+                )
+        return {
+            "total_new": total_new,
+            "posts_scanned": posts_scanned,
+            "pages_scanned": pages_scanned,
+            "has_more": has_more,
+            "sample": sample,
+        }
+
+    # -- failure mapping (adapter overrides) -------------------------------
+
+    def _failure_result(self, exc: Exception, _result) -> DownloadResult:
+        """Map a platform client-error to a typed failed DownloadResult. The base
+        gives a safe default; adapters override with their exception taxonomy."""
+        log.warning("%s ingest failed: %s", self._platform, exc)
+        return _result(
+            success=False, return_code=1,
+            error_type=ErrorType.UNKNOWN_ERROR, error_message=str(exc),
+        )
+
+    # -- seen-ledger (short-lived sessions) --------------------------------
+
+    def _seen_keys(self, source_id: int, keys: list[str]) -> set[str]:
+        """Which of `keys` are already in the seen-ledger for this source.
+
+        One short SELECT on its own session — opened and closed without any
+        network in between (the GETs happen after, in download_post).
+        """
+        if not keys:
+            return set()
+        with self.session_factory() as session:
+            rows = session.execute(
+                select(self._seen_model.filehash).where(
+                    self._seen_model.source_id == source_id,
+                    self._seen_model.filehash.in_(keys),
+                )
+            ).scalars().all()
+        return set(rows)
+
+    def _checkpoint_cursor(self, source_id: int, cursor: str) -> None:
+        """Persist the in-progress backfill cursor mid-walk (plan #705 #6).
+
+        ATOMIC, single-key UPDATE: cast the JSON column to jsonb, set just
+        `_backfill_cursor`, cast back — so it never clobbers operator config or
+        the other backfill keys (no read-modify-write race). The in-flight guard
+        means only this source's one download runs at a time; a concurrent
+        operator stop is benign (a stray cursor with no `_backfill_state` is
+        ignored by tick mode and cleared on the next start).
+        """
+        with self.session_factory() as session:
+            session.execute(
+                text(
+                    "UPDATE source SET config_overrides = jsonb_set("
+                    "  coalesce(config_overrides::jsonb, '{}'::jsonb),"
+                    "  '{_backfill_cursor}', to_jsonb(cast(:cur AS text))"
+                    ")::json WHERE id = :sid"
+                ),
+                {"cur": cursor, "sid": source_id},
+            )
+            session.commit()
+
+    def _still_running(self, source_id: int) -> bool:
+        """True while the source is armed for a deep walk (plan #708 B4).
+
+        An operator Stop (`source_service.stop_backfill`) pops `_backfill_state`,
+        so a False here means "cancel this chunk now". One short SELECT on its own
+        session — never held across the walk
+        ([[db-connection-held-across-subprocess]])."""
+        with self.session_factory() as session:
+            state = session.execute(
+                text(
+                    "SELECT config_overrides::jsonb ->> '_backfill_state' "
+                    "FROM source WHERE id = :sid"
+                ),
+                {"sid": source_id},
+            ).scalar_one_or_none()
+        return state == "running"
+
+    def _checkpoint_posts(self, source_id: int, posts: int) -> None:
+        """Persist the live backfill posts-processed count mid-walk (plan #704 #5).
+
+        Same atomic single-key jsonb_set dance as _checkpoint_cursor, on the
+        `_backfill_posts` key (cast to a JSON number) — so the progress badge
+        climbs DURING a chunk without clobbering operator config or the cursor.
+        The ingester OWNS this key now; download_service no longer accumulates it
+        post-chunk (which lagged a whole chunk and over-counted the resume page).
+        """
+        with self.session_factory() as session:
+            session.execute(
+                text(
+                    "UPDATE source SET config_overrides = jsonb_set("
+                    "  coalesce(config_overrides::jsonb, '{}'::jsonb),"
+                    "  '{_backfill_posts}', to_jsonb(cast(:posts AS int))"
+                    ")::json WHERE id = :sid"
+                ),
+                {"posts": posts, "sid": source_id},
+            )
+            session.commit()
+
+    def _mark_seen(self, source_id: int, items: list[tuple[str, str]]) -> None:
+        """Idempotent upsert of (filehash, post_id) seen-ledger rows for a page.
+
+        ON CONFLICT DO NOTHING against the (source_id, filehash) UNIQUE so a
+        re-sighting — or a concurrent walk — is a harmless no-op
+        ([[scalar_one_or_none-duplicates]]: never check-then-insert without the
+        DB constraint backing it). De-dup the batch locally first so a single
+        page can't present the same key twice to one INSERT.
+        """
+        seen_local: set[str] = set()
+        values = []
+        for key, post_id in items:
+            if key in seen_local:
+                continue
+            seen_local.add(key)
+            values.append(
+                {"source_id": source_id, "filehash": key, "post_id": post_id}
+            )
+        if not values:
+            return
+        with self.session_factory() as session:
+            stmt = pg_insert(self._seen_model).values(values)
+            stmt = stmt.on_conflict_do_nothing(constraint=self._seen_constraint)
+            session.execute(stmt)
+            session.commit()
+
+    # -- dead-letter ledger (plan #705 #7) ---------------------------------
+
+    def _dead_keys(self, source_id: int, keys: list[str]) -> set[str]:
+        """Which of `keys` have failed >= DEAD_LETTER_THRESHOLD times (dead).
+        One short SELECT; recovery never calls this (it re-attempts dead media)."""
+        if not keys:
+            return set()
+        with self.session_factory() as session:
+            rows = session.execute(
+                select(self._failed_model.filehash).where(
+                    self._failed_model.source_id == source_id,
+                    self._failed_model.filehash.in_(keys),
+                    self._failed_model.attempts >= DEAD_LETTER_THRESHOLD,
+                )
+            ).scalars().all()
+        return set(rows)
+
+    def _record_failures(
+        self, source_id: int, items: list[tuple[str, str, str]]
+    ) -> None:
+        """Upsert-increment the dead-letter ledger for failed media. On conflict
+        bump `attempts` and refresh last_error/last_failed_at (UNIQUE backs the
+        upsert — no check-then-insert). De-dup the batch (one row/key, last error
+        wins)."""
+        by_key: dict[str, str] = {}
+        for key, _post_id, err in items:
+            by_key[key] = (err or "")[:_ERROR_MAX]
+        if not by_key:
+            return
+        values = [
+            {"source_id": source_id, "filehash": k, "attempts": 1, "last_error": e}
+            for k, e in by_key.items()
+        ]
+        with self.session_factory() as session:
+            stmt = pg_insert(self._failed_model).values(values)
+            stmt = stmt.on_conflict_do_update(
+                constraint=self._failed_constraint,
+                set_={
+                    "attempts": self._failed_model.attempts + 1,
+                    "last_error": stmt.excluded.last_error,
+                    "last_failed_at": func.now(),
+                },
+            )
+            session.execute(stmt)
+            session.commit()
+
+    def _clear_failures(self, source_id: int, keys: list[str]) -> None:
+        """Drop dead-letter rows for media that just downloaded cleanly — they
+        recovered. A no-op DELETE for keys that were never failing."""
+        unique = list(dict.fromkeys(keys))
+        if not unique:
+            return
+        with self.session_factory() as session:
+            session.execute(
+                delete(self._failed_model).where(
+                    self._failed_model.source_id == source_id,
+                    self._failed_model.filehash.in_(unique),
+                )
+            )
+            session.commit()

backend/app/services/patreon_client.py

@@ -0,0 +1,610 @@
+"""Native Patreon JSON:API client (build step 1 of the native ingester).
+
+Clean-room reimplementation of the Patreon `/api/posts` read path. This is a
+plain, synchronous client over `requests` — the orchestrator already wraps
+sync importer calls, so nothing here needs to be async (mirrors
+patreon_resolver.py, which wraps its sync lookup in run_in_executor at the
+call site).
+
+Scope (build step 1): fetch + page + parse only. This module is NOT wired into
+download_service yet — that is a later step. The public surface here exists so
+the later step can drive it:
+  - PatreonClient(cookies_path).iter_posts(campaign_id)
+        → (post, included_index, page_cursor)
+  - extract_media(post, included_index) → list[MediaItem]
+  - parse_cursor_from_url(url) → cursor
+
+Drift detection is loud on purpose: Patreon ships JSON:API and the shapes we
+depend on (top-level `data`, media resources carrying `file_name`/`url`) are
+the contract. If a response comes back as an HTML login page or a media
+resource is missing the fields we resolve against, we raise PatreonDriftError
+rather than silently yielding empty media — so the later import step surfaces
+"Patreon changed something" instead of "creator has no posts".
+
+FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import http.cookiejar
+import logging
+import os
+import re
+import time
+from collections.abc import Iterator
+from dataclasses import dataclass
+from html import unescape
+from pathlib import Path
+from urllib.parse import parse_qs, urlsplit
+
+import requests
+
+from ..utils.paths import safe_ext
+
+log = logging.getLogger(__name__)
+
+_POSTS_URL = "https://www.patreon.com/api/posts"
+_USER_AGENT = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
+)
+_TIMEOUT_SECONDS = 30.0
+
+# 429 backoff (plan #703): ride out a transient API rate-limit instead of
+# failing the whole walk (which would stamp RATE_LIMITED → platform-wide
+# cooldown → every Patreon source dark). Honor the server's `Retry-After`;
+# otherwise exponential base·2^(n-1), capped. Only after the retries are
+# exhausted does the 429 propagate as terminal RATE_LIMITED.
+_MAX_429_RETRIES = 3
+_BACKOFF_BASE_SECONDS = 2.0
+_BACKOFF_CAP_SECONDS = 30.0
+
+
+def _retry_after_seconds(
+    resp: requests.Response,
+    attempt: int,
+    *,
+    base: float = _BACKOFF_BASE_SECONDS,
+    cap: float = _BACKOFF_CAP_SECONDS,
+) -> float:
+    """Backoff delay for a 429: the `Retry-After` seconds header if present and
+    numeric, else exponential `base·2^(attempt-1)`, both capped. (HTTP-date form
+    of Retry-After is rare here and falls through to exponential.)"""
+    header = resp.headers.get("Retry-After")
+    if header:
+        try:
+            return min(float(header), cap)
+        except (TypeError, ValueError):
+            pass
+    return min(base * (2 ** max(0, attempt - 1)), cap)
+
+# JSON:API request contract (observed from real traffic — see module plan).
+_INCLUDE = (
+    "campaign,access_rules,attachments,attachments_media,audio,images,media,"
+    "native_video_insights,user,user_defined_tags,ti_checks"
+)
+_FIELDS_POST = (
+    "content,post_file,image,post_type,published_at,title,url,patreon_url,"
+    "current_user_can_view"
+)
+_FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name"
+_FIELDS_CAMPAIGN = "name,url"
+
+# A CDN download URL embeds a 32-char hex (MD5) path segment; that segment is
+# Patreon's stable per-file identity and is what we dedup + ledger against.
+# Same role gallery-dl's _filehash plays. Match the FIRST 32-hex run anywhere
+# in the URL (path or query); real Patreon CDN URLs carry exactly one.
+_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})")
+
+# Inline post `content` is HTML; images are emitted as <img ... src="...">.
+# Pull every src; downstream dedup collapses any that duplicate a gallery item
+# by filehash. Tolerant of attribute ordering and single/double quotes.
+_CONTENT_IMG_RE = re.compile(r"<img\b[^>]*?\bsrc=[\"']([^\"']+)[\"']", re.IGNORECASE)
+
+
+class PatreonAPIError(Exception):
+    """Base for native Patreon client failures.
+
+    `status_code` carries the HTTP status when the failure was an HTTP response
+    (None for transport-level / parse failures), so the ingester can map it to a
+    DownloadResult.error_type (429 → rate_limited, 404 → not_found, …).
+    `retry_after` carries the server's `Retry-After` seconds on a terminal 429, so
+    the platform cooldown can match the server's hint instead of a flat default
+    (plan #708 B1).
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        status_code: int | None = None,
+        retry_after: float | None = None,
+    ):
+        super().__init__(message)
+        self.status_code = status_code
+        self.retry_after = retry_after
+
+
+class PatreonAuthError(PatreonAPIError):
+    """Authentication / authorization failure — missing or expired session
+    cookies, an insufficient pledge tier, or an HTML login/challenge page served
+    where JSON was expected. DISTINCT from drift: the fix is rotating the
+    credential, not updating the ingester. Maps to error_type 'auth_error'.
+    """
+
+
+class PatreonDriftError(PatreonAPIError):
+    """A JSON response did not match the JSON:API shape we depend on.
+
+    Raised for: a missing top-level `data` list, `data` not a list, or a media
+    resource lacking the `file_name`/`url` fields we resolve against. Fail loud
+    so the import step flags API drift (ingester needs update) instead of
+    silently importing nothing. An HTML-login / non-JSON body is auth, not
+    drift — that raises PatreonAuthError.
+    """
+
+
+@dataclass
+class MediaItem:
+    """One resolved downloadable item belonging to a post.
+
+    Fields:
+      url       — the CDN/download URL to fetch.
+      filename  — media `file_name` when present; otherwise the URL basename
+                  (NEVER a network call). Bounded/sane extension.
+      kind      — one of: "images", "image_large", "attachments", "postfile",
+                  "content". Mirrors gallery-dl's `files` content-type names so
+                  the later step can honor the same per-source content_types.
+      filehash  — the 32-char hex (MD5) segment from the CDN URL, or None if
+                  the URL carries no such segment. Used for in-post dedup and
+                  the cross-run seen-ledger.
+      post_id   — the owning post's id (so a flattened media list stays
+                  traceable to its post).
+    """
+
+    url: str
+    filename: str
+    kind: str
+    filehash: str | None
+    post_id: str
+
+
+def _load_session(cookies_path: str | Path | None) -> requests.Session:
+    session = requests.Session()
+    session.headers.update(
+        {
+            "User-Agent": _USER_AGENT,
+            "Accept": "application/vnd.api+json",
+        }
+    )
+    if cookies_path and os.path.isfile(str(cookies_path)):
+        try:
+            jar = http.cookiejar.MozillaCookieJar(str(cookies_path))
+            jar.load(ignore_discard=True, ignore_expires=True)
+            session.cookies = jar  # type: ignore[assignment]
+        except (OSError, http.cookiejar.LoadError) as exc:
+            log.warning("Could not load Patreon cookies from %s: %s", cookies_path, exc)
+    return session
+
+
+def _filehash(url: str) -> str | None:
+    if not url:
+        return None
+    match = _FILEHASH_RE.search(url)
+    return match.group(1).lower() if match else None
+
+
+def _basename_from_url(url: str) -> str:
+    """Derive a sane filename from a URL when the media has no file_name.
+
+    Strips query/fragment, takes the path basename, and drops a junk
+    extension (the importer._safe_ext gotcha) so we never write base64 noise
+    as a name. Falls back to the filehash, then to "file".
+    """
+    path = urlsplit(url).path
+    base = os.path.basename(path)
+    if base:
+        ext = safe_ext(base)
+        stem = base[: -len(Path(base).suffix)] if Path(base).suffix else base
+        # Keep the stem bounded; URL-encoded stems can be enormous.
+        stem = stem[:120] or "file"
+        return f"{stem}{ext}"
+    fh = _filehash(url)
+    return fh or "file"
+
+
+def parse_cursor_from_url(url: str | None) -> str | None:
+    """Extract the `page[cursor]` query param from a links.next URL."""
+    if not url:
+        return None
+    query = urlsplit(url).query
+    values = parse_qs(query).get("page[cursor]")
+    if values and values[0]:
+        return values[0]
+    return None
+
+
+class PatreonClient:
+    """Synchronous Patreon JSON:API read client.
+
+    Construct with a path to a Netscape cookies.txt (the same file
+    CredentialService.get_cookies_path materializes). Cookies are loaded into a
+    requests.Session; no secure-context APIs are used.
+    """
+
+    def __init__(
+        self,
+        cookies_path: str | Path | None,
+        *,
+        request_sleep: float = 0.0,
+        max_retries: int = _MAX_429_RETRIES,
+    ):
+        self.cookies_path = str(cookies_path) if cookies_path else None
+        self._session = _load_session(cookies_path)
+        # Politeness: seconds to sleep before each /api/posts page fetch (paces
+        # the rate-limited API endpoint). 0 = no pacing. plan #703.
+        self._request_sleep = request_sleep or 0.0
+        self._max_retries = max_retries
+
+    # -- request -----------------------------------------------------------
+
+    def _params(self, campaign_id: str, cursor: str | None) -> dict[str, str]:
+        params = {
+            "include": _INCLUDE,
+            "fields[post]": _FIELDS_POST,
+            "fields[media]": _FIELDS_MEDIA,
+            "fields[campaign]": _FIELDS_CAMPAIGN,
+            "filter[campaign_id]": campaign_id,
+            "filter[contains_exclusive_posts]": "true",
+            "filter[is_draft]": "false",
+            "sort": "-published_at",
+            "json-api-version": "1.0",
+        }
+        if cursor:
+            params["page[cursor]"] = cursor
+        return params
+
+    def _fetch(self, campaign_id: str, cursor: str | None) -> dict:
+        if self._request_sleep > 0:
+            time.sleep(self._request_sleep)  # pace the API endpoint
+        attempt = 0
+        while True:
+            try:
+                resp = self._session.get(
+                    _POSTS_URL,
+                    params=self._params(campaign_id, cursor),
+                    timeout=_TIMEOUT_SECONDS,
+                )
+            except requests.RequestException as exc:
+                raise PatreonAPIError(
+                    f"Patreon posts request failed (campaign_id={campaign_id}): {exc}"
+                ) from exc
+
+            # Transient rate-limit: back off and retry rather than failing the
+            # whole walk. Only a PERSISTENT 429 (retries exhausted) falls
+            # through to the terminal RATE_LIMITED raise below.
+            if resp.status_code == 429 and attempt < self._max_retries:
+                attempt += 1
+                delay = _retry_after_seconds(resp, attempt)
+                log.warning(
+                    "Patreon 429 (campaign_id=%s) — backing off %.1fs (retry %d/%d)",
+                    campaign_id, delay, attempt, self._max_retries,
+                )
+                time.sleep(delay)
+                continue
+            break
+
+        if resp.status_code in (401, 403):
+            # Auth rejected — expired/missing cookies or an insufficient tier.
+            # Actionable as "rotate credentials", so it's auth, not drift/http.
+            raise PatreonAuthError(
+                f"Patreon posts API returned HTTP {resp.status_code} — auth "
+                f"rejected (cookies expired or tier insufficient; "
+                f"campaign_id={campaign_id})",
+                status_code=resp.status_code,
+            )
+        if resp.status_code != 200:
+            # A persistent 429 (retries exhausted) is terminal RATE_LIMITED — carry
+            # the server's raw Retry-After seconds so the cooldown matches its hint
+            # (plan #708 B1). Header is uncapped here; the cooldown clamps it.
+            retry_after = None
+            if resp.status_code == 429:
+                hdr = resp.headers.get("Retry-After")
+                if hdr:
+                    try:
+                        retry_after = float(hdr)
+                    except (TypeError, ValueError):
+                        retry_after = None
+            raise PatreonAPIError(
+                f"Patreon posts API returned HTTP {resp.status_code} "
+                f"(campaign_id={campaign_id})",
+                status_code=resp.status_code,
+                retry_after=retry_after,
+            )
+        try:
+            payload = resp.json()
+        except ValueError as exc:
+            # A non-JSON body here is almost always the HTML login/challenge
+            # page served when cookies are missing/expired — that is an AUTH
+            # failure (rotate cookies), not API drift (update the ingester) and
+            # not a transient network error.
+            raise PatreonAuthError(
+                "Patreon posts API returned a non-JSON response (likely an "
+                f"HTML login/challenge page — session expired; "
+                f"campaign_id={campaign_id}): {exc}"
+            ) from exc
+        return payload
+
+    # -- parsing -----------------------------------------------------------
+
+    @staticmethod
+    def _transform(response: dict) -> dict:
+        """Flatten the JSON:API `included` array for relationship resolution.
+
+        Returns a dict keyed by `(type, id)` → that resource's `attributes`
+        (so a post's relationships can be resolved in O(1)). Missing/oddly
+        shaped `included` entries are skipped rather than fatal — drift
+        detection for the top-level shape lives in _validate_response.
+        """
+        index: dict[tuple[str, str], dict] = {}
+        for inc in response.get("included") or []:
+            if not isinstance(inc, dict):
+                continue
+            rtype = inc.get("type")
+            rid = inc.get("id")
+            if rtype is None or rid is None:
+                continue
+            index[(str(rtype), str(rid))] = inc.get("attributes") or {}
+        return index
+
+    @staticmethod
+    def _validate_response(response: dict) -> None:
+        if not isinstance(response, dict):
+            raise PatreonDriftError("Patreon response was not a JSON object")
+        if "data" not in response:
+            raise PatreonDriftError("Patreon response missing top-level 'data' key")
+        data = response.get("data")
+        if not isinstance(data, list):
+            raise PatreonDriftError("Patreon response 'data' was not a list")
+
+    def _related_ids(self, post: dict, rel_name: str) -> list[str]:
+        rels = post.get("relationships") or {}
+        rel = rels.get(rel_name) or {}
+        data = rel.get("data")
+        if isinstance(data, dict):  # to-one relationship
+            data = [data]
+        if not isinstance(data, list):
+            return []
+        ids: list[str] = []
+        for ref in data:
+            if isinstance(ref, dict) and ref.get("id") is not None:
+                ids.append(str(ref["id"]))
+        return ids
+
+    @staticmethod
+    def _media_url(attrs: dict) -> str | None:
+        """Pick the best fetchable URL for a media resource.
+
+        Prefer the full-size `download_url`; fall back to the largest
+        `image_urls` size. gallery-dl prefers download_url too, only dipping
+        into image_urls when a smaller configured size is requested — FC
+        always wants the original, so download_url first.
+        """
+        download_url = attrs.get("download_url")
+        if isinstance(download_url, str) and download_url:
+            return download_url
+        image_urls = attrs.get("image_urls")
+        if isinstance(image_urls, dict):
+            for key in ("original", "full", "large", "default"):
+                candidate = image_urls.get(key)
+                if isinstance(candidate, str) and candidate:
+                    return candidate
+            # Otherwise take any non-empty string value.
+            for candidate in image_urls.values():
+                if isinstance(candidate, str) and candidate:
+                    return candidate
+        return None
+
+    def _media_item(
+        self, attrs: dict, kind: str, post_id: str, *, require_file_name: bool
+    ) -> MediaItem:
+        url = self._media_url(attrs)
+        if not url:
+            raise PatreonDriftError(
+                f"Patreon media (post {post_id}, kind={kind}) had no resolvable URL "
+                f"(no download_url / image_urls)"
+            )
+        file_name = attrs.get("file_name")
+        if require_file_name and not (isinstance(file_name, str) and file_name):
+            raise PatreonDriftError(
+                f"Patreon media resource (post {post_id}, kind={kind}) missing "
+                f"'file_name'"
+            )
+        filename = file_name if isinstance(file_name, str) and file_name else _basename_from_url(url)
+        return MediaItem(
+            url=url,
+            filename=filename,
+            kind=kind,
+            filehash=_filehash(url),
+            post_id=post_id,
+        )
+
+    def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]:
+        """Resolve all downloadable media for one post.
+
+        Walks the kinds in the same order gallery-dl does — images,
+        image_large (post cover), attachments, postfile, content (inline
+        <img>) — and dedups within the post by filehash (first wins). The
+        image_large cover commonly duplicates a gallery image; deduping by
+        filehash collapses them to the gallery item (encountered first).
+        """
+        post_id = str(post.get("id") or "")
+        attrs = post.get("attributes") or {}
+        items: list[MediaItem] = []
+
+        def _resolve_rel(rel_name: str, kind: str) -> None:
+            for mid in self._related_ids(post, rel_name):
+                media_attrs = included_index.get(("media", mid))
+                if media_attrs is None:
+                    # Referenced but not in `included`: a media id with no
+                    # resource is drift (we asked for include=media).
+                    raise PatreonDriftError(
+                        f"Patreon post {post_id} references media {mid} "
+                        f"({rel_name}) not present in 'included'"
+                    )
+                items.append(
+                    self._media_item(
+                        media_attrs, kind, post_id, require_file_name=True
+                    )
+                )
+
+        # 1. gallery images
+        _resolve_rel("images", "images")
+
+        # 2. image_large — the post-level cover (`image.large_url`). Not a
+        #    media relationship; it lives on the post attributes.
+        image = attrs.get("image")
+        if isinstance(image, dict):
+            large_url = image.get("large_url") or image.get("url")
+            if isinstance(large_url, str) and large_url:
+                items.append(
+                    MediaItem(
+                        url=large_url,
+                        filename=_basename_from_url(large_url),
+                        kind="image_large",
+                        filehash=_filehash(large_url),
+                        post_id=post_id,
+                    )
+                )
+
+        # 3. attachments
+        _resolve_rel("attachments_media", "attachments")
+
+        # 4. postfile — the post's primary attached file (`post_file`).
+        post_file = attrs.get("post_file")
+        if isinstance(post_file, dict):
+            pf_url = post_file.get("url") or post_file.get("download_url")
+            if isinstance(pf_url, str) and pf_url:
+                pf_name = post_file.get("name")
+                filename = (
+                    pf_name
+                    if isinstance(pf_name, str) and pf_name
+                    else _basename_from_url(pf_url)
+                )
+                items.append(
+                    MediaItem(
+                        url=pf_url,
+                        filename=filename,
+                        kind="postfile",
+                        filehash=_filehash(pf_url),
+                        post_id=post_id,
+                    )
+                )
+
+        # 5. content — inline <img> in the post HTML body.
+        content = attrs.get("content")
+        if isinstance(content, str) and content:
+            for raw_src in _CONTENT_IMG_RE.findall(content):
+                src = unescape(raw_src)
+                if not src:
+                    continue
+                items.append(
+                    MediaItem(
+                        url=src,
+                        filename=_basename_from_url(src),
+                        kind="content",
+                        filehash=_filehash(src),
+                        post_id=post_id,
+                    )
+                )
+
+        return _dedup_by_filehash(items)
+
+    @staticmethod
+    def post_meta(post: dict) -> dict:
+        """Title + published date for a post — for the preview sample (plan #708
+        B4). Part of the client contract `ingest_core.Ingester.preview` calls."""
+        attrs = post.get("attributes") or {}
+        title = attrs.get("title")
+        published = attrs.get("published_at")
+        return {
+            "title": title if isinstance(title, str) else None,
+            "date": published if isinstance(published, str) else None,
+        }
+
+    # -- iteration ---------------------------------------------------------
+
+    def iter_posts(
+        self, campaign_id: str, cursor: str | None = None
+    ) -> Iterator[tuple[dict, dict, str | None]]:
+        """Yield (post, included_index, page_cursor) for every post in the feed.
+
+        Pages newest→oldest via `links.next`, validating each response for
+        drift before yielding. The triple gives a caller everything it needs to
+        resolve and checkpoint without re-fetching:
+          - post           — the raw post resource.
+          - included_index — the page's flattened `included` (the same object
+                             for every post on a page), to pass straight to
+                             extract_media(post, included_index).
+          - page_cursor    — the cursor that FETCHED this post's page (None for
+                             the first page). The caller checkpoints THIS value,
+                             matching the existing backfill cursor logic where
+                             the saved cursor re-fetches the page being
+                             processed (so a chunk cut mid-page resumes the page,
+                             not the one after it).
+        """
+        current_cursor = cursor
+        while True:
+            response = self._fetch(campaign_id, current_cursor)
+            self._validate_response(response)
+            page_cursor = current_cursor
+            included_index = self._transform(response)
+            for post in response.get("data") or []:
+                if isinstance(post, dict):
+                    yield post, included_index, page_cursor
+            next_url = (response.get("links") or {}).get("next")
+            next_cursor = parse_cursor_from_url(next_url)
+            if not next_cursor:
+                return
+            current_cursor = next_cursor
+
+    # -- verify ------------------------------------------------------------
+
+    def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
+        """Cheap auth probe: fetch the first `/api/posts` page and report whether
+        the credential authenticated, WITHOUT downloading anything.
+
+        Returns `(ok, message)` matching the credential-verify contract:
+          - True  — authenticated (the feed returned a valid JSON:API page).
+          - False — the credential was rejected (PatreonAuthError: 401/403, or an
+                    HTML login page → cookies expired / tier insufficient).
+          - None  — inconclusive: API drift (our parser is stale, not a cred
+                    problem) or a transient network/HTTP error.
+        """
+        try:
+            response = self._fetch(campaign_id, None)
+            self._validate_response(response)
+        except PatreonAuthError as exc:
+            return False, f"Patreon rejected the credential — {exc}"
+        except PatreonDriftError as exc:
+            return None, f"Couldn't verify — Patreon's API shape changed: {exc}"
+        except PatreonAPIError as exc:
+            return None, f"Couldn't verify (network/HTTP issue): {exc}"
+        return True, "Credentials valid — the Patreon feed authenticated."
+
+
+def _dedup_by_filehash(items: list[MediaItem]) -> list[MediaItem]:
+    """Drop later items sharing a filehash with an earlier one (first wins).
+
+    Items with no filehash (None) are never deduped against each other — we
+    can't prove they're the same file, so keep them all.
+    """
+    seen: set[str] = set()
+    out: list[MediaItem] = []
+    for item in items:
+        if item.filehash is not None:
+            if item.filehash in seen:
+                continue
+            seen.add(item.filehash)
+        out.append(item)
+    return out

backend/app/services/patreon_downloader.py

@@ -0,0 +1,501 @@
+"""Native Patreon media downloader (build step 2b of the native ingester).
+
+Given a Patreon post and its already-resolved `MediaItem`s (from
+patreon_client.extract_media), download the media to the EXACT on-disk layout
+gallery-dl produces, write a sidecar JSON the existing importer consumes, and
+report per-media outcomes.
+
+This module is PURE: no DB. The cross-run seen-ledger and the iter_posts
+orchestration are a LATER step. The tier-1 (seen) skip is an INJECTED predicate
+(`is_seen`), so this module needs no DB and is unit-testable without network.
+
+On-disk layout (matches gallery-dl):
+  <images_root>/<artist_slug>/patreon/<DIR>/<NN>_<filename>
+  where <DIR> = "<YYYY-MM-DD>_<post_id>_<title40>" (date prefix omitted when the
+  post's published_at is unparseable), <NN> is the 1-based index of the item in
+  the post zero-padded to 2, and <filename> is MediaItem.filename. The sidecar
+  is written next to the media as <NN>_<stem>.json (media_path.with_suffix).
+
+Video: a MediaItem whose URL is a Mux/HLS stream (host stream.mux.com or path
+endswith .m3u8) is fetched with yt-dlp (subprocess), passing the same
+Referer/Origin headers gallery-dl forwards for Mux playback (see gallery_dl.py
+_get_default_config). yt-dlp may remux to a container of its own choosing, so we
+accept the actual output extension and record the real path.
+
+FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import logging
+import os
+import subprocess
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from urllib.parse import urlsplit
+
+import requests
+
+from .file_validator import is_validatable, quarantine_file, validate_file
+from .patreon_client import (
+    _BACKOFF_CAP_SECONDS,
+    _load_session,
+    _retry_after_seconds,
+)
+
+log = logging.getLogger(__name__)
+
+_TITLE_MAX = 40
+_TIMEOUT_SECONDS = 120.0
+_CHUNK = 1 << 16
+# Retry a media GET that hits a TRANSIENT failure within the same pass (plan
+# #705 #8): a transport blip (connection reset / timeout / truncated stream), a
+# 429, or a 5xx. PERMANENT failures (404 gone, 403 forbidden) fail fast straight
+# to the error/dead-letter path — no point re-fetching them. Keeps a momentary
+# network hiccup from becoming a per-item error that waits for the next walk.
+_MAX_MEDIA_RETRIES = 3
+# requests transport errors worth retrying (vs. an HTTPError, which is a real
+# server response and is classified by status code).
+_TRANSIENT_TRANSPORT_EXC = (
+    requests.ConnectionError,
+    requests.Timeout,
+    requests.exceptions.ChunkedEncodingError,
+)
+
+# Referer/Origin yt-dlp must send for Mux-hosted Patreon video. Mux's JWT
+# playback policy checks Referer/Origin on every request, so yt-dlp must send
+# Patreon's, not its own default. (gallery-dl forwarded the same headers before
+# the #697 cutover removed its Patreon path; this is now the only place they
+# live.)
+_VIDEO_HEADERS = {
+    "Referer": "https://www.patreon.com/",
+    "Origin": "https://www.patreon.com",
+}
+
+# Characters Windows/gallery-dl path-restrict forbids, plus path separators.
+_FORBIDDEN = set('<>:"/\\|?*')
+
+
+def _sanitize(name: str) -> str:
+    """Make `name` safe for a single filesystem path segment.
+
+    Replaces path separators, the Windows-forbidden set <>:"/\\|?* and control
+    characters with `_`, then strips trailing dots/spaces (gallery-dl
+    path-restrict behavior). Never returns empty (falls back to "_").
+    """
+    out = []
+    for ch in name:
+        if ch in _FORBIDDEN or ord(ch) < 32:
+            out.append("_")
+        else:
+            out.append(ch)
+    cleaned = "".join(out).rstrip(". ")
+    return cleaned or "_"
+
+
+def _is_video_url(url: str) -> bool:
+    parts = urlsplit(url)
+    if parts.hostname and parts.hostname.lower() == "stream.mux.com":
+        return True
+    return parts.path.lower().endswith(".m3u8")
+
+
+def _post_dir_name(post: dict) -> str:
+    """Build the post directory name matching gallery-dl's layout."""
+    post_id = str(post.get("id") or "")
+    attrs = post.get("attributes") or {}
+    title = attrs.get("title")
+    title = title if isinstance(title, str) else ""
+    title40 = title[:_TITLE_MAX]
+
+    published = attrs.get("published_at")
+    date_prefix = None
+    if isinstance(published, str) and published:
+        s = published.strip()
+        if s.endswith("Z"):
+            s = s[:-1] + "+00:00"
+        try:
+            dt = datetime.fromisoformat(s)
+        except ValueError:
+            dt = None
+        if dt is not None:
+            date_prefix = f"{dt:%Y-%m-%d}"
+
+    if date_prefix:
+        raw = f"{date_prefix}_{post_id}_{title40}"
+    else:
+        raw = f"{post_id}_{title40}"
+    return _sanitize(raw)
+
+
+@dataclass
+class MediaOutcome:
+    """Per-media result of a download_post pass.
+
+    status is one of: "downloaded", "skipped_seen", "skipped_disk",
+    "quarantined", "error". `path` is the final on-disk path for "downloaded"
+    (the actual yt-dlp output for video), the path that already existed for
+    "skipped_disk", or the _quarantine destination for "quarantined"; None for
+    "skipped_seen" and (usually) "error". `error` carries the failure/validation
+    reason for "error"/"quarantined", else None.
+    """
+
+    media: object  # MediaItem (avoid importing the name for a bare annotation)
+    status: str
+    path: Path | None
+    error: str | None
+
+
+class PatreonDownloader:
+    """Download resolved Patreon media to gallery-dl's on-disk layout.
+
+    PURE: no DB. The HTTP session and the yt-dlp invocation are injectable seams
+    so tests run without network or a real subprocess:
+      - pass `session=` to stub `session.get`, or monkeypatch `_fetch_to_file`.
+      - monkeypatch `_run_ytdlp` to avoid spawning yt-dlp.
+    """
+
+    def __init__(
+        self,
+        images_root: Path,
+        cookies_path: str | None = None,
+        *,
+        validate: bool = True,
+        rate_limit: float = 0.0,
+        session: requests.Session | None = None,
+    ):
+        self.images_root = Path(images_root)
+        self.cookies_path = str(cookies_path) if cookies_path else None
+        self._validate = validate
+        # Politeness: seconds to sleep before each actual media download (paces
+        # the CDN; honors ImportSettings.download_rate_limit_seconds, the same
+        # value gallery-dl used as its between-downloads `sleep`). 0 = no pacing.
+        # Applied only to real downloads, not to seen/disk skips. plan #703.
+        self._rate_limit = rate_limit or 0.0
+        # Build a cookie-loaded session the same way patreon_client does, so the
+        # CDN GETs carry the creator's auth.
+        self.session = session if session is not None else _load_session(cookies_path)
+
+    # -- public ------------------------------------------------------------
+
+    def download_post(
+        self,
+        post: dict,
+        media_items: list,
+        artist_slug: str,
+        *,
+        is_seen: Callable[[object], bool] = lambda m: False,
+    ) -> list[MediaOutcome]:
+        """Download every media item of one post; return per-item outcomes.
+
+        Builds the post directory, iterates media (1-based NN), applies the
+        two-tier skip (injected is_seen, then disk), downloads (plain GET or
+        yt-dlp for video), writes the sidecar for each freshly-downloaded item,
+        validates, and returns outcomes. Resilient: one media's failure yields
+        an "error" outcome for that item; the rest proceed.
+        """
+        post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post)
+        outcomes: list[MediaOutcome] = []
+
+        for i, media in enumerate(media_items, start=1):
+            try:
+                outcomes.append(
+                    self._download_one(post, media, post_dir, artist_slug, i, is_seen)
+                )
+            except Exception as exc:  # resilient: isolate one item's failure
+                log.warning(
+                    "Patreon media failed (post %s, item %d): %s",
+                    post.get("id"), i, exc,
+                )
+                outcomes.append(
+                    MediaOutcome(media=media, status="error", path=None, error=str(exc))
+                )
+        return outcomes
+
+    # -- per-item ----------------------------------------------------------
+
+    def _download_one(
+        self,
+        post: dict,
+        media,
+        post_dir: Path,
+        artist_slug: str,
+        index: int,
+        is_seen: Callable[[object], bool],
+    ) -> MediaOutcome:
+        # tier-1: seen ledger (injected; no DB here).
+        if is_seen(media):
+            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
+
+        nn = f"{index:02d}"
+        final_name = _sanitize(f"{nn}_{media.filename}")
+        media_path = post_dir / final_name
+
+        # tier-2: already on disk.
+        if media_path.exists():
+            return MediaOutcome(
+                media=media, status="skipped_disk", path=media_path, error=None
+            )
+        # Video may land at a different extension; honor a pre-existing remux.
+        if _is_video_url(media.url):
+            existing = self._existing_video_output(media_path)
+            if existing is not None:
+                return MediaOutcome(
+                    media=media, status="skipped_disk", path=existing, error=None
+                )
+
+        post_dir.mkdir(parents=True, exist_ok=True)
+
+        # Pace real downloads only (the skips above already returned). plan #703.
+        if self._rate_limit > 0:
+            time.sleep(self._rate_limit)
+
+        if _is_video_url(media.url):
+            out_path = self._run_ytdlp(media.url, media_path, _VIDEO_HEADERS)
+            if out_path is None or not Path(out_path).exists():
+                return MediaOutcome(
+                    media=media,
+                    status="error",
+                    path=None,
+                    error="yt-dlp produced no output",
+                )
+            out_path = Path(out_path)
+        else:
+            out_path = self._fetch_get(media.url, media_path)
+            reason, quarantine_dest = self._validate_path(
+                out_path, artist_slug, media.url
+            )
+            if reason is not None:
+                # Quarantined (corrupt/invalid) — distinct from a download error
+                # so the run can report a real files_quarantined count + paths.
+                return MediaOutcome(
+                    media=media, status="quarantined",
+                    path=quarantine_dest, error=reason,
+                )
+
+        self._write_sidecar(post, out_path)
+        return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
+
+    # -- download seams ----------------------------------------------------
+
+    def _fetch_get(self, url: str, dest: Path) -> Path:
+        """Stream `url` to a .part file then atomic-rename to `dest`.
+
+        Thin wrapper over `_fetch_to_file` so tests can stub either the whole
+        GET path (`_fetch_to_file`) or just `session.get`.
+        """
+        part = dest.with_name(dest.name + ".part")
+        try:
+            self._fetch_to_file(url, part)
+        except Exception:
+            with contextlib.suppress(OSError):
+                part.unlink()
+            raise
+        os.replace(part, dest)
+        return dest
+
+    def _fetch_to_file(self, url: str, dest: Path) -> None:
+        """Stream a non-video URL to `dest` via the (stubbable) session, retrying
+        TRANSIENT failures within the same pass (plan #705 #8) and RESUMING from
+        the bytes already on disk via a Range request when a retry follows a
+        mid-download cut (plan #708 B5).
+
+        Retried (backoff): transport blips (connection reset / timeout /
+        truncated stream — incl. mid-download), HTTP 429 (honoring Retry-After),
+        and 5xx. Failed fast (no retry → HTTPError → per-item error → dead-letter
+        path): 4xx other than 429 (404 gone, 403 forbidden) — re-fetching a
+        permanent failure is pointless.
+
+        Resume: on a retry, if bytes already landed in `dest`, ask for the rest
+        with `Range: bytes=<have>-`. A 206 means the server honored it → append; a
+        200 means it ignored it (served the whole file) → start clean. The caller
+        (_fetch_get) stages into a `.part`, so a non-range server never corrupts
+        the output — the worst case is re-downloading from zero, as before.
+        """
+        attempt = 0
+        while True:
+            have = dest.stat().st_size if dest.exists() else 0
+            headers = {"Range": f"bytes={have}-"} if have > 0 else None
+            try:
+                resp = self.session.get(
+                    url, stream=True, timeout=_TIMEOUT_SECONDS, headers=headers,
+                )
+                if (resp.status_code == 429 or resp.status_code >= 500) \
+                        and attempt < _MAX_MEDIA_RETRIES:
+                    attempt += 1
+                    delay = _retry_after_seconds(resp, attempt)
+                    log.warning(
+                        "Patreon media transient HTTP %d (%s) — backing off "
+                        "%.1fs (retry %d/%d)",
+                        resp.status_code, url, delay, attempt, _MAX_MEDIA_RETRIES,
+                    )
+                    time.sleep(delay)
+                    continue
+                # A Range that starts at/past EOF (we already have the whole file)
+                # comes back 416 — the bytes we kept ARE the file.
+                if have > 0 and resp.status_code == 416:
+                    return
+                # 2xx → ok; 4xx-non-429 (or an exhausted 429/5xx) → HTTPError
+                # (permanent for this pass) → not caught below → per-item error.
+                resp.raise_for_status()
+                # 206 → server honored the Range; append after the kept bytes.
+                # Anything else (200) → it served the whole file → start clean.
+                mode = "ab" if (have > 0 and resp.status_code == 206) else "wb"
+                with open(dest, mode) as fh:
+                    for chunk in resp.iter_content(chunk_size=_CHUNK):
+                        if chunk:
+                            fh.write(chunk)
+                return
+            except _TRANSIENT_TRANSPORT_EXC as exc:
+                if attempt >= _MAX_MEDIA_RETRIES:
+                    raise  # exhausted → terminal error outcome
+                attempt += 1
+                delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS)
+                log.warning(
+                    "Patreon media transport error (%s) — backing off %.1fs "
+                    "(retry %d/%d): %s",
+                    url, delay, attempt, _MAX_MEDIA_RETRIES, exc,
+                )
+                time.sleep(delay)
+
+    def _run_ytdlp(self, url: str, dest: Path, headers: dict) -> Path | None:
+        """Invoke yt-dlp to fetch a Mux/HLS stream to (around) `dest`.
+
+        Returns the actual output path (yt-dlp may remux to a different
+        container, so we resolve the real file afterward). Overridden/
+        monkeypatched in tests to avoid spawning a real subprocess.
+
+        The output template uses `dest` without its extension; yt-dlp appends
+        the chosen container extension. We pass Referer/Origin (Mux JWT policy)
+        and the cookies file.
+
+        Mirrors the GET path's transient/permanent split (plan #705 #8): a hung
+        fetch (TimeoutExpired) or a spawn failure (OSError) is TRANSIENT — back
+        off and retry. A non-zero yt-dlp exit (CalledProcessError) is treated as
+        PERMANENT for this pass — yt-dlp already does its OWN internal network
+        retries, so a non-zero exit is effectively a real failure (private/gone/
+        geo-blocked), like a 4xx on the GET path: fail fast to the per-item error
+        → dead-letter path.
+        """
+        dest = Path(dest)
+        out_template = str(dest.with_suffix("")) + ".%(ext)s"
+        cmd = ["yt-dlp", "--no-progress", "-o", out_template]
+        for key, value in headers.items():
+            cmd += ["--add-header", f"{key}:{value}"]
+        if self.cookies_path and os.path.isfile(self.cookies_path):
+            cmd += ["--cookies", self.cookies_path]
+        cmd.append(url)
+        attempt = 0
+        while True:
+            try:
+                subprocess.run(
+                    cmd,
+                    check=True,
+                    capture_output=True,
+                    text=True,
+                    timeout=_TIMEOUT_SECONDS,
+                )
+                break
+            except subprocess.CalledProcessError as exc:
+                # Permanent for this pass — fail fast (no retry).
+                log.warning(
+                    "yt-dlp failed (exit %s) for %s: %s",
+                    exc.returncode, url, (exc.stderr or "").strip() or exc,
+                )
+                return None
+            except (OSError, subprocess.TimeoutExpired) as exc:
+                # Transient — back off and retry, like a transport blip on a GET.
+                if attempt >= _MAX_MEDIA_RETRIES:
+                    log.warning(
+                        "yt-dlp transient failure exhausted for %s: %s", url, exc
+                    )
+                    return None
+                attempt += 1
+                delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS)
+                log.warning(
+                    "yt-dlp transient failure (%s) — backing off %.1fs "
+                    "(retry %d/%d): %s",
+                    url, delay, attempt, _MAX_MEDIA_RETRIES, exc,
+                )
+                time.sleep(delay)
+        return self._existing_video_output(dest)
+
+    def _existing_video_output(self, dest: Path) -> Path | None:
+        """Find a yt-dlp output for `dest` regardless of chosen extension.
+
+        Returns `dest` itself if present, else any sibling sharing the same
+        stem (the remuxed container). None if nothing matched.
+        """
+        dest = Path(dest)
+        if dest.exists():
+            return dest
+        stem = dest.with_suffix("").name
+        parent = dest.parent
+        if not parent.is_dir():
+            return None
+        for cand in sorted(parent.iterdir()):
+            if cand.is_file() and cand.stem == stem and cand.name != dest.name:
+                return cand
+        return None
+
+    # -- validation --------------------------------------------------------
+
+    def _validate_path(
+        self, path: Path, artist_slug: str, source_url: str | None = None
+    ) -> tuple[str | None, Path | None]:
+        """Validate a freshly-written file; quarantine if bad.
+
+        Uses the shared `file_validator.quarantine_file` — same move + provenance
+        sidecar gallery-dl writes (the native path used to skip the sidecar; that
+        parity gap is closed here). Returns `(reason, quarantine_dest)` when
+        quarantined (dest is the original path if the move itself failed), else
+        `(None, None)` (ok / not validatable / disabled). plan #704: the dest is
+        surfaced so the run reports a real quarantined-paths list.
+        """
+        if not self._validate or not is_validatable(path):
+            return None, None
+        try:
+            result = validate_file(path)
+        except Exception as exc:
+            log.warning("Validator raised on %s: %s", path, exc)
+            return None, None
+        if result.ok:
+            return None, None
+        dest = quarantine_file(
+            self.images_root, path, artist_slug, "patreon",
+            url=source_url, result=result,
+        )
+        return (result.reason or "validation failed"), (dest or path)
+
+    # -- sidecar -----------------------------------------------------------
+
+    def _write_sidecar(self, post: dict, media_path: Path) -> Path:
+        """Write the importer-consumed sidecar next to `media_path`.
+
+        Patreon uses base-default sidecar keys (parse_sidecar maps
+        category->platform, id->external_post_id, title->post_title,
+        content->description, published_at->post_date, url->post_url). Patreon
+        registers no derive_post_url, so `url` is trusted as the permalink — we
+        pass the post's attributes.url.
+        """
+        attrs = post.get("attributes") or {}
+        title = attrs.get("title")
+        content = attrs.get("content")
+        published = attrs.get("published_at")
+        url = attrs.get("url")
+        data = {
+            "category": "patreon",
+            "id": str(post.get("id") or ""),
+            "title": title if isinstance(title, str) else "",
+            "content": content if isinstance(content, str) else "",
+            "published_at": published if isinstance(published, str) else None,
+            "url": url if isinstance(url, str) else None,
+        }
+        sidecar_path = media_path.with_suffix(".json")
+        sidecar_path.write_text(json.dumps(data, indent=2))
+        return sidecar_path

backend/app/services/patreon_ingester.py

@@ -0,0 +1,195 @@
+"""Native Patreon ingester — the Patreon ADAPTER over the platform-agnostic core.
+
+The orchestration that drives a native subscription walk (page a feed → extract
+media → tiered skip → download → mark-seen / record-failures / checkpoint-cursor
+→ return a gallery-dl-shaped `DownloadResult`, across tick/backfill/recovery)
+now lives in `ingest_core.Ingester` — it's identical for every platform. This
+module is the thin Patreon adapter: it wires the Patreon `client`/`downloader`/
+ledger models/constraints/key into the core and supplies the Patreon-specific
+failure mapping. `download_service.download_source` calls `PatreonIngester.run`
+exactly as before; the public surface (this class, `_ledger_key`,
+`DEAD_LETTER_THRESHOLD`, `verify_patreon_credential`) is unchanged.
+
+Three modes (selected by `download_service` from `config_overrides` state):
+  - tick      — newest→oldest, skip seen (tier-1 ledger + tier-2 disk), early-out
+                after N contiguous already-have-it items (the cheap native
+                equivalent of gallery-dl's `exit:20`, now free of per-file HEADs).
+  - backfill  — full-history walk in a time-boxed chunk, resuming from the
+                pagination cursor checkpoint; reaches the bottom → "complete".
+  - recovery  — like backfill but BYPASSES the tier-1 seen-ledger AND the
+                dead-letter ledger, so deliberately-dropped-and-deleted near-dups
+                get re-fetched and re-evaluated under the current pHash threshold
+                (tier-2 disk skip still spares files we kept).
+
+The seen/dead-letter ledgers live in Postgres (`patreon_seen_media` /
+`patreon_failed_media`); the core opens SHORT-LIVED sync sessions per page batch
+— never held across a network fetch ([[db-connection-held-across-subprocess]]).
+FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Callable
+from pathlib import Path
+
+from ..models import PatreonFailedMedia, PatreonSeenMedia
+from .gallery_dl import DownloadResult, ErrorType
+from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester
+from .patreon_client import (
+    MediaItem,
+    PatreonAPIError,
+    PatreonAuthError,
+    PatreonClient,
+    PatreonDriftError,
+)
+from .patreon_downloader import PatreonDownloader
+from .patreon_resolver import resolve_campaign_id_for_source
+
+__all__ = [
+    "DEAD_LETTER_THRESHOLD",
+    "PatreonIngester",
+    "_ledger_key",
+    "verify_patreon_credential",
+]
+
+log = logging.getLogger(__name__)
+
+# Ledger keys are stored in patreon_seen_media.filehash VARCHAR(128); bound any
+# synthesized key so a pathologically long file_name can't overflow the column.
+_LEDGER_KEY_MAX = 128
+
+
+def _ledger_key(media: MediaItem) -> str:
+    """Stable per-media identity for the cross-run seen-ledger.
+
+    A Patreon CDN URL carries a 32-char MD5 (`media.filehash`) — that is the
+    natural key. Some media have none: Mux/HLS video (`stream.mux.com`, no
+    content hash at discovery) and the odd inline-content `<img>` pointing at a
+    hashless URL. The plan calls the video case the ``video:<post_id>:<media_id>``
+    sentinel; `MediaItem` carries no media_id, so the post-scoped filename is the
+    stable proxy. Bounded to the column width.
+    """
+    if media.filehash:
+        return media.filehash
+    return f"{media.post_id}:{media.filename}"[:_LEDGER_KEY_MAX]
+
+
+class PatreonIngester(Ingester):
+    """Walk a Patreon campaign's posts, download unseen media, return a
+    `DownloadResult`. A thin adapter over `ingest_core.Ingester`.
+
+    Construct with the per-source `cookies_path` and a sync sessionmaker for the
+    ledgers. `client` / `downloader` are injectable seams so unit tests run
+    without network, subprocess, or a real CDN.
+    """
+
+    def __init__(
+        self,
+        images_root: Path,
+        cookies_path: str | None,
+        session_factory: Callable[[], object],
+        *,
+        validate: bool = True,
+        rate_limit: float = 0.0,
+        request_sleep: float = 0.0,
+        client: PatreonClient | None = None,
+        downloader: PatreonDownloader | None = None,
+    ):
+        self.images_root = Path(images_root)
+        self.cookies_path = str(cookies_path) if cookies_path else None
+        # Pacing (plan #703): request_sleep paces the API page fetches,
+        # rate_limit paces the media downloads. Injected client/downloader (in
+        # tests) already carry their own pacing, so these only apply to the
+        # default-constructed ones.
+        resolved_client = (
+            client
+            if client is not None
+            else PatreonClient(cookies_path, request_sleep=request_sleep)
+        )
+        resolved_downloader = (
+            downloader
+            if downloader is not None
+            else PatreonDownloader(
+                self.images_root, cookies_path, validate=validate, rate_limit=rate_limit
+            )
+        )
+        super().__init__(
+            client=resolved_client,
+            downloader=resolved_downloader,
+            session_factory=session_factory,
+            seen_model=PatreonSeenMedia,
+            failed_model=PatreonFailedMedia,
+            seen_constraint="uq_patreon_seen_media_source_id",
+            failed_constraint="uq_patreon_failed_media_source_id",
+            ledger_key=_ledger_key,
+            platform="patreon",
+            error_base=PatreonAPIError,
+        )
+
+    # -- failure mapping (Patreon exception taxonomy) ----------------------
+
+    def _failure_result(self, exc: Exception, _result) -> DownloadResult:
+        """Map a client-level exception to a loud, typed failed DownloadResult.
+
+        We NEVER return a silent zero-download "success" — the whole point of the
+        native ingester is to fail RED when Patreon's API shape or our auth
+        changes. The typed mapping lets FailingSourcesCard render the right chip
+        and tells the operator what to do:
+          - PatreonAuthError  → AUTH_ERROR  (rotate cookies)
+          - PatreonDriftError → API_DRIFT   (ingester field-set/parser needs update)
+          - HTTP 429 / 404    → RATE_LIMITED / NOT_FOUND
+          - other HTTP status → HTTP_ERROR;  transport failure → NETWORK_ERROR
+
+        PatreonAuthError and PatreonDriftError both subclass PatreonAPIError, so
+        they must be matched before the generic HTTP/transport fallthrough.
+        """
+        message = str(exc)
+        if isinstance(exc, PatreonAuthError):
+            error_type = ErrorType.AUTH_ERROR
+        elif isinstance(exc, PatreonDriftError):
+            error_type = ErrorType.API_DRIFT
+            message = f"Patreon API changed — ingester needs update: {message}"
+        else:  # generic PatreonAPIError: HTTP non-2xx (status_code set) or transport
+            status = getattr(exc, "status_code", None)
+            if status == 429:
+                error_type = ErrorType.RATE_LIMITED
+            elif status == 404:
+                error_type = ErrorType.NOT_FOUND
+            elif status is not None:
+                error_type = ErrorType.HTTP_ERROR
+            else:
+                error_type = ErrorType.NETWORK_ERROR
+        log.warning("Patreon ingest failed (%s): %s", error_type.value, message)
+        result = _result(
+            success=False, return_code=1,
+            error_type=error_type, error_message=message,
+        )
+        # plan #708 B1: carry the server's Retry-After up to the cooldown.
+        if error_type == ErrorType.RATE_LIMITED:
+            result.retry_after_seconds = getattr(exc, "retry_after", None)
+        return result
+
+
+async def verify_patreon_credential(
+    url: str,
+    cookies_path: str | None,
+    overrides: dict | None,
+) -> tuple[bool | None, str]:
+    """Native Patreon credential probe — the verify counterpart to the ingester's
+    download path, sharing its campaign-id resolution. Resolves the campaign id
+    (override / id: URL / vanity) then does ONE authenticated `/api/posts` fetch
+    via PatreonClient.verify_auth. Returns the uniform `(ok, message)` contract
+    (True / False / None) so download_backends.verify_credential can treat it
+    interchangeably with the gallery-dl probe. No download, no DB.
+    """
+    campaign_id, _ = await resolve_campaign_id_for_source(url, cookies_path, overrides)
+    if not campaign_id:
+        return None, (
+            "Couldn't resolve the Patreon campaign id from the source URL — "
+            "can't verify (cookies expired, or the creator moved/renamed?)."
+        )
+    client = PatreonClient(cookies_path)
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(None, client.verify_auth, campaign_id)

backend/app/services/patreon_resolver.py

@@ -19,12 +19,22 @@ import asyncio
 import http.cookiejar
 import logging
 import os
+import re
 
 import requests
 
 log = logging.getLogger(__name__)
 
 _CAMPAIGNS_URL = "https://www.patreon.com/api/campaigns"
+
+# A source URL of the form `.../id:<digits>` already carries the campaign id
+# (no lookup needed). The vanity regex deliberately EXCLUDES the id: form so the
+# two paths don't overlap.
+_ID_URL_RE = re.compile(r"/id:(\d+)")
+_VANITY_RE = re.compile(
+    r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)",
+    re.IGNORECASE,
+)
 _USER_AGENT = (
     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
     "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
@@ -100,3 +110,38 @@ async def resolve_campaign_id(
     Never raises."""
     loop = asyncio.get_running_loop()
     return await loop.run_in_executor(None, _sync_lookup, vanity, cookies_path)
+
+
+def extract_vanity(url: str) -> str | None:
+    """The vanity slug from a Patreon creator URL, or None for an `id:` URL."""
+    m = _VANITY_RE.match(url or "")
+    return m.group(1) if m else None
+
+
+async def resolve_campaign_id_for_source(
+    url: str,
+    cookies_path: str | None,
+    overrides: dict | None,
+) -> tuple[str | None, str | None]:
+    """Resolve a Patreon source to its campaign id — the single resolution path
+    shared by the download ingester and the credential-verify probe.
+
+    Order: cached `patreon_campaign_id` override → an `id:<digits>` URL → a
+    vanity lookup against the campaigns API. Returns
+    `(campaign_id, newly_resolved_id)`: `newly_resolved_id` is non-None ONLY when
+    a vanity lookup actually ran, so the caller knows to cache it on the source
+    (the override/id: paths needed no lookup). `(None, None)` when unresolvable.
+    Never raises.
+    """
+    overrides = overrides or {}
+    cached = overrides.get("patreon_campaign_id")
+    if cached:
+        return cached, None
+    id_match = _ID_URL_RE.search(url or "")
+    if id_match:
+        return id_match.group(1), None
+    vanity = extract_vanity(url)
+    if vanity:
+        resolved = await resolve_campaign_id(vanity, cookies_path)
+        return resolved, resolved
+    return None, None

backend/app/services/source_service.py

@@ -67,6 +67,12 @@ class SourceRecord:
     # plan #693: derived from config_overrides for the UI badge.
     backfill_state: str | None  # "running" | "complete" | "stalled" | None (idle)
     backfill_chunks: int
+    # plan #697: a running deep-walk that bypasses the Patreon seen-ledger
+    # (recovery) vs. a normal backfill. Lets the badge label it "Recovering".
+    backfill_bypass_seen: bool
+    # plan #704: cumulative posts processed across the walk's chunks — live
+    # progress for the badge.
+    backfill_posts: int
 
     def to_dict(self) -> dict:
         return {
@@ -87,6 +93,8 @@ class SourceRecord:
             "backfill_runs_remaining": self.backfill_runs_remaining,
             "backfill_state": self.backfill_state,
             "backfill_chunks": self.backfill_chunks,
+            "backfill_bypass_seen": self.backfill_bypass_seen,
+            "backfill_posts": self.backfill_posts,
         }
 
 
@@ -162,6 +170,8 @@ class SourceService:
             backfill_runs_remaining=source.backfill_runs_remaining or 0,
             backfill_state=co.get("_backfill_state"),
             backfill_chunks=int(co.get("_backfill_chunks", 0)),
+            backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")),
+            backfill_posts=int(co.get("_backfill_posts", 0)),
         )
 
     async def _row_to_record(self, source: Source) -> SourceRecord:
@@ -312,7 +322,37 @@ class SourceService:
             raise LookupError(f"source id={source_id} not found")
         co = dict(source.config_overrides or {})
         co["_backfill_state"] = "running"
-        for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks"):
+        for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
+                  "_backfill_posts"):
+            co.pop(k, None)
+        source.config_overrides = co
+        source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
+        await self.session.commit()
+        return await self._row_to_record(source)
+
+    async def start_recovery(self, source_id: int) -> SourceRecord:
+        """Plan #697: arm a RECOVERY walk — a backfill that bypasses the Patreon
+        seen-ledger so deliberately-dropped-and-deleted near-dups get re-fetched
+        and re-evaluated under the CURRENT pHash threshold (tier-2 disk still
+        spares files we kept). Reuses the entire #693 backfill state machine
+        (time-boxed chunks, cursor checkpoint, complete/stall lifecycle) plus the
+        `_backfill_bypass_seen` flag that flips download mode to recovery. Clears
+        any prior cursor/chunk/stall state so it walks fresh from the top. The
+        flag is cleared on completion (download_service) and on stop.
+
+        Recovery is Patreon-only (the seen-ledger is Patreon's); for other
+        platforms the flag is inert (download_service ignores it) and the walk
+        runs as a plain backfill. The UI gates the action to Patreon sources."""
+        source = (await self.session.execute(
+            select(Source).where(Source.id == source_id)
+        )).scalar_one_or_none()
+        if source is None:
+            raise LookupError(f"source id={source_id} not found")
+        co = dict(source.config_overrides or {})
+        co["_backfill_state"] = "running"
+        co["_backfill_bypass_seen"] = True
+        for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
+                  "_backfill_posts"):
             co.pop(k, None)
         source.config_overrides = co
         source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
@@ -329,7 +369,7 @@ class SourceService:
             raise LookupError(f"source id={source_id} not found")
         co = dict(source.config_overrides or {})
         for k in ("_backfill_state", "_backfill_cursor", "_backfill_cursor_stalls",
-                  "_backfill_chunks"):
+                  "_backfill_chunks", "_backfill_bypass_seen", "_backfill_posts"):
             co.pop(k, None)
         source.config_overrides = co
         source.backfill_runs_remaining = 0

backend/app/tasks/download.py

@@ -143,6 +143,11 @@ def download_source(self, source_id: int) -> int:
                         gdl=gdl,
                         importer=importer,
                         cred_service=cred_service,
+                        # The native Patreon ingester opens its own short-lived
+                        # sync sessions for the seen-ledger (never held across
+                        # the walk). Same factory the importer's sync session
+                        # comes from — a different DB connection per checkout.
+                        sync_session_factory=SyncFactory,
                     )
                     return await svc.download_source(source_id)
         finally:

backend/app/utils/paths.py

@@ -2,6 +2,26 @@
 
 from pathlib import Path
 
+_MAX_EXT_LEN = 16
+
+
+def safe_ext(name: str | Path) -> str:
+    """Conservatively extract a short, alphanumeric file extension.
+
+    gallery-dl and Patreon CDN URLs produce basenames with URL-encoded
+    query-string artifacts, so `Path.suffix` can return 50+ chars of base64-ish
+    junk that blows bounded VARCHAR columns (e.g. PostAttachment.ext varchar(32)).
+    Accept only a suffix ≤16 chars whose post-dot characters are all alphanumeric;
+    otherwise return "" (no known extension). Operator-flagged 2026-05-25 — ONE
+    impl for the importer and the native Patreon client.
+    """
+    suffix = Path(name).suffix.lower()
+    if not suffix or len(suffix) > _MAX_EXT_LEN:
+        return ""
+    if not all(c.isalnum() for c in suffix[1:]):
+        return ""
+    return suffix
+
 
 def derive_subdir(source_path: Path, import_root: Path) -> str:
     """Returns the relative subdirectory of source_path under import_root.

frontend/src/components/subscriptions/FailingSourcesCard.vue

@@ -93,6 +93,7 @@ const ERROR_TYPE_COLOR = {
   unsupported_url: 'error',
   http_error: 'error',
   unknown_error: 'error',
+  api_drift: 'error',
   partial: 'info',
   tier_limited: 'info',
   no_new_content: 'info',
@@ -108,6 +109,7 @@ const ERROR_TYPE_HINT = {
   http_error: 'Generic HTTP error — see Logs.',
   unsupported_url: 'gallery-dl does not support this URL pattern.',
   unknown_error: 'Could not classify — see Logs.',
+  api_drift: 'Patreon changed its API shape — the native ingester needs a code update.',
 }
 function errorTypeColor(t) { return ERROR_TYPE_COLOR[t] || 'error' }
 function errorTypeHint(t) { return ERROR_TYPE_HINT[t] || '' }

frontend/src/components/subscriptions/PreviewDialog.vue

@@ -0,0 +1,113 @@
+<template>
+  <!-- Plan #708 B4: dry-run preview — what a backfill WOULD download, without
+       downloading. Self-fetches on open; loading / error / empty / result. -->
+  <v-dialog
+    :model-value="modelValue" max-width="520"
+    @update:model-value="$emit('update:modelValue', $event)"
+  >
+    <v-card v-if="source">
+      <v-card-title class="d-flex align-center">
+        <v-icon class="mr-2">mdi-eye-outline</v-icon>
+        Preview · {{ source.artist_name }}
+      </v-card-title>
+      <v-card-subtitle class="fc-preview__url">{{ source.url }}</v-card-subtitle>
+
+      <v-card-text>
+        <div v-if="loading" class="text-center py-8">
+          <v-progress-circular indeterminate color="accent" />
+          <div class="text-caption mt-3 text-medium-emphasis">Walking the feed…</div>
+        </div>
+
+        <v-alert
+          v-else-if="error" type="error" variant="tonal" density="comfortable"
+        >{{ error }}</v-alert>
+
+        <div v-else-if="result">
+          <div class="text-h5">
+            {{ result.total_new }} new item{{ result.total_new === 1 ? '' : 's' }}
+          </div>
+          <div class="text-body-2 text-medium-emphasis mb-3">
+            in the {{ result.posts_scanned }} most recent
+            post{{ result.posts_scanned === 1 ? '' : 's' }}<span
+              v-if="result.has_more"> · more pages not scanned</span>
+          </div>
+
+          <div
+            v-if="result.total_new === 0"
+            class="text-body-2 text-medium-emphasis"
+          >You’re caught up — nothing new to download.</div>
+
+          <v-list v-else density="compact" class="fc-preview__list" bg-color="transparent">
+            <v-list-item v-for="(row, i) in result.sample" :key="i" class="px-0">
+              <template #prepend>
+                <v-chip
+                  size="x-small" color="info" variant="tonal" label class="mr-3"
+                >+{{ row.new }}</v-chip>
+              </template>
+              <v-list-item-title class="text-body-2">{{ row.title }}</v-list-item-title>
+              <v-list-item-subtitle v-if="row.date" class="text-caption">
+                {{ formatRelative(row.date) }}
+              </v-list-item-subtitle>
+            </v-list-item>
+          </v-list>
+        </div>
+      </v-card-text>
+
+      <v-card-actions>
+        <v-spacer />
+        <v-btn variant="text" @click="$emit('update:modelValue', false)">Close</v-btn>
+        <v-btn
+          v-if="result && result.total_new > 0"
+          color="accent" variant="flat"
+          @click="$emit('backfill', source)"
+        >Start backfill</v-btn>
+      </v-card-actions>
+    </v-card>
+  </v-dialog>
+</template>
+
+<script setup>
+import { ref, watch } from 'vue'
+import { useSourcesStore } from '../../stores/sources.js'
+import { formatRelative } from '../../utils/date.js'
+
+const props = defineProps({
+  modelValue: { type: Boolean, default: false },
+  source: { type: Object, default: null },
+})
+defineEmits(['update:modelValue', 'backfill'])
+
+const store = useSourcesStore()
+const loading = ref(false)
+const error = ref('')
+const result = ref(null)
+
+watch(
+  () => props.modelValue,
+  async (open) => {
+    if (!open || !props.source) return
+    loading.value = true
+    error.value = ''
+    result.value = null
+    try {
+      result.value = await store.previewSource(props.source.id)
+    } catch (e) {
+      error.value = e?.body?.detail || e?.message || 'Preview failed'
+    } finally {
+      loading.value = false
+    }
+  },
+)
+</script>
+
+<style scoped>
+.fc-preview__url {
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+.fc-preview__list {
+  max-height: 320px;
+  overflow-y: auto;
+}
+</style>

frontend/src/components/subscriptions/SourceCard.vue

@@ -27,7 +27,10 @@
       <v-chip
         v-else-if="source.backfill_state === 'running'"
         size="x-small" color="info" variant="tonal" label
-      >Backfilling{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }}</v-chip>
+      >{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling'
+        }}{{ source.backfill_posts
+              ? ` · ${source.backfill_posts} posts`
+              : (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
       <v-chip
         v-else-if="source.backfill_state === 'complete'"
         size="x-small" color="success" variant="tonal" label
@@ -53,7 +56,29 @@
       >
         <v-icon>{{ source.backfill_state === 'running' ? 'mdi-stop' : 'mdi-magnify-scan' }}</v-icon>
         <v-tooltip activator="parent" location="top">
-          {{ source.backfill_state === 'running' ? 'Stop backfill' : 'Backfill full history' }}
+          {{ source.backfill_state === 'running'
+            ? (source.backfill_bypass_seen ? 'Stop recovery' : 'Stop backfill')
+            : 'Backfill full history' }}
+        </v-tooltip>
+      </v-btn>
+      <v-btn
+        v-if="source.platform === 'patreon' && source.backfill_state !== 'running'"
+        size="x-small" variant="text"
+        @click.stop="$emit('preview', source)"
+      >
+        <v-icon>mdi-eye-outline</v-icon>
+        <v-tooltip activator="parent" location="top">
+          Preview — count what a backfill would download (no download)
+        </v-tooltip>
+      </v-btn>
+      <v-btn
+        v-if="source.platform === 'patreon' && source.backfill_state !== 'running'"
+        size="x-small" variant="text"
+        @click.stop="$emit('recover', source)"
+      >
+        <v-icon>mdi-backup-restore</v-icon>
+        <v-tooltip activator="parent" location="top">
+          Recover — re-fetch dropped near-dups &amp; re-evaluate under the current threshold
         </v-tooltip>
       </v-btn>
       <v-btn size="x-small" variant="text" @click.stop="$emit('edit', source)">
@@ -83,7 +108,7 @@ const props = defineProps({
   checking: { type: Boolean, default: false },
   warningThreshold: { type: Number, default: 5 },
 })
-const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill'])
+const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover', 'preview'])
 
 function onToggleEnabled(value) {
   emit('toggle', { source: props.source, enabled: value })

frontend/src/components/subscriptions/SourceRow.vue

@@ -34,7 +34,10 @@
       <v-chip
         v-else-if="source.backfill_state === 'running'"
         size="x-small" color="info" variant="tonal" label
-      >Backfilling{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }}</v-chip>
+      >{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling'
+        }}{{ source.backfill_posts
+              ? ` · ${source.backfill_posts} posts`
+              : (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
       <v-chip
         v-else-if="source.backfill_state === 'complete'"
         size="x-small" color="success" variant="tonal" label
@@ -62,10 +65,30 @@
         <v-icon>{{ source.backfill_state === 'running' ? 'mdi-stop' : 'mdi-magnify-scan' }}</v-icon>
         <v-tooltip activator="parent" location="top">
           {{ source.backfill_state === 'running'
-            ? 'Stop backfill'
+            ? (source.backfill_bypass_seen ? 'Stop recovery' : 'Stop backfill')
             : 'Backfill — walk the full post history until complete' }}
         </v-tooltip>
       </v-btn>
+      <v-btn
+        v-if="source.platform === 'patreon' && source.backfill_state !== 'running'"
+        size="x-small" variant="text"
+        @click.stop="$emit('preview', source)"
+      >
+        <v-icon>mdi-eye-outline</v-icon>
+        <v-tooltip activator="parent" location="top">
+          Preview — count what a backfill would download (no download)
+        </v-tooltip>
+      </v-btn>
+      <v-btn
+        v-if="source.platform === 'patreon' && source.backfill_state !== 'running'"
+        size="x-small" variant="text"
+        @click.stop="$emit('recover', source)"
+      >
+        <v-icon>mdi-backup-restore</v-icon>
+        <v-tooltip activator="parent" location="top">
+          Recover — re-fetch dropped near-dups &amp; re-evaluate under the current threshold
+        </v-tooltip>
+      </v-btn>
       <v-btn
         icon="mdi-pencil" size="x-small" variant="text"
         @click.stop="$emit('edit', source)"
@@ -93,7 +116,7 @@ const props = defineProps({
   checking: { type: Boolean, default: false },
   warningThreshold: { type: Number, default: 5 },
 })
-const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill'])
+const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover', 'preview'])
 
 function onToggleEnabled(value) {
   emit('toggle', { source: props.source, enabled: value })

frontend/src/components/subscriptions/SubscriptionsTab.vue

@@ -175,6 +175,8 @@
                     @toggle="toggleSourceEnabled"
                     @check="onCheck"
                     @backfill="onBackfill"
+                    @recover="onRecover"
+                    @preview="onPreview"
                   />
                   <tr v-if="item.sources.length === 0">
                     <td colspan="8" class="fc-subs__sources-empty">
@@ -251,6 +253,8 @@
             @toggle="toggleSourceEnabled"
             @check="onCheck"
             @backfill="onBackfill"
+            @recover="onRecover"
+            @preview="onPreview"
           />
           <div v-if="item.sources.length === 0" class="fc-subs__sources-empty">
             No sources yet. Tap + to add one.
@@ -266,6 +270,11 @@
       @saved="onSourceSaved"
     />
     <ArtistCreateDialog v-model="showArtistDialog" @created="onArtistCreated" />
+    <PreviewDialog
+      v-model="showPreviewDialog"
+      :source="previewTarget"
+      @backfill="onPreviewBackfill"
+    />
   </div>
 </template>
 
@@ -282,6 +291,7 @@ import SourceCard from './SourceCard.vue'
 import SourceHealthDot from './SourceHealthDot.vue'
 import SourceFormDialog from './SourceFormDialog.vue'
 import ArtistCreateDialog from './ArtistCreateDialog.vue'
+import PreviewDialog from './PreviewDialog.vue'
 import PlatformChip from './PlatformChip.vue'
 import SchedulerStatusBar from './SchedulerStatusBar.vue'
 import { formatRelative } from '../../utils/date.js'
@@ -332,6 +342,8 @@ const showSourceDialog = ref(false)
 const editingSource = ref(null)
 const editingArtist = ref(null)
 const showArtistDialog = ref(false)
+const showPreviewDialog = ref(false)
+const previewTarget = ref(null)
 
 const artistFilter = computed(() => {
   const raw = route.query.artist_id
@@ -548,12 +560,40 @@ async function onBackfill(source) {
     await store.loadAll()
   } catch (e) {
     toast({
-      text: `Backfill ${running ? 'stop' : 'start'} failed: ${e?.detail || e?.message || e}`,
+      text: `Backfill ${running ? 'stop' : 'start'} failed: ${e?.body?.detail || e?.message || e}`,
       type: 'error',
     })
   }
 }
 
+// Plan #697: arm a recovery walk (Patreon-only) — re-fetches dropped-and-deleted
+// near-dups and re-evaluates them under the current pHash threshold. Reuses the
+// backfill lifecycle/badge; stop via the same Stop control (onBackfill).
+async function onRecover(source) {
+  try {
+    await store.recoverSource(source.id, source.artist_id)
+    toast({ text: `Recovery started for ${source.artist_name}`, type: 'success' })
+    await store.loadAll()
+  } catch (e) {
+    toast({
+      text: `Recovery start failed: ${e?.body?.detail || e?.message || e}`,
+      type: 'error',
+    })
+  }
+}
+
+// Plan #708 B4: open the dry-run preview dialog (it self-fetches on open).
+function onPreview(source) {
+  previewTarget.value = source
+  showPreviewDialog.value = true
+}
+
+// "Start backfill" from inside the preview dialog → arm it + close.
+async function onPreviewBackfill(source) {
+  showPreviewDialog.value = false
+  await onBackfill(source)
+}
+
 async function checkAll(group) {
   let ok = 0
   let conflict = 0

frontend/src/stores/sources.js

@@ -98,6 +98,20 @@ export const useSourcesStore = defineStore('sources', () => {
     _invalidate(artistIdHint ?? body.artist_id)
     return body
   }
+  // Plan #697: arm a recovery walk — a backfill that bypasses the Patreon
+  // seen-ledger to re-fetch dropped-and-deleted near-dups and re-evaluate them
+  // under the current pHash threshold. Stop with stopBackfill (shared lifecycle).
+  async function recoverSource(id, artistIdHint = null) {
+    const body = await api.post(`/api/sources/${id}/backfill`, { body: { action: 'recover' } })
+    _invalidate(artistIdHint ?? body.artist_id)
+    return body
+  }
+
+  // Plan #708 B4: dry-run preview — count what a backfill WOULD download
+  // (native platforms), without downloading. Read-only, so no cache invalidate.
+  async function previewSource(id) {
+    return await api.post(`/api/sources/${id}/preview`)
+  }
 
   function sourcesByArtistGrouped() {
     // returns [{artist: {id,name,slug}, sources: [...]}, ...]
@@ -127,6 +141,8 @@ export const useSourcesStore = defineStore('sources', () => {
     checkNow,
     startBackfill,
     stopBackfill,
+    recoverSource,
+    previewSource,
     findOrCreateArtist, autocompleteArtist,
     loadScheduleStatus,
     sourcesByArtistGrouped,

tests/fixtures/patreon_posts_page1.json

@@ -0,0 +1,121 @@
+{
+  "data": [
+    {
+      "id": "1001",
+      "type": "post",
+      "attributes": {
+        "title": "Image gallery post",
+        "published_at": "2026-05-01T12:00:00.000+00:00",
+        "post_type": "image_file",
+        "content": "<p>just a gallery</p>",
+        "url": "https://www.patreon.com/posts/1001",
+        "image": {
+          "large_url": "https://c10.patreonusercontent.com/4/large/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg",
+          "url": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg"
+        }
+      },
+      "relationships": {
+        "images": {
+          "data": [
+            {"id": "9001", "type": "media"},
+            {"id": "9002", "type": "media"}
+          ]
+        }
+      }
+    },
+    {
+      "id": "1002",
+      "type": "post",
+      "attributes": {
+        "title": "Attachment post",
+        "published_at": "2026-04-15T09:30:00.000+00:00",
+        "post_type": "text_only",
+        "content": "<p>here is a zip</p>",
+        "url": "https://www.patreon.com/posts/1002",
+        "post_file": {
+          "name": "bonus-pack.zip",
+          "url": "https://www.patreon.com/file?h=cccccccccccccccccccccccccccccccc&i=1002"
+        }
+      },
+      "relationships": {
+        "attachments_media": {
+          "data": [
+            {"id": "9003", "type": "media"}
+          ]
+        }
+      }
+    },
+    {
+      "id": "1003",
+      "type": "post",
+      "attributes": {
+        "title": "Inline content post",
+        "published_at": "2026-04-01T08:00:00.000+00:00",
+        "post_type": "text_only",
+        "content": "<p>look</p><figure><img src=\"https://c10.patreonusercontent.com/4/orig/dddddddddddddddddddddddddddddddd/inline.png?token=x&amp;v=2\" alt=\"inline\"></figure>",
+        "url": "https://www.patreon.com/posts/1003"
+      },
+      "relationships": {}
+    },
+    {
+      "id": "1004",
+      "type": "post",
+      "attributes": {
+        "title": "Video post",
+        "published_at": "2026-03-20T18:45:00.000+00:00",
+        "post_type": "video_external_file",
+        "content": "<p>watch</p>",
+        "url": "https://www.patreon.com/posts/1004",
+        "post_file": {
+          "name": "clip.m3u8",
+          "url": "https://stream.mux.com/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee.m3u8?token=jwt"
+        }
+      },
+      "relationships": {}
+    }
+  ],
+  "included": [
+    {
+      "id": "9001",
+      "type": "media",
+      "attributes": {
+        "file_name": "gallery-one.jpg",
+        "download_url": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg",
+        "image_urls": {
+          "original": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg",
+          "default": "https://c10.patreonusercontent.com/4/default/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg"
+        }
+      }
+    },
+    {
+      "id": "9002",
+      "type": "media",
+      "attributes": {
+        "file_name": "gallery-two.jpg",
+        "download_url": "https://c10.patreonusercontent.com/4/orig/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/gallery-two.jpg",
+        "image_urls": {
+          "original": "https://c10.patreonusercontent.com/4/orig/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/gallery-two.jpg"
+        }
+      }
+    },
+    {
+      "id": "9003",
+      "type": "media",
+      "attributes": {
+        "file_name": "bonus-pack.zip",
+        "download_url": "https://www.patreon.com/file/download?h=cccccccccccccccccccccccccccccccc"
+      }
+    },
+    {
+      "id": "5555",
+      "type": "campaign",
+      "attributes": {
+        "name": "Example Creator",
+        "url": "https://www.patreon.com/example"
+      }
+    }
+  ],
+  "links": {
+    "next": "https://www.patreon.com/api/posts?filter%5Bcampaign_id%5D=5555&page%5Bcursor%5D=NEXTCURSOR123&sort=-published_at"
+  }
+}

tests/test_api_credentials.py

@@ -155,6 +155,8 @@ async def test_verify_no_enabled_source_is_untestable(client):
 
 @pytest.mark.asyncio
 async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypatch):
+    """A gallery-dl platform routes through GalleryDLService.verify; success
+    stamps last_verified (platform-agnostic endpoint behavior)."""
     from backend.app.models import Artist, Source
     from backend.app.services import gallery_dl as gdl_mod
 
@@ -163,6 +165,45 @@ async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypa
         return (True, "Credentials valid — the feed authenticated.")
     monkeypatch.setattr(gdl_mod.GalleryDLService, "verify", _fake_verify)
 
+    await client.post("/api/credentials", json={
+        "platform": "subscribestar", "credential_type": "cookies", "data": _NETSCAPE,
+    })
+    artist = Artist(name="Maewix", slug="maewix")
+    db.add(artist)
+    await db.flush()
+    db.add(Source(
+        artist_id=artist.id, platform="subscribestar",
+        url="https://www.subscribestar.com/maewix", enabled=True, config_overrides={},
+    ))
+    await db.commit()
+
+    resp = await client.post("/api/credentials/subscribestar/verify")
+    body = await resp.get_json()
+    assert body["valid"] is True
+    assert body["last_verified"] is not None
+
+    # The stamp is persisted on the credential record.
+    rec = await (await client.get("/api/credentials/subscribestar")).get_json()
+    assert rec["last_verified"] is not None
+
+
+@pytest.mark.asyncio
+async def test_verify_patreon_uses_native_ingester_not_gallery_dl(client, db, monkeypatch):
+    """plan #697 cutover: Patreon credential verify routes through the native
+    ingester (verify_patreon_credential), NOT gallery-dl --simulate."""
+    from backend.app.models import Artist, Source
+    from backend.app.services import gallery_dl as gdl_mod
+    from backend.app.services import patreon_ingester as pi_mod
+
+    async def _native_verify(url, cookies_path, overrides):
+        return (True, "Credentials valid — the Patreon feed authenticated.")
+    monkeypatch.setattr(pi_mod, "verify_patreon_credential", _native_verify)
+
+    # gallery-dl must NOT be consulted for Patreon.
+    async def _boom(self, *args, **kwargs):
+        raise AssertionError("gallery-dl verify must not run for patreon")
+    monkeypatch.setattr(gdl_mod.GalleryDLService, "verify", _boom)
+
     await client.post("/api/credentials", json={
         "platform": "patreon", "credential_type": "cookies", "data": _NETSCAPE,
     })
@@ -180,10 +221,6 @@ async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypa
     assert body["valid"] is True
     assert body["last_verified"] is not None
 
-    # The stamp is persisted on the credential record.
-    rec = await (await client.get("/api/credentials/patreon")).get_json()
-    assert rec["last_verified"] is not None
-
 
 @pytest.mark.asyncio
 async def test_verify_reports_auth_failure(client, db, monkeypatch):

tests/test_api_sources.py

@@ -5,6 +5,20 @@ from backend.app.models import Artist, Source
 pytestmark = pytest.mark.integration
 
 
+@pytest.fixture(autouse=True)
+def _stub_preflight_verify(monkeypatch):
+    """Backfill/recover arms run a pre-flight credential verify (plan #703 #2)
+    which for Patreon would hit the network. Default it to 'valid' so the
+    endpoint tests stay network-free; the dedicated pre-flight tests override
+    this with a rejection/inconclusive verdict."""
+    from backend.app.services import download_backends as db_mod
+
+    async def _ok(**kwargs):
+        return (True, "Credentials valid")
+
+    monkeypatch.setattr(db_mod, "verify_source_credential", _ok)
+
+
 @pytest.fixture
 async def artist(db):
     a = Artist(name="Alice", slug="alice")
@@ -250,3 +264,117 @@ async def test_backfill_endpoint_rejects_bad_action(client, artist, db):
 async def test_backfill_endpoint_404_when_source_missing(client):
     resp = await client.post("/api/sources/999999/backfill", json={"action": "start"})
     assert resp.status_code == 404
+
+
+@pytest.mark.asyncio
+async def test_backfill_endpoint_recover_arms_bypass(client, artist, db):
+    """Plan #697: action='recover' arms a backfill that bypasses the Patreon
+    seen-ledger; the response exposes backfill_bypass_seen=True so the UI badge
+    can label it 'Recovering'. Stop clears it."""
+    src = Source(
+        artist_id=artist.id, platform="patreon",
+        url="https://patreon.com/alice-recover", enabled=True,
+    )
+    db.add(src)
+    await db.commit()
+    sid = src.id
+
+    resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "recover"})
+    assert resp.status_code == 200
+    body = await resp.get_json()
+    assert body["backfill_state"] == "running"
+    assert body["backfill_bypass_seen"] is True
+
+    stopped = await client.post(f"/api/sources/{sid}/backfill", json={"action": "stop"})
+    stopped_body = await stopped.get_json()
+    assert stopped_body["backfill_state"] is None
+    assert stopped_body["backfill_bypass_seen"] is False
+
+
+# --- Plan #703 #2: pre-flight credential verify on backfill/recover arm -----
+
+
+@pytest.mark.asyncio
+async def test_arm_blocked_when_credential_rejected(client, artist, db, monkeypatch):
+    """A definitively-rejected credential refuses the arm (409 + reason) instead
+    of starting a doomed walk; the source is NOT armed."""
+    from backend.app.services import download_backends as db_mod
+
+    async def _reject(**kwargs):
+        return (False, "Patreon rejected the credential — cookies expired")
+    monkeypatch.setattr(db_mod, "verify_source_credential", _reject)
+
+    src = Source(
+        artist_id=artist.id, platform="patreon",
+        url="https://patreon.com/alice-preflight", enabled=True,
+    )
+    db.add(src)
+    await db.commit()
+    sid = src.id
+
+    resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "recover"})
+    assert resp.status_code == 409
+    assert "expired" in ((await resp.get_json()).get("detail") or "")
+
+    # Not armed.
+    one = await (await client.get(f"/api/sources/{sid}")).get_json()
+    assert one["backfill_state"] is None
+
+
+@pytest.mark.asyncio
+async def test_arm_proceeds_when_credential_inconclusive(client, artist, db, monkeypatch):
+    """An inconclusive verify (network blip / drift) must NOT block the arm."""
+    from backend.app.services import download_backends as db_mod
+
+    async def _inconclusive(**kwargs):
+        return (None, "couldn't verify (network)")
+    monkeypatch.setattr(db_mod, "verify_source_credential", _inconclusive)
+
+    src = Source(
+        artist_id=artist.id, platform="patreon",
+        url="https://patreon.com/alice-incon", enabled=True,
+    )
+    db.add(src)
+    await db.commit()
+
+    resp = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "start"})
+    assert resp.status_code == 200
+    assert (await resp.get_json())["backfill_state"] == "running"
+
+
+@pytest.mark.asyncio
+async def test_stop_never_pre_flights(client, artist, db, monkeypatch):
+    from backend.app.services import download_backends as db_mod
+
+    async def _boom(**kwargs):
+        raise AssertionError("stop must not pre-flight verify")
+    monkeypatch.setattr(db_mod, "verify_source_credential", _boom)
+
+    src = Source(
+        artist_id=artist.id, platform="patreon",
+        url="https://patreon.com/alice-stop", enabled=True,
+    )
+    db.add(src)
+    await db.commit()
+    resp = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "stop"})
+    assert resp.status_code == 200
+
+
+@pytest.mark.asyncio
+async def test_gallery_dl_platform_arm_skips_pre_flight(client, artist, db, monkeypatch):
+    """Pre-flight is gated to native-ingester platforms (cheap verify); a
+    gallery-dl source arms without the slow --simulate probe."""
+    from backend.app.services import download_backends as db_mod
+
+    async def _boom(**kwargs):
+        raise AssertionError("gallery-dl arm must not pre-flight verify")
+    monkeypatch.setattr(db_mod, "verify_source_credential", _boom)
+
+    src = Source(
+        artist_id=artist.id, platform="subscribestar",
+        url="https://www.subscribestar.com/x", enabled=True,
+    )
+    db.add(src)
+    await db.commit()
+    resp = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "start"})
+    assert resp.status_code == 200

tests/test_download_backends.py

@@ -0,0 +1,23 @@
+"""download_backends — the single predicate that routes a platform to the
+native ingester vs. gallery-dl. Pure, no DB."""
+
+from backend.app.services.download_backends import (
+    NATIVE_INGESTER_PLATFORMS,
+    uses_native_ingester,
+)
+
+
+def test_patreon_is_native():
+    assert uses_native_ingester("patreon") is True
+    assert "patreon" in NATIVE_INGESTER_PLATFORMS
+
+
+def test_gallery_dl_platforms_are_not_native():
+    # The five platforms still served by gallery-dl must NOT route to the
+    # native ingester — guards an accidental over-broad migration.
+    for platform in ("subscribestar", "hentaifoundry", "discord", "pixiv", "deviantart"):
+        assert uses_native_ingester(platform) is False
+
+
+def test_unknown_platform_is_not_native():
+    assert uses_native_ingester("nonsense") is False

tests/test_download_service.py

@@ -59,7 +59,7 @@ def _make_jpg(path: Path, split: str = "h"):
 def _make_fake_dl_result(
     *, success=True, written_paths=None, quarantined_paths=None,
     files_downloaded=0, error_type=None, error_message=None,
-    stdout="", stderr="",
+    stdout="", stderr="", cursor=None, run_stats=None, posts_processed=0,
 ):
     return SimpleNamespace(
         success=success,
@@ -78,12 +78,22 @@ def _make_fake_dl_result(
         duration_seconds=1.23,
         started_at="2026-05-20T14:00:00+00:00",
         completed_at="2026-05-20T14:01:00+00:00",
+        # plan #704: native ingester now returns the cursor + run_stats
+        # structurally (no more stderr `Cursor:` scraping).
+        cursor=cursor,
+        run_stats=run_stats,
+        posts_processed=posts_processed,
     )
 
 
 def _fake_gdl_with_result(result):
     fake = MagicMock()
     fake.download = AsyncMock(return_value=result)
+    # Real values for the attrs run_download's native branch reads off the gdl
+    # (the native pacing config, plan #703) — a MagicMock would break the
+    # arithmetic (max(0.5, rate_limit/4)).
+    fake._rate_limit = 3.0
+    fake._validate_files = True
     fake._compute_run_stats = lambda *a, **k: {
         "exit_code": 0, "downloaded_count": len(result.written_paths),
         "skipped_count": 0, "per_item_failures": 0,
@@ -94,6 +104,28 @@ def _fake_gdl_with_result(result):
     return fake
 
 
+def _stub_patreon_ingester(svc, result, resolved_campaign_id=None):
+    """Route the phase-2 dispatch (download_backends.run_download via
+    DownloadService._run_download, plan #707 A5) to a canned DownloadResult so
+    these tests exercise download_service's phase-2-result→phase-3 handling
+    (status mapping, backfill lifecycle, health) without the real ingester's
+    network/DB. The ingester itself is covered by test_patreon_ingester.py; the
+    native construction/resolution by the run_download tests below. Returns a list
+    capturing (ctx, source_config, skip_value, mode) per call so a test can assert
+    mode / resume_cursor threading."""
+    calls = []
+
+    async def fake(*, ctx, source_config, skip_value, mode):
+        calls.append({
+            "ctx": ctx, "source_config": source_config,
+            "skip_value": skip_value, "mode": mode,
+        })
+        return result, resolved_campaign_id
+
+    svc._run_download = fake
+    return calls
+
+
 @pytest.mark.asyncio
 async def test_download_source_attaches_written_files(
     db, db_sync, tmp_path, seed_artist_and_source,
@@ -112,12 +144,13 @@ async def test_download_source_attaches_written_files(
     _make_jpg(f1, split="h")
     _make_jpg(f2, split="v")
 
-    fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
+    result = _make_fake_dl_result(
         success=True,
         written_paths=[str(f1), str(f2)],
         files_downloaded=2,
         stdout=f"{f1}\n{f2}\n",
-    ))
+    )
+    fake_gdl = _fake_gdl_with_result(result)
 
     sync_settings = db_sync.execute(
         select(ImportSettings).where(ImportSettings.id == 1)
@@ -137,6 +170,7 @@ async def test_download_source_attaches_written_files(
         async_session=db, sync_session=db_sync,
         gdl=fake_gdl, importer=importer, cred_service=cred_service,
     )
+    _stub_patreon_ingester(svc, result)
     event_id = await svc.download_source(source.id)
 
     ev = (await db.execute(
@@ -214,40 +248,19 @@ async def test_in_flight_idempotency_returns_existing_event(
 
 
 @pytest.mark.asyncio
-async def test_patreon_retry_caches_campaign_id(
-    db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
+async def test_patreon_resolved_campaign_id_is_cached(
+    db, db_sync, tmp_path, seed_artist_and_source,
 ):
+    """plan #697: the native ingester resolves a Patreon vanity → campaign id
+    on the fly (replacing gallery-dl's reactive campaign-id retry). When phase 2
+    reports a freshly-resolved id, phase 3 caches it on the source so later runs
+    skip the lookup. Stub the ingester to report a resolved id; assert it lands
+    in config_overrides."""
     from backend.app.services.download_service import DownloadService
     from backend.app.services.importer import Importer
 
     _artist, source = seed_artist_and_source
 
-    failed = _make_fake_dl_result(
-        success=False, written_paths=[], stdout="",
-        stderr="[patreon][error] Failed to extract campaign ID for alice\n",
-    )
-    succeeded = _make_fake_dl_result(success=True, written_paths=[])
-
-    download_calls = []
-
-    async def fake_download(url, **k):
-        download_calls.append(url)
-        return failed if len(download_calls) == 1 else succeeded
-
-    fake_gdl = MagicMock()
-    fake_gdl.download = fake_download
-    fake_gdl._compute_run_stats = lambda *a, **k: {
-        "exit_code": 0, "downloaded_count": 0, "skipped_count": 0,
-        "per_item_failures": 0, "warning_count": 0, "tier_gated_count": 0,
-    }
-    fake_gdl._extract_errors_warnings = lambda *a, **k: ""
-    fake_gdl._truncate_log = lambda x, **k: x
-
-    monkeypatch.setattr(
-        "backend.app.services.download_service.resolve_campaign_id",
-        AsyncMock(return_value="99"),
-    )
-
     sync_settings = db_sync.execute(
         select(ImportSettings).where(ImportSettings.id == 1)
     ).scalar_one()
@@ -261,19 +274,97 @@ async def test_patreon_retry_caches_campaign_id(
 
     svc = DownloadService(
         async_session=db, sync_session=db_sync,
-        gdl=fake_gdl, importer=importer, cred_service=cred_service,
+        gdl=_fake_gdl_with_result(_make_fake_dl_result(success=True)),
+        importer=importer, cred_service=cred_service,
+    )
+    _stub_patreon_ingester(
+        svc, _make_fake_dl_result(success=True, written_paths=[]),
+        resolved_campaign_id="99",
     )
     await svc.download_source(source.id)
 
-    assert len(download_calls) == 2
-    assert "id:99" in download_calls[1]
-
     overrides = db_sync.execute(
         select(Source.config_overrides).where(Source.id == source.id)
     ).scalar_one()
     assert overrides.get("patreon_campaign_id") == "99"
 
 
+@pytest.mark.asyncio
+async def test_run_download_native_resolves_vanity_and_runs(
+    db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
+):
+    """run_download (native branch, plan #707 A5): with no cached campaign id,
+    resolve the vanity and pass the resolved id straight to the ingester; report
+    it back so phase 3 can cache it."""
+    from backend.app.services import download_backends as db_mod
+    from backend.app.services.gallery_dl import SourceConfig
+
+    _artist, source = seed_artist_and_source
+
+    monkeypatch.setattr(
+        db_mod, "resolve_campaign_id_for_source",
+        AsyncMock(return_value=("4242", "4242")),
+    )
+    run_kwargs = {}
+
+    class _FakeIngester:
+        def __init__(self, **kw):
+            pass
+
+        def run(self, **kw):
+            run_kwargs.update(kw)
+            return _make_fake_dl_result(success=True, written_paths=[])
+
+    monkeypatch.setattr(db_mod, "PatreonIngester", _FakeIngester)
+
+    ctx = {
+        "platform": "patreon",
+        "source_id": source.id, "url": "https://patreon.com/alice",
+        "artist_slug": "alice", "cookies_path": None,
+        "config_overrides": {},
+    }
+    result, resolved = await db_mod.run_download(
+        ctx=ctx, source_config=SourceConfig.from_dict({}), skip_value=True,
+        mode="tick",
+        gdl=_fake_gdl_with_result(_make_fake_dl_result(success=True)),
+        sync_session_factory=MagicMock(),
+    )
+    assert result.success is True
+    assert resolved == "4242"
+    assert run_kwargs["campaign_id"] == "4242"
+    assert run_kwargs["mode"] == "tick"
+
+
+@pytest.mark.asyncio
+async def test_run_download_native_unresolvable_fails_loud(
+    db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
+):
+    """A campaign id we can't resolve is a loud NOT_FOUND failure, never a
+    silent empty success."""
+    from backend.app.services import download_backends as db_mod
+    from backend.app.services.gallery_dl import ErrorType, SourceConfig
+
+    _artist, source = seed_artist_and_source
+
+    monkeypatch.setattr(
+        db_mod, "resolve_campaign_id_for_source",
+        AsyncMock(return_value=(None, None)),
+    )
+    ctx = {
+        "platform": "patreon",
+        "source_id": source.id, "url": "https://patreon.com/alice",
+        "artist_slug": "alice", "cookies_path": None,
+        "config_overrides": {},
+    }
+    result, resolved = await db_mod.run_download(
+        ctx=ctx, source_config=SourceConfig.from_dict({}), skip_value=True,
+        mode="tick", gdl=MagicMock(), sync_session_factory=MagicMock(),
+    )
+    assert result.success is False
+    assert result.error_type == ErrorType.NOT_FOUND
+    assert resolved is None
+
+
 # --- FC-3d: finalize hook updates Source health columns -------------------
 
 
@@ -346,6 +437,40 @@ async def test_finalize_error_increments_failures_and_sets_error(db):
     assert row.last_checked_at is not None
 
 
+@pytest.mark.asyncio
+async def test_rate_limited_cooldown_honors_retry_after(db, monkeypatch):
+    """plan #708 B1: a RATE_LIMITED result with a Retry-After stamps the platform
+    cooldown at that duration, clamped to [60, 3600]; no hint → the flat default."""
+    from unittest.mock import AsyncMock, MagicMock
+
+    from backend.app.services import download_service as dl_mod
+    from backend.app.services.download_service import DownloadService
+
+    source_id, _event_id = await _seed_source_with_health(db, suffix="rl")
+    cooldown = AsyncMock()
+    monkeypatch.setattr(dl_mod, "set_platform_cooldown", cooldown)
+    svc = DownloadService(
+        async_session=db, sync_session=None,
+        gdl=MagicMock(), importer=MagicMock(), cred_service=MagicMock(),
+    )
+
+    async def _health(retry):
+        cooldown.reset_mock()
+        await svc._update_source_health(
+            source_id=source_id, status="error", error_message="rate limited",
+            error_type="rate_limited", retry_after_seconds=retry,
+        )
+
+    await _health(120.0)
+    assert cooldown.await_args.kwargs["seconds"] == 120  # honored as-is
+    await _health(5.0)
+    assert cooldown.await_args.kwargs["seconds"] == 60  # floored
+    await _health(99999.0)
+    assert cooldown.await_args.kwargs["seconds"] == 3600  # capped
+    await _health(None)
+    assert "seconds" not in cooldown.await_args.kwargs  # default cooldown
+
+
 @pytest.mark.asyncio
 async def test_finalize_skipped_preserves_failures_clears_error(db):
     from backend.app.services.download_service import DownloadService
@@ -376,8 +501,12 @@ async def test_finalize_skipped_preserves_failures_clears_error(db):
 
 
 def _backfill_svc(db, db_sync, tmp_path, result):
-    """DownloadService wired with a fake gdl returning `result` + a real
-    importer (so an empty written_paths just attaches nothing)."""
+    """DownloadService wired with a real importer (so empty written_paths just
+    attaches nothing) whose Patreon phase-2 branch is stubbed to return `result`.
+
+    The seeded source is Patreon, so phase 2 routes to the native ingester
+    (plan #697); the stub returns the canned result + captures the per-call
+    (ctx, source_config, mode). Returns (svc, ingester_calls)."""
     from backend.app.services.download_service import DownloadService
     from backend.app.services.importer import Importer
 
@@ -395,7 +524,8 @@ def _backfill_svc(db, db_sync, tmp_path, result):
         async_session=db, sync_session=db_sync,
         gdl=fake_gdl, importer=importer, cred_service=cred_service,
     )
-    return svc, fake_gdl
+    calls = _stub_patreon_ingester(svc, result)
+    return svc, calls
 
 
 @pytest.mark.asyncio
@@ -412,7 +542,7 @@ async def test_backfill_chunk_progress_advances_cursor(
 
     svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
         success=False, written_paths=[], files_downloaded=0,
-        stderr="[patreon][debug] Cursor: 03:PAGE2:xyz\n",
+        cursor="03:PAGE2:xyz",
     ))
     await svc.download_source(source.id)
 
@@ -430,28 +560,25 @@ async def test_backfill_chunk_progress_advances_cursor(
 async def test_backfill_state_running_selects_backfill_mode_and_resumes(
     db, db_sync, tmp_path, seed_artist_and_source,
 ):
-    """state=='running' drives backfill mode (skip:True + chunk budget) and
-    threads the stored cursor to gallery-dl as the resume point."""
-    from backend.app.services.gallery_dl import (
-        BACKFILL_CHUNK_SECONDS,
-        BACKFILL_SKIP_VALUE,
-    )
+    """state=='running' drives backfill mode (chunk budget) and threads the
+    stored cursor to the native ingester as the resume point."""
+    from backend.app.services.gallery_dl import BACKFILL_CHUNK_SECONDS
 
     _artist, source = seed_artist_and_source
     source.config_overrides = {"_backfill_state": "running", "_backfill_cursor": "03:RESUME:here"}
     source.backfill_runs_remaining = 5
     await db.commit()
 
-    svc, fake_gdl = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
+    svc, calls = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
         success=False, written_paths=[],
-        stderr="[patreon][debug] Cursor: 03:RESUME2:next\n",
+        cursor="03:RESUME2:next",
     ))
     await svc.download_source(source.id)
 
-    kwargs = fake_gdl.download.call_args.kwargs
-    assert kwargs["skip_value"] is BACKFILL_SKIP_VALUE
-    assert kwargs["source_config"].resume_cursor == "03:RESUME:here"
-    assert kwargs["source_config"].timeout == BACKFILL_CHUNK_SECONDS
+    assert len(calls) == 1
+    assert calls[0]["mode"] == "backfill"
+    assert calls[0]["source_config"].resume_cursor == "03:RESUME:here"
+    assert calls[0]["source_config"].timeout == BACKFILL_CHUNK_SECONDS
 
     co = (await db.execute(
         select(Source.config_overrides).where(Source.id == source.id)
@@ -497,7 +624,7 @@ async def test_backfill_cap_exhaustion_stalls(
 
     svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
         success=False, written_paths=[],
-        stderr="[patreon][debug] Cursor: 03:MORE:left\n",
+        cursor="03:MORE:left",
     ))
     await svc.download_source(source.id)
 
@@ -533,7 +660,7 @@ async def test_backfill_stuck_guard_stalls_after_two_no_advance(
         "config_overrides": {"_backfill_state": "running", "_backfill_cursor": "stuck"},
         "backfill_runs_remaining": 5,
     }
-    stuck = _make_fake_dl_result(success=False, stderr="Cursor: stuck\n")
+    stuck = _make_fake_dl_result(success=False, cursor="stuck")
 
     await svc._apply_backfill_lifecycle(ctx, stuck)
     await db.commit()
@@ -553,6 +680,41 @@ async def test_backfill_stuck_guard_stalls_after_two_no_advance(
     assert "_backfill_cursor" not in co2
 
 
+@pytest.mark.asyncio
+async def test_backfill_lifecycle_leaves_posts_to_ingester(
+    db, db_sync, tmp_path, seed_artist_and_source,
+):
+    """plan #704 (#5), revised: _backfill_posts is OWNED by the ingester now (it
+    writes a monotonic absolute live mid-walk), so the post-chunk lifecycle must
+    NOT touch it — it carries the ingester's committed value forward unchanged."""
+    from unittest.mock import MagicMock
+
+    from backend.app.services.download_service import DownloadService
+
+    _artist, source = seed_artist_and_source
+    source.config_overrides = {"_backfill_state": "running", "_backfill_posts": 12}
+    source.backfill_runs_remaining = 5
+    await db.commit()
+
+    svc = DownloadService(
+        async_session=db, sync_session=db_sync,
+        gdl=MagicMock(), importer=MagicMock(), cred_service=MagicMock(),
+    )
+    ctx = {
+        "source_id": source.id, "platform": "patreon",
+        "config_overrides": {"_backfill_state": "running", "_backfill_posts": 12},
+        "backfill_runs_remaining": 5,
+    }
+    await svc._apply_backfill_lifecycle(
+        ctx, _make_fake_dl_result(success=False, cursor="C1", posts_processed=10),
+    )
+    await db.commit()
+    co = (await db.execute(
+        select(Source.config_overrides).where(Source.id == source.id)
+    )).scalar_one()
+    assert co.get("_backfill_posts") == 12  # untouched — the ingester owns it
+
+
 @pytest.mark.asyncio
 async def test_backfill_timeout_chunk_reclassified_to_ok(
     db, db_sync, tmp_path, seed_artist_and_source,
@@ -571,7 +733,7 @@ async def test_backfill_timeout_chunk_reclassified_to_ok(
         success=False, files_downloaded=3,
         error_type=ErrorType.TIMEOUT,
         error_message="Download timed out",
-        stderr="[patreon][debug] Cursor: 03:NEXT:page\n",
+        cursor="03:NEXT:page",
     ))
     event_id = await svc.download_source(source.id)
 
@@ -605,6 +767,32 @@ async def test_tick_mode_when_not_running_leaves_state_untouched(
     assert (co or {}).get("_backfill_state") is None
 
 
+@pytest.mark.asyncio
+async def test_recovery_mode_selected_and_flag_cleared_on_complete(
+    db, db_sync, tmp_path, seed_artist_and_source,
+):
+    """plan #697: `_backfill_bypass_seen` alongside a running backfill selects
+    recovery mode (ingester bypasses the seen-ledger). A clean rc=0 walk
+    completes the shared lifecycle AND clears the bypass flag so the next tick
+    honors the ledger again."""
+    _artist, source = seed_artist_and_source
+    source.config_overrides = {"_backfill_state": "running", "_backfill_bypass_seen": True}
+    source.backfill_runs_remaining = 5
+    await db.commit()
+
+    svc, calls = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
+        success=True, written_paths=[], files_downloaded=0,
+    ))
+    await svc.download_source(source.id)
+
+    assert calls[0]["mode"] == "recovery"
+    co = (await db.execute(
+        select(Source.config_overrides).where(Source.id == source.id)
+    )).scalar_one()
+    assert co.get("_backfill_state") == "complete"
+    assert "_backfill_bypass_seen" not in co
+
+
 @pytest.mark.asyncio
 async def test_partial_error_type_maps_to_ok_status(
     db, db_sync, tmp_path, seed_artist_and_source,
@@ -642,6 +830,7 @@ async def test_partial_error_type_maps_to_ok_status(
         async_session=db, sync_session=db_sync,
         gdl=fake_gdl, importer=importer, cred_service=cred_service,
     )
+    _stub_patreon_ingester(svc, fake_result)
     event_id = await svc.download_source(source.id)
 
     ev = (await db.execute(
@@ -677,10 +866,11 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image(
     _make_jpg(f1, split="h")
     _make_jpg(f2, split="v")
 
-    fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
+    result = _make_fake_dl_result(
         success=True, written_paths=[str(f1), str(f2)],
         files_downloaded=2, stdout=f"{f1}\n{f2}\n",
-    ))
+    )
+    fake_gdl = _fake_gdl_with_result(result)
 
     sync_settings = db_sync.execute(
         select(ImportSettings).where(ImportSettings.id == 1)
@@ -713,6 +903,7 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image(
         async_session=db, sync_session=db_sync,
         gdl=fake_gdl, importer=importer, cred_service=cred_service,
     )
+    _stub_patreon_ingester(svc, result)
     await svc.download_source(source.id)
 
     # Two files attached → two thumbnail enqueues + two ML enqueues, IDs
@@ -727,11 +918,11 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image(
 async def test_releases_db_connections_before_subprocess(
     db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
 ):
-    """Phase 1's DB connections must be released BEFORE the (up to ~19.5-min
-    in backfill) gallery-dl subprocess, so they don't idle-die and strand
-    phase 3 with ConnectionDoesNotExistError (Anduo #40014). Spy on the
-    async session close and assert it happened before gdl.download runs;
-    phase 3 must still finalize the event."""
+    """Phase 1's DB connections must be released BEFORE the (multi-minute)
+    phase-2 fetch, so they don't idle-die and strand phase 3 with
+    ConnectionDoesNotExistError (Anduo #40014). Spy on the async session close
+    and assert it happened before the phase-2 work (here the native Patreon
+    ingester) runs; phase 3 must still finalize the event."""
     from backend.app.services.download_service import DownloadService
     from backend.app.services.importer import Importer
 
@@ -747,19 +938,9 @@ async def test_releases_db_connections_before_subprocess(
     monkeypatch.setattr(db, "close", spy_close)
 
     seen = {}
-
-    async def fake_download(url, **k):
-        seen["closed_before_download"] = closed["async"]
-        return _make_fake_dl_result(success=True, written_paths=[], stdout="")
-
-    fake_gdl = MagicMock()
-    fake_gdl.download = fake_download
-    fake_gdl._compute_run_stats = lambda *a, **k: {
-        "exit_code": 0, "downloaded_count": 0, "skipped_count": 0,
-        "per_item_failures": 0, "warning_count": 0, "tier_gated_count": 0,
-    }
-    fake_gdl._extract_errors_warnings = lambda *a, **k: ""
-    fake_gdl._truncate_log = lambda x, **k: x
+    fake_gdl = _fake_gdl_with_result(
+        _make_fake_dl_result(success=True, written_paths=[], stdout="")
+    )
 
     sync_settings = db_sync.execute(
         select(ImportSettings).where(ImportSettings.id == 1)
@@ -776,9 +957,15 @@ async def test_releases_db_connections_before_subprocess(
         async_session=db, sync_session=db_sync,
         gdl=fake_gdl, importer=importer, cred_service=cred_service,
     )
+
+    async def fake_ingest(*, ctx, source_config, skip_value, mode):
+        seen["closed_before_phase2"] = closed["async"]
+        return _make_fake_dl_result(success=True, written_paths=[], stdout=""), None
+
+    svc._run_download = fake_ingest
     event_id = await svc.download_source(source.id)
 
-    assert seen["closed_before_download"] is True
+    assert seen["closed_before_phase2"] is True
     ev = (await db.execute(
         select(DownloadEvent).where(DownloadEvent.id == event_id)
     )).scalar_one()

tests/test_gallery_dl_service.py

@@ -8,7 +8,6 @@ from backend.app.services.gallery_dl import (
     ErrorType,
     GalleryDLService,
     SourceConfig,
-    parse_last_cursor,
 )
 
 
@@ -260,7 +259,7 @@ def test_build_config_emits_tick_skip_value(gdl):
     scans once 20 contiguous archived items are seen."""
     from backend.app.services.gallery_dl import TICK_SKIP_VALUE
     cfg = gdl._build_config_for_source(
-        platform="patreon",
+        platform="subscribestar",
         source_config=SourceConfig(),
         artist_slug="alice",
         skip_value=TICK_SKIP_VALUE,
@@ -273,7 +272,7 @@ def test_build_config_emits_backfill_skip_value(gdl):
     full post history."""
     from backend.app.services.gallery_dl import BACKFILL_SKIP_VALUE
     cfg = gdl._build_config_for_source(
-        platform="patreon",
+        platform="subscribestar",
         source_config=SourceConfig(),
         artist_slug="alice",
         skip_value=BACKFILL_SKIP_VALUE,
@@ -316,49 +315,6 @@ def test_categorize_tier_limited_wins_over_partial(gdl):
         return_code=1, stdout=stdout, stderr=stderr,
     )
     assert etype == ErrorType.TIER_LIMITED
-
-
-def test_default_config_forwards_patreon_referer_to_ytdl(gdl):
-    """Operator-flagged 2026-06-01: Mux video playback restrictions reject
-    yt-dlp's manifest fetch when Referer/Origin don't match Patreon. The
-    fix is a static `downloader.ytdl.raw-options.http_headers` block, so
-    it's enough to assert the keys land in the default config."""
-    cfg = gdl._get_default_config()
-    headers = cfg["downloader"]["ytdl"]["raw-options"]["http_headers"]
-    assert headers["Referer"] == "https://www.patreon.com/"
-    assert headers["Origin"] == "https://www.patreon.com"
-
-
-# --- Cursor-paged backfill (plan #689) -------------------------------------
-
-
-def test_parse_last_cursor_returns_last_match_across_streams():
-    # gallery-dl logs `Cursor: <token>` per page (debug → stderr); the last
-    # is the furthest-progressed resume point.
-    stderr = (
-        "[patreon][debug] Cursor: 03:AAAA:bbb\n"
-        "[urllib3] GET ...\n"
-        "[patreon][debug] Cursor: 03:CCCC:ddd\n"
-    )
-    assert parse_last_cursor("", stderr) == "03:CCCC:ddd"
-
-
-def test_parse_last_cursor_scans_stdout_too_and_none_when_absent():
-    assert parse_last_cursor("Cursor: 99:ZZZ", "") == "99:ZZZ"
-    assert parse_last_cursor("no marker here", "still nothing") is None
-    assert parse_last_cursor("", "") is None
-
-
-def test_build_config_patreon_resume_cursor_overrides_default(gdl):
-    # No resume cursor → PLATFORM_DEFAULTS leaves the log-only `cursor: True`.
-    fresh = gdl._build_config_for_source(
-        "patreon", SourceConfig(), "alice", skip_value=True,
-    )
-    assert fresh["extractor"]["patreon"]["cursor"] is True
-
-    # Resume cursor set → gallery-dl restarts the walk from that page.
-    resumed = gdl._build_config_for_source(
-        "patreon", SourceConfig(resume_cursor="03:CCCC:ddd"), "alice",
-        skip_value=True,
-    )
-    assert resumed["extractor"]["patreon"]["cursor"] == "03:CCCC:ddd"
+# (The cursor-parsing tests were removed in plan #704: parse_last_cursor is gone
+# — the native ingester now carries its checkpoint cursor as a structured
+# DownloadResult.cursor field instead of a `Cursor:` log line to scrape.)

tests/test_patreon_client.py

@@ -0,0 +1,351 @@
+"""PatreonClient tests — parsing only, no network.
+
+The fixture is loaded from disk and fed directly to the pure parsing methods
+(_transform / extract_media / _validate_response / parse_cursor_from_url); no
+live requests call is exercised here.
+"""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from backend.app.services import patreon_client as pc_mod
+from backend.app.services.patreon_client import (
+    MediaItem,
+    PatreonAPIError,
+    PatreonAuthError,
+    PatreonClient,
+    PatreonDriftError,
+    _retry_after_seconds,
+    parse_cursor_from_url,
+)
+
+_FIXTURE = Path(__file__).parent / "fixtures" / "patreon_posts_page1.json"
+
+
+@pytest.fixture
+def response():
+    return json.loads(_FIXTURE.read_text())
+
+
+@pytest.fixture
+def client():
+    # No cookies file → empty session, but we never make a request in tests.
+    return PatreonClient(cookies_path=None)
+
+
+def _post_by_id(response, post_id):
+    for post in response["data"]:
+        if post["id"] == post_id:
+            return post
+    raise AssertionError(f"no post {post_id} in fixture")
+
+
+def test_transform_indexes_included(client, response):
+    index = client._transform(response)
+    assert index[("media", "9001")]["file_name"] == "gallery-one.jpg"
+    assert index[("media", "9003")]["file_name"] == "bonus-pack.zip"
+    assert index[("campaign", "5555")]["name"] == "Example Creator"
+    # Unknown keys are absent, not errors.
+    assert ("media", "0000") not in index
+
+
+def test_extract_gallery_dedups_image_large(client, response):
+    index = client._transform(response)
+    post = _post_by_id(response, "1001")
+    items = client.extract_media(post, index)
+
+    # Two gallery images; the image_large cover shares filehash aaaa... with
+    # gallery image 9001 → collapses to one, leaving exactly 2 items.
+    assert len(items) == 2
+    assert all(isinstance(m, MediaItem) for m in items)
+
+    kinds = [m.kind for m in items]
+    assert kinds == ["images", "images"]  # image_large deduped away
+
+    filenames = {m.filename for m in items}
+    assert filenames == {"gallery-one.jpg", "gallery-two.jpg"}
+
+    hashes = {m.filehash for m in items}
+    assert hashes == {"a" * 32, "b" * 32}
+    assert all(m.post_id == "1001" for m in items)
+
+
+def test_extract_attachment_postfile_same_file_collapses(client, response):
+    index = client._transform(response)
+    post = _post_by_id(response, "1002")
+    items = client.extract_media(post, index)
+
+    # The attachment (attachments_media 9003) and the post_file are the SAME
+    # zip — both carry filehash c…. attachments resolve before postfile, so the
+    # postfile collapses into the attachment by filehash → exactly one survives.
+    # (postfile-as-a-kind is exercised separately by the video post test.)
+    assert len(items) == 1
+    item = items[0]
+    assert item.kind == "attachments"
+    assert item.filename == "bonus-pack.zip"
+    assert item.filehash == "c" * 32
+
+
+def test_extract_inline_content_img(client, response):
+    index = client._transform(response)
+    post = _post_by_id(response, "1003")
+    items = client.extract_media(post, index)
+
+    assert len(items) == 1
+    item = items[0]
+    assert item.kind == "content"
+    assert item.filehash == "d" * 32
+    # & in the src must be unescaped to & .
+    assert "&" not in item.url
+    assert "token=x&v=2" in item.url
+
+
+def test_extract_video_postfile(client, response):
+    index = client._transform(response)
+    post = _post_by_id(response, "1004")
+    items = client.extract_media(post, index)
+
+    assert len(items) == 1
+    item = items[0]
+    assert item.kind == "postfile"
+    assert item.url.startswith("https://stream.mux.com/")
+    assert item.filehash == "e" * 32
+
+
+def test_parse_cursor_from_url_extracts_cursor(response):
+    next_url = response["links"]["next"]
+    assert parse_cursor_from_url(next_url) == "NEXTCURSOR123"
+
+
+def test_parse_cursor_from_url_none_when_absent():
+    assert parse_cursor_from_url("https://www.patreon.com/api/posts?sort=-x") is None
+    assert parse_cursor_from_url(None) is None
+
+
+def test_iter_posts_yields_triples_and_paginates(client, response, monkeypatch):
+    # Page 1 = the fixture (4 posts, links.next → NEXTCURSOR123); page 2 = one
+    # post, no links.next → stop. Monkeypatch _fetch so no network is touched.
+    page2 = {
+        "data": [{"id": "1005", "type": "post", "attributes": {"title": "older"}}],
+        "included": [],
+        "links": {},
+    }
+
+    def fake_fetch(campaign_id, cursor):
+        return page2 if cursor == "NEXTCURSOR123" else response
+
+    monkeypatch.setattr(client, "_fetch", fake_fetch)
+
+    seen = list(client.iter_posts("5555"))
+    # 4 posts on page 1 + 1 on page 2.
+    assert [p["id"] for p, _idx, _cur in seen] == ["1001", "1002", "1003", "1004", "1005"]
+    # page_cursor is the cursor that FETCHED the post's page.
+    cursors = {p["id"]: cur for p, _idx, cur in seen}
+    assert cursors["1001"] is None and cursors["1005"] == "NEXTCURSOR123"
+    # included_index is the page's flattened resources, ready for extract_media.
+    page1_index = next(idx for p, idx, _cur in seen if p["id"] == "1001")
+    assert page1_index[("media", "9001")]["file_name"] == "gallery-one.jpg"
+
+
+def test_missing_data_raises_drift(client):
+    with pytest.raises(PatreonDriftError):
+        client._validate_response({"included": []})
+
+
+def test_non_list_data_raises_drift(client):
+    with pytest.raises(PatreonDriftError):
+        client._validate_response({"data": {"id": "1"}})
+
+
+def test_media_missing_file_name_raises_drift(client):
+    # A media resource referenced by a gallery relationship but lacking
+    # file_name is drift from the contract.
+    post = {
+        "id": "2000",
+        "type": "post",
+        "attributes": {"title": "broken"},
+        "relationships": {"images": {"data": [{"id": "7777", "type": "media"}]}},
+    }
+    index = {
+        ("media", "7777"): {
+            "download_url": "https://c10.patreonusercontent.com/4/orig/"
+            + "f" * 32
+            + "/x.jpg"
+        }
+    }
+    with pytest.raises(PatreonDriftError):
+        client.extract_media(post, index)
+
+
+def test_media_referenced_but_absent_raises_drift(client):
+    post = {
+        "id": "2001",
+        "type": "post",
+        "attributes": {"title": "dangling"},
+        "relationships": {"images": {"data": [{"id": "8888", "type": "media"}]}},
+    }
+    with pytest.raises(PatreonDriftError):
+        client.extract_media(post, {})
+
+
+# -- HTTP status classification (plan #697 step 4) -------------------------
+
+
+class _FakeResp:
+    def __init__(self, status_code, *, json_data=None, raise_json=False, headers=None):
+        self.status_code = status_code
+        self._json_data = json_data
+        self._raise_json = raise_json
+        self.headers = headers or {}
+
+    def json(self):
+        if self._raise_json:
+            raise ValueError("Expecting value: line 1 column 1")
+        return self._json_data
+
+
+def _client_returning(monkeypatch, resp):
+    client = PatreonClient(cookies_path=None)
+    monkeypatch.setattr(client._session, "get", lambda *a, **k: resp)
+    return client
+
+
+@pytest.mark.parametrize("status", [401, 403])
+def test_fetch_auth_status_raises_auth_error(monkeypatch, status):
+    client = _client_returning(monkeypatch, _FakeResp(status))
+    with pytest.raises(PatreonAuthError) as ei:
+        client._fetch("5555", None)
+    assert ei.value.status_code == status
+
+
+def test_fetch_non_json_body_is_auth_not_drift(monkeypatch):
+    # An HTML login/challenge page (non-JSON) means the session expired — auth,
+    # actionable as "rotate cookies", NOT API drift.
+    client = _client_returning(monkeypatch, _FakeResp(200, raise_json=True))
+    with pytest.raises(PatreonAuthError):
+        client._fetch("5555", None)
+
+
+@pytest.mark.parametrize("status", [404, 429, 500])
+def test_fetch_other_status_raises_api_error_with_code(monkeypatch, status):
+    client = _client_returning(monkeypatch, _FakeResp(status))
+    with pytest.raises(PatreonAPIError) as ei:
+        client._fetch("5555", None)
+    assert ei.value.status_code == status
+    # Not auth — the ingester maps these by status, not to auth_error.
+    assert not isinstance(ei.value, PatreonAuthError)
+
+
+def test_fetch_ok_returns_payload(monkeypatch):
+    client = _client_returning(monkeypatch, _FakeResp(200, json_data={"data": []}))
+    assert client._fetch("5555", None) == {"data": []}
+
+
+# -- verify_auth (credential probe; (True/False/None, message) contract) ---
+
+
+def test_verify_auth_ok_when_page_authenticates(monkeypatch):
+    client = _client_returning(monkeypatch, _FakeResp(200, json_data={"data": []}))
+    ok, msg = client.verify_auth("5555")
+    assert ok is True
+    assert "valid" in msg.lower()
+
+
+@pytest.mark.parametrize("status", [401, 403])
+def test_verify_auth_false_when_rejected(monkeypatch, status):
+    client = _client_returning(monkeypatch, _FakeResp(status))
+    ok, _msg = client.verify_auth("5555")
+    assert ok is False
+
+
+def test_verify_auth_inconclusive_on_drift(monkeypatch):
+    # 200 + JSON but missing top-level 'data' → drift → inconclusive, NOT a
+    # credential verdict (our parser is stale, the cookie may be fine).
+    client = _client_returning(monkeypatch, _FakeResp(200, json_data={"included": []}))
+    ok, msg = client.verify_auth("5555")
+    assert ok is None
+    assert "api shape changed" in msg.lower()
+
+
+def test_verify_auth_inconclusive_on_network(monkeypatch):
+    import requests
+    client = PatreonClient(cookies_path=None)
+
+    def _raise(*a, **k):
+        raise requests.ConnectionError("boom")
+    monkeypatch.setattr(client._session, "get", _raise)
+    ok, _msg = client.verify_auth("5555")
+    assert ok is None
+
+
+# -- 429 backoff + pacing (plan #703) --------------------------------------
+
+
+def test_retry_after_seconds_honors_numeric_header():
+    assert _retry_after_seconds(_FakeResp(429, headers={"Retry-After": "7"}), 1) == 7.0
+
+
+def test_retry_after_seconds_exponential_without_header():
+    resp = _FakeResp(429)
+    assert _retry_after_seconds(resp, 1) == 2.0
+    assert _retry_after_seconds(resp, 2) == 4.0
+    assert _retry_after_seconds(resp, 3) == 8.0
+
+
+def test_retry_after_seconds_caps():
+    assert _retry_after_seconds(_FakeResp(429), 10) == 30.0
+    assert _retry_after_seconds(_FakeResp(429, headers={"Retry-After": "999"}), 1) == 30.0
+
+
+def _client_with_sequence(monkeypatch, responses):
+    """A client whose session yields `responses` in order; time.sleep stubbed so
+    backoff never really sleeps. Returns (client, slept_list)."""
+    client = PatreonClient(cookies_path=None)
+    it = iter(responses)
+    monkeypatch.setattr(client._session, "get", lambda *a, **k: next(it))
+    slept: list[float] = []
+    monkeypatch.setattr(pc_mod.time, "sleep", lambda s: slept.append(s))
+    return client, slept
+
+
+def test_fetch_retries_then_succeeds_on_429(monkeypatch):
+    client, slept = _client_with_sequence(monkeypatch, [
+        _FakeResp(429, headers={"Retry-After": "5"}),
+        _FakeResp(200, json_data={"data": []}),
+    ])
+    assert client._fetch("5555", None) == {"data": []}
+    assert 5.0 in slept  # backed off once before the retry
+
+
+def test_fetch_persistent_429_raises_after_retries(monkeypatch):
+    client, _slept = _client_with_sequence(monkeypatch, [_FakeResp(429)] * 10)
+    with pytest.raises(PatreonAPIError) as ei:
+        client._fetch("5555", None)
+    assert ei.value.status_code == 429
+    assert not isinstance(ei.value, PatreonAuthError)
+
+
+def test_fetch_persistent_429_surfaces_retry_after(monkeypatch):
+    """plan #708 B1: a terminal 429 carries the server's raw Retry-After seconds
+    (uncapped here — the cooldown clamps) so the platform cooldown matches it."""
+    client, _slept = _client_with_sequence(
+        monkeypatch, [_FakeResp(429, headers={"Retry-After": "120"})] * 10,
+    )
+    with pytest.raises(PatreonAPIError) as ei:
+        client._fetch("5555", None)
+    assert ei.value.status_code == 429
+    assert ei.value.retry_after == 120.0
+
+
+def test_request_sleep_paces_before_fetch(monkeypatch):
+    client = PatreonClient(cookies_path=None, request_sleep=1.5)
+    monkeypatch.setattr(
+        client._session, "get",
+        lambda *a, **k: _FakeResp(200, json_data={"data": []}),
+    )
+    slept: list[float] = []
+    monkeypatch.setattr(pc_mod.time, "sleep", lambda s: slept.append(s))
+    client._fetch("5555", None)
+    assert slept == [1.5]

tests/test_patreon_contract.py

@@ -0,0 +1,72 @@
+"""Patreon API contract test (native ingester build step 4).
+
+The drift tripwire the plan calls for: if the field-set we SEND or the parser we
+resolve responses against drifts from what real Patreon returns, this build goes
+RED — instead of the ingester silently importing nothing in production (the exact
+failure mode the native ingester exists to escape).
+
+Two halves:
+  - the recorded `/api/posts` fixture must still parse end-to-end (no drift), and
+  - the request params must still carry every field the parser depends on (the
+    fixture already has the data, so trimming the request would NOT trip the
+    parse half — this half guards the request shape directly).
+
+Pure: no network, no DB.
+"""
+
+import json
+from pathlib import Path
+
+from backend.app.services.patreon_client import MediaItem, PatreonClient
+
+_FIXTURE = Path(__file__).parent / "fixtures" / "patreon_posts_page1.json"
+
+# Pinned media total across the 4 fixture posts: 1001→2 (gallery, image_large
+# cover deduped), 1002→1 (attachment+postfile collapse), 1003→1 (inline content
+# img), 1004→1 (video postfile). A parser change that drops or doubles media
+# trips this.
+_EXPECTED_MEDIA_TOTAL = 5
+_KNOWN_KINDS = {"images", "image_large", "attachments", "postfile", "content"}
+
+
+def test_recorded_fixture_parses_end_to_end():
+    response = json.loads(_FIXTURE.read_text())
+    client = PatreonClient(cookies_path=None)
+
+    client._validate_response(response)  # top-level shape must hold
+    index = client._transform(response)
+
+    total = 0
+    for post in response["data"]:
+        items = client.extract_media(post, index)  # must not raise drift
+        for m in items:
+            assert isinstance(m, MediaItem)
+            assert m.url, f"empty url in post {post['id']}"
+            assert m.filename, f"empty filename in post {post['id']}"
+            assert m.kind in _KNOWN_KINDS, f"unknown kind {m.kind!r}"
+            assert m.post_id == post["id"]
+        total += len(items)
+
+    assert total == _EXPECTED_MEDIA_TOTAL
+
+
+def test_request_field_set_carries_parser_dependencies():
+    """The params we SEND must keep requesting the fields the parser resolves
+    against. Trimming `fields[media]=file_name`, an `include` relationship, or a
+    `fields[post]` key would make real responses parse to nothing — caught here,
+    not by the fixture parse above."""
+    params = PatreonClient(cookies_path=None)._params("5555", None)
+
+    media_fields = params["fields[media]"]
+    assert "file_name" in media_fields
+    assert "image_urls" in media_fields or "download_url" in media_fields
+
+    includes = params["include"]
+    for rel in ("images", "attachments_media", "media"):
+        assert rel in includes, f"missing include relationship {rel}"
+
+    post_fields = params["fields[post]"]
+    for field in ("content", "post_file", "image"):
+        assert field in post_fields, f"missing post field {field}"
+
+    assert params["filter[campaign_id]"] == "5555"

tests/test_patreon_downloader.py

@@ -0,0 +1,519 @@
+"""Unit tests for PatreonDownloader (build step 2b).
+
+No network, no real subprocess. The HTTP layer is stubbed via the `session`
+seam (a fake session whose `.get` returns a fake streaming response) and yt-dlp
+via monkeypatching `_run_ytdlp`. PURE module → no DB, so unmarked (fast lane).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+import requests
+
+from backend.app.services import patreon_downloader as pd_mod
+from backend.app.services.patreon_client import MediaItem
+from backend.app.services.patreon_downloader import (
+    MediaOutcome,
+    PatreonDownloader,
+    _sanitize,
+)
+from backend.app.utils.sidecar import find_sidecar
+
+# Minimal valid PNG so the file_validator passes on the happy path.
+_PNG_HEAD = b"\x89PNG\r\n\x1a\n"
+_PNG_TAIL = b"\x00\x00\x00\x00IEND\xaeB`\x82"
+_PNG_BYTES = _PNG_HEAD + (b"\x00" * 16) + _PNG_TAIL
+
+
+class _FakeResponse:
+    def __init__(self, payload: bytes, status_code: int = 200, headers=None):
+        self._payload = payload
+        self.status_code = status_code
+        self.headers = headers or {}
+
+    def raise_for_status(self):
+        if self.status_code >= 400:
+            raise requests.HTTPError(f"HTTP {self.status_code}")
+        return None
+
+    def iter_content(self, chunk_size=65536):
+        for i in range(0, len(self._payload), chunk_size):
+            yield self._payload[i : i + chunk_size]
+
+
+class _FakeSession:
+    """Records GETs and returns canned bytes (per-URL, default _PNG_BYTES)."""
+
+    def __init__(self, payloads: dict[str, bytes] | None = None):
+        self.payloads = payloads or {}
+        self.calls: list[str] = []
+
+    def get(self, url, stream=False, timeout=None, headers=None):
+        self.calls.append(url)
+        return _FakeResponse(self.payloads.get(url, _PNG_BYTES))
+
+
+class _SeqSession:
+    """Returns the queued items in order (for retry tests). An Exception item is
+    RAISED by get() instead of returned (simulates a transport error)."""
+
+    def __init__(self, responses):
+        self._responses = list(responses)
+        self.calls: list[str] = []
+
+    def get(self, url, stream=False, timeout=None, headers=None):
+        self.calls.append(url)
+        item = self._responses.pop(0)
+        if isinstance(item, Exception):
+            raise item
+        return item
+
+
+class _FlakyResp:
+    """A streaming response that yields its payload then optionally raises a
+    mid-stream transport error (for the Range-resume tests, plan #708 B5)."""
+
+    headers: dict = {}
+
+    def __init__(self, payload, status_code=200, die=False):
+        self._payload = payload
+        self.status_code = status_code
+        self._die = die
+
+    def raise_for_status(self):
+        return None
+
+    def iter_content(self, chunk_size=65536):
+        yield self._payload
+        if self._die:
+            raise requests.exceptions.ChunkedEncodingError("cut mid-stream")
+
+
+def _post(post_id="1001", title="My Post Title", published="2026-05-01T12:00:00Z"):
+    return {
+        "id": post_id,
+        "attributes": {
+            "title": title,
+            "content": "<p>body</p>",
+            "published_at": published,
+            "url": f"https://www.patreon.com/posts/{post_id}",
+        },
+    }
+
+
+def _img(name, post_id="1001", url=None):
+    return MediaItem(
+        url=url or f"https://cdn.patreon.com/{name}",
+        filename=name,
+        kind="images",
+        filehash=None,
+        post_id=post_id,
+    )
+
+
+def _downloader(tmp_path: Path, session=None, validate=True):
+    return PatreonDownloader(
+        images_root=tmp_path,
+        cookies_path=None,
+        validate=validate,
+        session=session or _FakeSession(),
+    )
+
+
+def test_path_layout_two_images(tmp_path):
+    dl = _downloader(tmp_path)
+    items = [_img("media1.png"), _img("media2.png")]
+    outcomes = dl.download_post(_post(), items, "artist-x")
+
+    post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title"
+    p1 = post_dir / "01_media1.png"
+    p2 = post_dir / "02_media2.png"
+    assert p1.is_file()
+    assert p2.is_file()
+    assert [o.status for o in outcomes] == ["downloaded", "downloaded"]
+    assert outcomes[0].path == p1
+    assert p1.read_bytes() == _PNG_BYTES
+
+
+def test_dir_sanitized(tmp_path):
+    dl = _downloader(tmp_path)
+    post = _post(title='bad/name:with*chars?')
+    dl.download_post(post, [_img("a.png")], "artist-x")
+    patreon_dir = tmp_path / "artist-x" / "patreon"
+    children = list(patreon_dir.iterdir())
+    assert len(children) == 1
+    name = children[0].name
+    assert "/" not in name
+    assert not any(c in name for c in '<>:"\\|?*')
+
+
+def test_no_date_prefix_when_unparseable(tmp_path):
+    dl = _downloader(tmp_path)
+    post = _post(published="not-a-date")
+    dl.download_post(post, [_img("a.png")], "artist-x")
+    patreon_dir = tmp_path / "artist-x" / "patreon"
+    children = [c.name for c in patreon_dir.iterdir()]
+    assert children == ["1001_My Post Title"]
+
+
+def test_sidecar_written_and_findable(tmp_path):
+    dl = _downloader(tmp_path)
+    outcomes = dl.download_post(_post(), [_img("media1.png")], "artist-x")
+    media_path = outcomes[0].path
+
+    sidecar = find_sidecar(media_path)
+    assert sidecar is not None
+    assert sidecar.name == "01_media1.json"
+
+    import json
+
+    data = json.loads(sidecar.read_text())
+    assert data["category"] == "patreon"
+    assert data["id"] == "1001"
+    assert data["title"] == "My Post Title"
+    assert data["url"] == "https://www.patreon.com/posts/1001"
+
+
+def test_skip_seen(tmp_path):
+    session = _FakeSession()
+    dl = _downloader(tmp_path, session=session)
+    outcomes = dl.download_post(
+        _post(), [_img("media1.png")], "artist-x", is_seen=lambda m: True
+    )
+    assert outcomes[0].status == "skipped_seen"
+    assert outcomes[0].path is None
+    assert session.calls == []
+    # No file written anywhere.
+    assert not (tmp_path / "artist-x").exists()
+
+
+def test_skip_disk(tmp_path):
+    dl = _downloader(tmp_path)
+    post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title"
+    post_dir.mkdir(parents=True)
+    existing = post_dir / "01_media1.png"
+    existing.write_bytes(b"already here")
+
+    outcomes = dl.download_post(_post(), [_img("media1.png")], "artist-x")
+    assert outcomes[0].status == "skipped_disk"
+    assert outcomes[0].path == existing
+    assert existing.read_bytes() == b"already here"  # untouched
+
+
+def test_video_routes_to_ytdlp(tmp_path, monkeypatch):
+    session = _FakeSession()
+    dl = _downloader(tmp_path, session=session)
+    captured = {}
+
+    def fake_ytdlp(url, dest, headers):
+        captured["url"] = url
+        captured["headers"] = headers
+        out = Path(dest).with_suffix(".mp4")
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out.write_bytes(b"video-bytes")
+        return out
+
+    monkeypatch.setattr(dl, "_run_ytdlp", fake_ytdlp)
+
+    video = MediaItem(
+        url="https://stream.mux.com/abc123.m3u8",
+        filename="clip.mp4",
+        kind="postfile",
+        filehash=None,
+        post_id="1001",
+    )
+    outcomes = dl.download_post(_post(), [video], "artist-x")
+
+    assert captured["url"] == "https://stream.mux.com/abc123.m3u8"
+    assert captured["headers"]["Referer"] == "https://www.patreon.com/"
+    assert captured["headers"]["Origin"] == "https://www.patreon.com"
+    assert session.calls == []  # NOT the plain-GET path
+    assert outcomes[0].status == "downloaded"
+    assert outcomes[0].path.read_bytes() == b"video-bytes"
+    # Sidecar written next to the actual (remuxed) output.
+    assert find_sidecar(outcomes[0].path) is not None
+
+
+def _video_item():
+    return MediaItem(
+        url="https://stream.mux.com/abc123.m3u8", filename="clip.mp4",
+        kind="postfile", filehash=None, post_id="1001",
+    )
+
+
+def test_video_transient_failure_retried_then_succeeds(tmp_path, monkeypatch):
+    """plan #705 #8: a TRANSIENT yt-dlp failure (TimeoutExpired) is retried within
+    the pass; a later success yields a downloaded outcome. (The GET path already
+    retried transients; the video path didn't until #8 closed the gap.)"""
+    calls = {"n": 0}
+
+    def fake_run(cmd, **kwargs):
+        calls["n"] += 1
+        if calls["n"] == 1:
+            raise pd_mod.subprocess.TimeoutExpired(cmd, kwargs.get("timeout"))
+        out = Path(cmd[cmd.index("-o") + 1].replace(".%(ext)s", ".mp4"))
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out.write_bytes(b"video-bytes")
+        return pd_mod.subprocess.CompletedProcess(cmd, 0, "", "")
+
+    monkeypatch.setattr(pd_mod.subprocess, "run", fake_run)
+    monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a, **_k: None)
+
+    dl = _downloader(tmp_path, session=_FakeSession())
+    outcomes = dl.download_post(_post(), [_video_item()], "artist-x")
+    assert calls["n"] == 2  # retried once after the transient blip
+    assert outcomes[0].status == "downloaded"
+    assert outcomes[0].path.read_bytes() == b"video-bytes"
+
+
+def test_video_permanent_failure_fails_fast(tmp_path, monkeypatch):
+    """plan #705 #8: a non-zero yt-dlp exit (CalledProcessError) is PERMANENT —
+    yt-dlp already did its own network retries — so we don't re-spawn; the item
+    errors straight to the per-item/dead-letter path."""
+    calls = {"n": 0}
+
+    def fake_run(cmd, **kwargs):
+        calls["n"] += 1
+        raise pd_mod.subprocess.CalledProcessError(1, cmd, stderr="ERROR: gone")
+
+    monkeypatch.setattr(pd_mod.subprocess, "run", fake_run)
+    monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a, **_k: None)
+
+    dl = _downloader(tmp_path, session=_FakeSession())
+    outcomes = dl.download_post(_post(), [_video_item()], "artist-x")
+    assert calls["n"] == 1  # no retry on a permanent failure
+    assert outcomes[0].status == "error"
+
+
+def test_one_failure_isolated(tmp_path):
+    class _BoomSession(_FakeSession):
+        def get(self, url, stream=False, timeout=None, headers=None):
+            self.calls.append(url)
+            if "bad" in url:
+                raise RuntimeError("network boom")
+            return _FakeResponse(_PNG_BYTES)
+
+    dl = _downloader(tmp_path, session=_BoomSession())
+    items = [
+        _img("good.png", url="https://cdn.patreon.com/good.png"),
+        _img("bad.png", url="https://cdn.patreon.com/bad.png"),
+    ]
+    outcomes = dl.download_post(_post(), items, "artist-x")
+    assert outcomes[0].status == "downloaded"
+    assert outcomes[1].status == "error"
+    assert "boom" in outcomes[1].error
+
+
+def test_validation_quarantines_corrupt(tmp_path):
+    # A .png whose bytes are not a valid PNG → validator fails → "quarantined"
+    # outcome (distinct from a download error) carrying the dest path +
+    # quarantine move; with validate=False it would pass. plan #704 (#4).
+    bad = b"not a real png"
+    session = _FakeSession({"https://cdn.patreon.com/corrupt.png": bad})
+    dl = _downloader(tmp_path, session=session, validate=True)
+    item = _img("corrupt.png", url="https://cdn.patreon.com/corrupt.png")
+    outcomes = dl.download_post(_post(), [item], "artist-x")
+    assert outcomes[0].status == "quarantined"
+    assert outcomes[0].path is not None
+    assert "_quarantine" in str(outcomes[0].path)
+    post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title"
+    assert not (post_dir / "01_corrupt.png").exists()
+    quarantine = tmp_path / "_quarantine" / "artist-x" / "patreon"
+    assert quarantine.is_dir()
+    assert any(quarantine.iterdir())
+
+
+def test_quarantine_writes_provenance_sidecar(tmp_path):
+    """A1 parity fix: the native path now writes the same `.quarantine.json`
+    provenance sidecar gallery-dl writes (it skipped it before the shared
+    file_validator.quarantine_file helper)."""
+    import json
+
+    bad = b"not a real png"
+    url = "https://cdn.patreon.com/corrupt.png"
+    session = _FakeSession({url: bad})
+    dl = _downloader(tmp_path, session=session, validate=True)
+    outcomes = dl.download_post(_post(), [_img("corrupt.png", url=url)], "artist-x")
+
+    dest = outcomes[0].path
+    sidecar = dest.with_suffix(dest.suffix + ".quarantine.json")
+    assert sidecar.is_file()
+    meta = json.loads(sidecar.read_text())
+    assert meta["source_url"] == url
+    assert meta["platform"] == "patreon"
+    assert meta["artist_slug"] == "artist-x"
+    assert meta["reason"]
+
+
+def test_validation_off_keeps_bytes(tmp_path):
+    bad = b"not a real png"
+    session = _FakeSession({"https://cdn.patreon.com/x.png": bad})
+    dl = _downloader(tmp_path, session=session, validate=False)
+    item = _img("x.png", url="https://cdn.patreon.com/x.png")
+    outcomes = dl.download_post(_post(), [item], "artist-x")
+    assert outcomes[0].status == "downloaded"
+    assert outcomes[0].path.read_bytes() == bad
+
+
+def test_sanitize_helper():
+    assert _sanitize('a/b') == "a_b"
+    assert _sanitize('x<>:"|?*y') == "x_______y"
+    assert _sanitize("trail.  ") == "trail"
+    assert _sanitize("") == "_"
+
+
+def test_media_outcome_shape():
+    o = MediaOutcome(media=None, status="downloaded", path=Path("/tmp/x"), error=None)
+    assert o.status == "downloaded"
+    assert o.path == Path("/tmp/x")
+
+
+# -- pacing + 429 backoff (plan #703) --------------------------------------
+
+
+def test_rate_limit_paces_each_real_download(tmp_path, monkeypatch):
+    slept: list[float] = []
+    monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s))
+    dl = PatreonDownloader(
+        images_root=tmp_path, cookies_path=None, validate=False,
+        rate_limit=2.0, session=_FakeSession(),
+    )
+    dl.download_post(_post(), [_img("a.png"), _img("b.png")], "artist-x")
+    # One pacing sleep per actual download (two media downloaded).
+    assert slept == [2.0, 2.0]
+
+
+def test_skips_do_not_pace(tmp_path, monkeypatch):
+    slept: list[float] = []
+    monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s))
+    dl = PatreonDownloader(
+        images_root=tmp_path, cookies_path=None, validate=False,
+        rate_limit=2.0, session=_FakeSession(),
+    )
+    # is_seen → skipped_seen, never reaches the download/pacing path.
+    dl.download_post(_post(), [_img("a.png")], "artist-x", is_seen=lambda m: True)
+    assert slept == []
+
+
+def test_media_429_retried_then_succeeds(tmp_path, monkeypatch):
+    slept: list[float] = []
+    monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s))
+    session = _SeqSession([
+        _FakeResponse(b"", status_code=429, headers={"Retry-After": "4"}),
+        _FakeResponse(_PNG_BYTES, status_code=200),
+    ])
+    dl = PatreonDownloader(
+        images_root=tmp_path, cookies_path=None, validate=False, session=session,
+    )
+    outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
+    assert outcomes[0].status == "downloaded"
+    assert 4.0 in slept  # backed off before the retry
+    assert len(session.calls) == 2
+
+
+def _dl(tmp_path, session, monkeypatch):
+    monkeypatch.setattr(pd_mod.time, "sleep", lambda s: None)
+    return PatreonDownloader(
+        images_root=tmp_path, cookies_path=None, validate=False, session=session,
+    )
+
+
+def test_transient_transport_error_retried_within_pass(tmp_path, monkeypatch):
+    """plan #705 #8: a connection blip on the GET is retried in-place, not a
+    terminal error that waits for the next walk."""
+    session = _SeqSession([
+        requests.ConnectionError("connection reset"),
+        _FakeResponse(_PNG_BYTES, status_code=200),
+    ])
+    dl = _dl(tmp_path, session, monkeypatch)
+    outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
+    assert outcomes[0].status == "downloaded"
+    assert len(session.calls) == 2
+
+
+def test_5xx_retried_within_pass(tmp_path, monkeypatch):
+    session = _SeqSession([
+        _FakeResponse(b"", status_code=503),
+        _FakeResponse(_PNG_BYTES, status_code=200),
+    ])
+    dl = _dl(tmp_path, session, monkeypatch)
+    outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
+    assert outcomes[0].status == "downloaded"
+    assert len(session.calls) == 2
+
+
+def test_404_fails_fast_no_retry(tmp_path, monkeypatch):
+    """A permanent 4xx (gone/forbidden) is NOT retried — it errors immediately
+    (and the ingester's dead-letter ledger takes over across walks)."""
+    session = _SeqSession([
+        _FakeResponse(b"", status_code=404),
+        _FakeResponse(_PNG_BYTES, status_code=200),  # would succeed if it retried
+    ])
+    dl = _dl(tmp_path, session, monkeypatch)
+    outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
+    assert outcomes[0].status == "error"
+    assert len(session.calls) == 1  # no retry on a permanent failure
+
+
+def test_exhausted_transient_becomes_error(tmp_path, monkeypatch):
+    session = _SeqSession([requests.Timeout("t")] * 10)  # always times out
+    dl = _dl(tmp_path, session, monkeypatch)
+    outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
+    assert outcomes[0].status == "error"
+
+
+def test_partial_download_resumes_via_range(tmp_path, monkeypatch):
+    """plan #708 B5: a mid-download transport cut resumes from the bytes already
+    on disk via a Range request, not a refetch from zero."""
+    full = bytes(i % 256 for i in range(1000))
+    split = 400
+
+    class _ResumeSession:
+        def __init__(self):
+            self.headers_seen = []
+
+        def get(self, url, stream=False, timeout=None, headers=None):
+            self.headers_seen.append(headers)
+            if len(self.headers_seen) == 1:
+                return _FlakyResp(full[:split], die=True)  # partial then cut
+            start = int(headers["Range"].split("=")[1].rstrip("-"))
+            return _FlakyResp(full[start:], status_code=206)
+
+    monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a: None)
+    session = _ResumeSession()
+    dl = PatreonDownloader(
+        images_root=tmp_path, cookies_path=None, validate=False, session=session,
+    )
+    outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
+    assert outcomes[0].status == "downloaded"
+    assert outcomes[0].path.read_bytes() == full
+    assert session.headers_seen[0] is None                      # first GET: no Range
+    assert session.headers_seen[1] == {"Range": "bytes=400-"}   # resume from offset
+
+
+def test_range_ignored_by_server_refetches_clean(tmp_path, monkeypatch):
+    """plan #708 B5: if the server ignores Range (200, not 206) on the resume, we
+    truncate and start clean — never append into a corrupt double-write."""
+    full = b"ABCD" * 250  # 1000 bytes
+    split = 400
+
+    class _NoRangeSession:
+        def __init__(self):
+            self.n = 0
+
+        def get(self, url, stream=False, timeout=None, headers=None):
+            self.n += 1
+            if self.n == 1:
+                return _FlakyResp(full[:split], die=True)
+            return _FlakyResp(full, status_code=200)  # ignores Range, whole file
+
+    monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a: None)
+    dl = PatreonDownloader(
+        images_root=tmp_path, cookies_path=None, validate=False,
+        session=_NoRangeSession(),
+    )
+    outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
+    assert outcomes[0].status == "downloaded"
+    assert outcomes[0].path.read_bytes() == full  # clean, not full[:400] + full

tests/test_patreon_ingester.py

@@ -0,0 +1,711 @@
+"""PatreonIngester tests (native ingester build step 3).
+
+The HTTP client and the media downloader are stubbed (injected seams), so these
+exercise the ingester's own logic — mode behavior, the two-tier skip, cursor
+emission, the budget cutoff, and the Postgres seen-ledger — without network or a
+real CDN. The ledger is real (a sync sessionmaker bound to the test engine), so
+the tier-1 skip and the idempotent mark-seen run against actual rows.
+"""
+
+import pytest
+from sqlalchemy import func, select
+from sqlalchemy.orm import sessionmaker
+
+from backend.app.models import (
+    Artist,
+    PatreonFailedMedia,
+    PatreonSeenMedia,
+    Source,
+)
+from backend.app.services.gallery_dl import ErrorType
+from backend.app.services.patreon_client import (
+    MediaItem,
+    PatreonAPIError,
+    PatreonAuthError,
+    PatreonDriftError,
+)
+from backend.app.services.patreon_downloader import MediaOutcome
+from backend.app.services.patreon_ingester import PatreonIngester, _ledger_key
+
+pytestmark = pytest.mark.integration
+
+
+# --- fakes ----------------------------------------------------------------
+
+
+def _media(post_id, n, *, filehash=None, kind="images"):
+    # Deterministic 32-hex-ish key, unique per (post_id, n).
+    fh = filehash if filehash is not None else f"{post_id}{n}".encode().hex().ljust(32, "0")[:32]
+    return MediaItem(
+        url=f"https://cdn.patreon.com/{fh}/{n}.jpg",
+        filename=f"{n}.jpg",
+        kind=kind,
+        filehash=fh,
+        post_id=post_id,
+    )
+
+
+class _FakeClient:
+    """Stub PatreonClient. `pages` is a list of (page_cursor, [posts]); each post
+    is (post_id, [MediaItem]). `raise_on_first` lets a test trip drift."""
+
+    def __init__(self, pages, raise_exc=None):
+        self._pages = pages
+        self._raise_exc = raise_exc
+        self.consumed_posts = 0
+
+    def iter_posts(self, campaign_id, cursor=None):
+        if self._raise_exc is not None:
+            raise self._raise_exc
+        for page_cursor, posts in self._pages:
+            for post_id, media in posts:
+                self.consumed_posts += 1
+                yield {"id": post_id, "_media": media}, {}, page_cursor
+
+    def extract_media(self, post, included_index):
+        return post["_media"]
+
+    def post_meta(self, post):
+        return {"title": post.get("id"), "date": None}
+
+
+class _FakeDownloader:
+    """Stub PatreonDownloader. Honors the injected is_seen (tier-1); media in
+    `on_disk` report skipped_disk (tier-2); media in `quarantine` report
+    quarantined; everything else downloads."""
+
+    def __init__(self, tmp_path, on_disk=None, quarantine=None, error=None):
+        self.tmp_path = tmp_path
+        self.on_disk = set(on_disk or ())
+        self.quarantine = set(quarantine or ())
+        self.error = set(error or ())
+        self.download_calls = 0
+
+    def download_post(self, post, media_items, artist_slug, *, is_seen):
+        outcomes = []
+        for m in media_items:
+            if is_seen(m):
+                outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
+            elif _ledger_key(m) in self.on_disk:
+                p = self.tmp_path / f"{m.post_id}_{m.filename}"
+                outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None))
+            elif _ledger_key(m) in self.error:
+                self.download_calls += 1
+                outcomes.append(MediaOutcome(media=m, status="error", path=None, error="boom"))
+            elif _ledger_key(m) in self.quarantine:
+                self.download_calls += 1
+                q = self.tmp_path / "_quarantine" / f"{m.post_id}_{m.filename}"
+                outcomes.append(MediaOutcome(media=m, status="quarantined", path=q, error="bad image"))
+            else:
+                self.download_calls += 1
+                p = self.tmp_path / f"{m.post_id}_{m.filename}"
+                p.write_bytes(b"x")
+                outcomes.append(MediaOutcome(media=m, status="downloaded", path=p, error=None))
+        return outcomes
+
+
+@pytest.fixture
+async def source_id(db):
+    artist = Artist(name="Ingest", slug="ingest")
+    db.add(artist)
+    await db.flush()
+    source = Source(
+        artist_id=artist.id, platform="patreon",
+        url="https://patreon.com/ingest", enabled=True, config_overrides={},
+    )
+    db.add(source)
+    await db.commit()
+    return source.id
+
+
+def _ingester(sync_engine, tmp_path, client, downloader):
+    factory = sessionmaker(sync_engine, expire_on_commit=False)
+    return PatreonIngester(
+        images_root=tmp_path, cookies_path=None, session_factory=factory,
+        client=client, downloader=downloader,
+    )
+
+
+def _count_ledger(sync_engine, source_id):
+    factory = sessionmaker(sync_engine, expire_on_commit=False)
+    with factory() as s:
+        return s.execute(
+            select(func.count(PatreonSeenMedia.id)).where(
+                PatreonSeenMedia.source_id == source_id
+            )
+        ).scalar_one()
+
+
+# --- tick -----------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_tick_downloads_unseen_and_marks_seen(source_id, sync_engine, tmp_path):
+    m1, m2 = _media("p1", 1), _media("p1", 2)
+    client = _FakeClient([(None, [("p1", [m1, m2])])])
+    downloader = _FakeDownloader(tmp_path)
+    ing = _ingester(sync_engine, tmp_path, client, downloader)
+
+    result = ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="tick",
+    )
+
+    assert result.success is True
+    assert result.return_code == 0
+    assert result.files_downloaded == 2
+    assert len(result.written_paths) == 2
+    # plan #704: structured run_stats carry the real counts.
+    assert result.run_stats["downloaded_count"] == 2
+    assert result.posts_processed == 1
+    assert _count_ledger(sync_engine, source_id) == 2
+
+
+@pytest.mark.asyncio
+async def test_quarantined_media_surfaced_in_result(source_id, sync_engine, tmp_path):
+    """plan #704 (#4): a media that fails validation is reported as quarantined —
+    a real files_quarantined count + paths + run_stats — not a blank 0, and is
+    NOT written or marked seen."""
+    m1, m2 = _media("p1", 1), _media("p1", 2)
+    client = _FakeClient([(None, [("p1", [m1, m2])])])
+    downloader = _FakeDownloader(tmp_path, quarantine={_ledger_key(m2)})
+    ing = _ingester(sync_engine, tmp_path, client, downloader)
+
+    result = ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="tick",
+    )
+    assert result.files_downloaded == 1          # only m1
+    assert result.files_quarantined == 1         # m2
+    assert len(result.quarantined_paths) == 1
+    assert result.run_stats["quarantined_count"] == 1
+    assert result.run_stats["downloaded_count"] == 1
+    assert len(result.written_paths) == 1        # quarantined NOT written
+    # Quarantined media is NOT marked seen (a fixed file may be re-fetched).
+    assert _count_ledger(sync_engine, source_id) == 1
+
+
+@pytest.mark.asyncio
+async def test_tick_skips_seen_via_ledger(source_id, sync_engine, tmp_path):
+    m1 = _media("p1", 1)
+    # Pre-seed the ledger with m1's key.
+    factory = sessionmaker(sync_engine, expire_on_commit=False)
+    with factory() as s:
+        s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
+        s.commit()
+
+    client = _FakeClient([(None, [("p1", [m1])])])
+    downloader = _FakeDownloader(tmp_path)
+    ing = _ingester(sync_engine, tmp_path, client, downloader)
+
+    result = ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="tick",
+    )
+    assert result.files_downloaded == 0
+    assert downloader.download_calls == 0
+
+
+@pytest.mark.asyncio
+async def test_tick_early_out_after_threshold(source_id, sync_engine, tmp_path):
+    # Three posts, one already-seen media each; threshold 2 → stop before the 3rd.
+    seen_media = [_media(f"p{i}", 1) for i in range(1, 4)]
+    factory = sessionmaker(sync_engine, expire_on_commit=False)
+    with factory() as s:
+        for m in seen_media:
+            s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m), post_id=m.post_id))
+        s.commit()
+
+    pages = [(None, [(m.post_id, [m]) for m in seen_media])]
+    client = _FakeClient(pages)
+    downloader = _FakeDownloader(tmp_path)
+    ing = _ingester(sync_engine, tmp_path, client, downloader)
+
+    result = ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="tick", seen_threshold=2,
+    )
+    assert result.success is True
+    # Stopped after the 2nd contiguous seen item — never consumed the 3rd post.
+    assert client.consumed_posts == 2
+
+
+# --- backfill -------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_backfill_reaches_bottom_signals_complete(source_id, sync_engine, tmp_path):
+    pages = [
+        (None, [("p1", [_media("p1", 1)])]),
+        ("CUR2", [("p2", [_media("p2", 1)])]),
+    ]
+    client = _FakeClient(pages)
+    downloader = _FakeDownloader(tmp_path)
+    ing = _ingester(sync_engine, tmp_path, client, downloader)
+
+    result = ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="backfill",
+    )
+    # rc 0 + error_type None is what _apply_backfill_lifecycle reads as "complete".
+    assert result.return_code == 0
+    assert result.error_type is None
+    assert result.files_downloaded == 2
+    # The page-2 cursor is carried structurally for checkpointing (plan #704).
+    assert result.cursor == "CUR2"
+    assert result.posts_processed == 2
+    assert result.run_stats["downloaded_count"] == 2
+
+
+@pytest.mark.asyncio
+async def test_backfill_checkpoints_cursor_mid_walk(source_id, sync_engine, tmp_path, db):
+    """plan #705 #6: the cursor is persisted to the DB at each page boundary
+    DURING the walk (not just at the end via phase 3), so a worker SIGKILL
+    mid-chunk resumes near the crash. After the walk the source carries the last
+    page's cursor — written by the ingester, not phase 3 (which the unit run
+    never reaches)."""
+    pages = [
+        (None, [("p1", [_media("p1", 1)])]),
+        ("CUR2", [("p2", [_media("p2", 1)])]),
+        ("CUR3", [("p3", [_media("p3", 1)])]),
+    ]
+    ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path))
+    ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="backfill",
+    )
+    co = (await db.execute(
+        select(Source.config_overrides).where(Source.id == source_id)
+    )).scalar_one()
+    assert co.get("_backfill_cursor") == "CUR3"
+
+
+@pytest.mark.asyncio
+async def test_backfill_persists_live_posts_count(source_id, sync_engine, tmp_path, db):
+    """plan #704 #5: the ingester writes a monotonic _backfill_posts absolute
+    DURING the walk (seeded from prior chunks via posts_base), EXCLUDING the
+    re-walked resume page so the count doesn't inflate across chunks."""
+    # Resume from CUR2 (a prior chunk left off here): its page is re-walked and
+    # must NOT be re-counted. posts_base=4 stands in for the prior chunks.
+    pages = [
+        ("CUR2", [("p1", [_media("p1", 1)])]),   # the resumed page — not counted
+        ("CUR3", [("p2", [_media("p2", 1)])]),   # net-new
+        ("CUR4", [("p3", [_media("p3", 1)])]),   # net-new
+    ]
+    ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path))
+    ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="backfill",
+        resume_cursor="CUR2", posts_base=4,
+    )
+    co = (await db.execute(
+        select(Source.config_overrides).where(Source.id == source_id)
+    )).scalar_one()
+    # 4 prior + 2 net-new (p2, p3); p1 on the resumed CUR2 page is excluded.
+    assert co.get("_backfill_posts") == 6
+
+
+@pytest.mark.asyncio
+async def test_tick_does_not_persist_posts(source_id, sync_engine, tmp_path, db):
+    """A tick has no resumable backfill state, so it must not write _backfill_posts
+    (the badge is a backfill/recovery concept)."""
+    ing = _ingester(
+        sync_engine, tmp_path,
+        _FakeClient([("CUR2", [("p1", [_media("p1", 1)])])]),
+        _FakeDownloader(tmp_path),
+    )
+    ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="tick",
+    )
+    co = (await db.execute(
+        select(Source.config_overrides).where(Source.id == source_id)
+    )).scalar_one()
+    assert "_backfill_posts" not in (co or {})
+
+
+@pytest.mark.asyncio
+async def test_tick_does_not_checkpoint_cursor(source_id, sync_engine, tmp_path, db):
+    """A tick has no resumable backfill state, so it must not write a cursor."""
+    ing = _ingester(
+        sync_engine, tmp_path,
+        _FakeClient([("CUR2", [("p1", [_media("p1", 1)])])]),
+        _FakeDownloader(tmp_path),
+    )
+    ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="tick",
+    )
+    co = (await db.execute(
+        select(Source.config_overrides).where(Source.id == source_id)
+    )).scalar_one()
+    assert "_backfill_cursor" not in (co or {})
+
+
+@pytest.mark.asyncio
+async def test_backfill_budget_cut_returns_partial_with_progress(
+    source_id, sync_engine, tmp_path, monkeypatch,
+):
+    # Deterministic clock: start=0, post1 check=10 (ok), post2 check=200 (>budget).
+    # run() lives in ingest_core now, so patch the clock there (the same global
+    # time module object, but we reference it through the module that uses it).
+    import backend.app.services.ingest_core as core
+    ticks = iter([0.0, 10.0, 200.0, 250.0])
+    last = [0.0]
+
+    def fake_monotonic():
+        try:
+            last[0] = next(ticks)
+        except StopIteration:
+            pass
+        return last[0]
+
+    monkeypatch.setattr(core.time, "monotonic", fake_monotonic)
+
+    pages = [
+        ("CUR1", [("p1", [_media("p1", 1)])]),
+        ("CUR2", [("p2", [_media("p2", 1)])]),
+    ]
+    client = _FakeClient(pages)
+    downloader = _FakeDownloader(tmp_path)
+    ing = _ingester(sync_engine, tmp_path, client, downloader)
+
+    result = ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="backfill",
+        resume_cursor=None, time_budget_seconds=100.0,
+    )
+    assert result.error_type == ErrorType.PARTIAL
+    assert result.return_code == -1
+    assert result.files_downloaded == 1  # post1 downloaded before the cut
+    assert downloader.download_calls == 1  # post2's body never ran
+    # The checkpoint is the page we were CUT ON (CUR2 — entered, cursor emitted,
+    # then the budget check broke before processing it); the next chunk resumes
+    # there. Structured (plan #704), matching the old parse_last_cursor(last).
+    assert result.cursor == "CUR2"
+
+
+@pytest.mark.asyncio
+async def test_preview_counts_new_media_without_downloading(
+    source_id, sync_engine, tmp_path,
+):
+    """plan #708 B4: preview walks + counts media not already seen/dead, downloads
+    nothing, and samples only posts that have new media."""
+    m1, m2, m3 = _media("p1", 1), _media("p2", 1), _media("p2", 2)
+    # Seed m1 as already-seen → only p2's two media are "new".
+    factory = sessionmaker(sync_engine, expire_on_commit=False)
+    with factory() as s:
+        s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
+        s.commit()
+    client = _FakeClient([(None, [("p1", [m1]), ("p2", [m2, m3])])])
+    downloader = _FakeDownloader(tmp_path)
+    ing = _ingester(sync_engine, tmp_path, client, downloader)
+
+    result = ing.preview(source_id, "c1")
+    assert result["total_new"] == 2          # p2's m2 + m3 (m1 already seen)
+    assert result["posts_scanned"] == 2
+    assert result["has_more"] is False
+    assert downloader.download_calls == 0     # dry-run — nothing downloaded
+    # The sample lists only posts WITH new media (p2), not the all-seen p1.
+    assert [row["new"] for row in result["sample"]] == [2]
+
+
+@pytest.mark.asyncio
+async def test_preview_page_limit_caps_walk(source_id, sync_engine, tmp_path):
+    """plan #708 B4: preview stops after page_limit pages and flags has_more."""
+    pages = [(f"CUR{i}", [(f"p{i}", [_media(f"p{i}", 1)])]) for i in range(6)]
+    ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path))
+    result = ing.preview(source_id, "c1", page_limit=2)
+    assert result["pages_scanned"] == 2
+    assert result["posts_scanned"] == 2   # one post per page, 2 pages
+    assert result["has_more"] is True
+
+
+@pytest.mark.asyncio
+async def test_still_running_reads_backfill_state(source_id, sync_engine, tmp_path, db):
+    """plan #708 B4: _still_running reflects the source's `_backfill_state` — the
+    flag an operator Stop pops to cancel a live chunk."""
+    ing = _ingester(sync_engine, tmp_path, _FakeClient([]), _FakeDownloader(tmp_path))
+    assert ing._still_running(source_id) is False  # absent → not running
+    src = (await db.execute(select(Source).where(Source.id == source_id))).scalar_one()
+    src.config_overrides = {"_backfill_state": "running"}
+    await db.commit()
+    assert ing._still_running(source_id) is True
+
+
+@pytest.mark.asyncio
+async def test_backfill_stops_when_operator_cancels_mid_walk(
+    source_id, sync_engine, tmp_path,
+):
+    """plan #708 B4: with the running state latched, a later page boundary that
+    finds it gone (Stop) bails the chunk with PARTIAL instead of finishing."""
+    pages = [
+        ("CUR2", [("p1", [_media("p1", 1)])]),
+        ("CUR3", [("p2", [_media("p2", 1)])]),
+        ("CUR4", [("p3", [_media("p3", 1)])]),
+    ]
+    downloader = _FakeDownloader(tmp_path)
+    ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), downloader)
+
+    # Latch on the page-2 boundary (running), then "Stop" before page 3.
+    calls = {"n": 0}
+
+    def fake_still_running(_source_id):
+        calls["n"] += 1
+        return calls["n"] == 1
+
+    ing._still_running = fake_still_running
+
+    result = ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="backfill",
+    )
+    assert result.error_type == ErrorType.PARTIAL
+    assert "Stopped by operator" in result.error_message
+    assert downloader.download_calls == 1  # only p1 ran; cancelled before p2
+
+
+# --- recovery -------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_recovery_bypasses_seen_ledger(source_id, sync_engine, tmp_path):
+    m1 = _media("p1", 1)
+    factory = sessionmaker(sync_engine, expire_on_commit=False)
+    with factory() as s:
+        s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
+        s.commit()
+
+    client = _FakeClient([(None, [("p1", [m1])])])
+    downloader = _FakeDownloader(tmp_path)
+    ing = _ingester(sync_engine, tmp_path, client, downloader)
+
+    result = ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="recovery",
+    )
+    # Ledger says seen, but recovery bypasses tier-1 → re-downloaded for a fresh
+    # pHash evaluation under the current threshold.
+    assert result.files_downloaded == 1
+    assert downloader.download_calls == 1
+
+
+@pytest.mark.asyncio
+async def test_recovery_tier2_disk_still_skips(source_id, sync_engine, tmp_path):
+    m1 = _media("p1", 1)
+    client = _FakeClient([(None, [("p1", [m1])])])
+    # File still on disk (a kept image) → tier-2 spares it even under recovery.
+    downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
+    ing = _ingester(sync_engine, tmp_path, client, downloader)
+
+    result = ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="recovery",
+    )
+    assert result.files_downloaded == 0
+    assert downloader.download_calls == 0
+    # Disk-skip still reconciles the ledger so a later tick skips at tier-1.
+    assert _count_ledger(sync_engine, source_id) == 1
+
+
+# --- dead-letter ledger (plan #705 #7) ------------------------------------
+
+
+def _count_failed(sync_engine, source_id):
+    factory = sessionmaker(sync_engine, expire_on_commit=False)
+    with factory() as s:
+        return s.execute(
+            select(func.count(PatreonFailedMedia.id)).where(
+                PatreonFailedMedia.source_id == source_id
+            )
+        ).scalar_one()
+
+
+def _seed_failed(sync_engine, source_id, key, attempts):
+    factory = sessionmaker(sync_engine, expire_on_commit=False)
+    with factory() as s:
+        s.add(PatreonFailedMedia(source_id=source_id, filehash=key, attempts=attempts))
+        s.commit()
+
+
+@pytest.mark.asyncio
+async def test_repeated_failures_dead_letter_then_skip(source_id, sync_engine, tmp_path):
+    """A media that errors DEAD_LETTER_THRESHOLD times is recorded; the next walk
+    skips it (no download attempt) and counts it dead-lettered."""
+    from backend.app.services.patreon_ingester import DEAD_LETTER_THRESHOLD
+
+    m1 = _media("p1", 1)
+
+    def _run():
+        ing = _ingester(
+            sync_engine, tmp_path,
+            _FakeClient([(None, [("p1", [m1])])]),
+            _FakeDownloader(tmp_path, error={_ledger_key(m1)}),
+        )
+        result = ing.run(
+            source_id=source_id, campaign_id="c1", artist_slug="ingest",
+            url="https://patreon.com/ingest", mode="tick",
+        )
+        return result, ing.downloader
+
+    # Fail it up to the threshold — each run attempts the download and errors.
+    for _ in range(DEAD_LETTER_THRESHOLD):
+        result, dl = _run()
+        assert dl.download_calls == 1
+    assert _count_failed(sync_engine, source_id) == 1
+
+    # Now attempts == threshold → dead: the next walk skips it without trying.
+    result, dl = _run()
+    assert dl.download_calls == 0
+    assert result.run_stats["dead_lettered_count"] == 1
+
+
+@pytest.mark.asyncio
+async def test_recovery_reattempts_dead_lettered_and_clears_on_success(
+    source_id, sync_engine, tmp_path,
+):
+    """Recovery bypasses the dead-letter ledger (re-attempts dead media); a clean
+    download clears the row (it recovered)."""
+    from backend.app.services.patreon_ingester import DEAD_LETTER_THRESHOLD
+
+    m1 = _media("p1", 1)
+    _seed_failed(sync_engine, source_id, _ledger_key(m1), DEAD_LETTER_THRESHOLD)
+
+    downloader = _FakeDownloader(tmp_path)  # succeeds
+    ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader)
+    result = ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="recovery",
+    )
+    assert downloader.download_calls == 1   # re-attempted despite being dead
+    assert result.files_downloaded == 1
+    assert _count_failed(sync_engine, source_id) == 0  # cleared on success
+
+
+@pytest.mark.asyncio
+async def test_clean_download_clears_below_threshold_failure(
+    source_id, sync_engine, tmp_path,
+):
+    """A media with a sub-threshold failure that now downloads cleanly (tick) has
+    its dead-letter row cleared."""
+    m1 = _media("p1", 1)
+    _seed_failed(sync_engine, source_id, _ledger_key(m1), 1)  # not yet dead
+
+    downloader = _FakeDownloader(tmp_path)
+    ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader)
+    result = ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="tick",
+    )
+    assert result.files_downloaded == 1
+    assert _count_failed(sync_engine, source_id) == 0
+
+
+# --- ledger + drift -------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_mark_seen_is_idempotent(source_id, sync_engine, tmp_path):
+    m1 = _media("p1", 1)
+    client = _FakeClient([(None, [("p1", [m1])])])
+    downloader = _FakeDownloader(tmp_path)
+    ing = _ingester(sync_engine, tmp_path, client, downloader)
+
+    # Same media downloaded twice across two runs — the second marks seen again.
+    ing._mark_seen(source_id, [(_ledger_key(m1), "p1")])
+    ing._mark_seen(source_id, [(_ledger_key(m1), "p1")])
+    assert _count_ledger(sync_engine, source_id) == 1
+
+
+def _run_with_client_error(source_id, sync_engine, tmp_path, exc):
+    client = _FakeClient([], raise_exc=exc)
+    ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path))
+    return ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="tick",
+    )
+
+
+@pytest.mark.asyncio
+async def test_drift_maps_to_api_drift_and_fails_loud(source_id, sync_engine, tmp_path):
+    result = _run_with_client_error(
+        source_id, sync_engine, tmp_path,
+        PatreonDriftError("missing top-level 'data' key"),
+    )
+    assert result.success is False
+    assert result.error_type == ErrorType.API_DRIFT
+    assert "Patreon API changed" in (result.error_message or "")
+
+
+@pytest.mark.asyncio
+async def test_auth_error_maps_to_auth_error(source_id, sync_engine, tmp_path):
+    result = _run_with_client_error(
+        source_id, sync_engine, tmp_path,
+        PatreonAuthError("HTTP 403 — auth rejected", status_code=403),
+    )
+    assert result.success is False
+    assert result.error_type == ErrorType.AUTH_ERROR
+
+
+@pytest.mark.asyncio
+async def test_rate_limit_status_maps_to_rate_limited(source_id, sync_engine, tmp_path):
+    result = _run_with_client_error(
+        source_id, sync_engine, tmp_path,
+        PatreonAPIError("HTTP 429", status_code=429),
+    )
+    assert result.error_type == ErrorType.RATE_LIMITED
+
+
+@pytest.mark.asyncio
+async def test_not_found_status_maps_to_not_found(source_id, sync_engine, tmp_path):
+    result = _run_with_client_error(
+        source_id, sync_engine, tmp_path,
+        PatreonAPIError("HTTP 404", status_code=404),
+    )
+    assert result.error_type == ErrorType.NOT_FOUND
+
+
+@pytest.mark.asyncio
+async def test_transport_error_maps_to_network_error(source_id, sync_engine, tmp_path):
+    result = _run_with_client_error(
+        source_id, sync_engine, tmp_path,
+        PatreonAPIError("connection reset"),  # no status_code → transport
+    )
+    assert result.error_type == ErrorType.NETWORK_ERROR
+
+
+def test_ledger_key_video_has_no_filehash():
+    video = MediaItem(
+        url="https://stream.mux.com/abc.m3u8", filename="clip.mp4",
+        kind="postfile", filehash=None, post_id="p9",
+    )
+    assert _ledger_key(video) == "p9:clip.mp4"
+
+
+# --- verify_patreon_credential (the verify counterpart, plan #697) ---------
+
+
+@pytest.mark.asyncio
+async def test_verify_patreon_credential_unresolvable_is_inconclusive():
+    from backend.app.services.patreon_ingester import verify_patreon_credential
+
+    # No campaign id resolvable from a non-Patreon URL → can't test → None.
+    ok, msg = await verify_patreon_credential("not-a-patreon-url", None, {})
+    assert ok is None
+    assert "campaign id" in msg.lower()
+
+
+@pytest.mark.asyncio
+async def test_verify_patreon_credential_delegates_to_client(monkeypatch):
+    import backend.app.services.patreon_ingester as mod
+
+    async def _fake_resolve(url, cookies_path, overrides):
+        return "123", None
+    monkeypatch.setattr(mod, "resolve_campaign_id_for_source", _fake_resolve)
+    monkeypatch.setattr(
+        mod.PatreonClient, "verify_auth",
+        lambda self, campaign_id: (True, f"ok:{campaign_id}"),
+    )
+    ok, msg = await mod.verify_patreon_credential("https://patreon.com/x", None, {})
+    assert ok is True
+    assert msg == "ok:123"

tests/test_patreon_resolver.py

@@ -2,7 +2,10 @@ from unittest.mock import MagicMock, patch
 
 import pytest
 
-from backend.app.services.patreon_resolver import resolve_campaign_id
+from backend.app.services.patreon_resolver import (
+    resolve_campaign_id,
+    resolve_campaign_id_for_source,
+)
 
 
 @pytest.mark.asyncio
@@ -86,3 +89,48 @@ async def test_loads_cookies_file_when_present(tmp_path):
     assert result == "9"
     _, kwargs = mock_get.call_args
     assert kwargs.get("cookies") is not None
+
+
+# -- resolve_campaign_id_for_source (shared by download + verify) ----------
+
+
+@pytest.mark.asyncio
+async def test_for_source_uses_cached_override_without_lookup():
+    with patch("backend.app.services.patreon_resolver.requests.get") as mock_get:
+        cid, resolved = await resolve_campaign_id_for_source(
+            "https://patreon.com/alice", None, {"patreon_campaign_id": "999"},
+        )
+    assert cid == "999"
+    assert resolved is None  # cached → no newly-resolved id to cache
+    mock_get.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_for_source_extracts_id_url_without_lookup():
+    with patch("backend.app.services.patreon_resolver.requests.get") as mock_get:
+        cid, resolved = await resolve_campaign_id_for_source(
+            "https://www.patreon.com/id:4242", None, {},
+        )
+    assert cid == "4242"
+    assert resolved is None
+    mock_get.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_for_source_resolves_vanity_and_reports_it():
+    fake = MagicMock()
+    fake.status_code = 200
+    fake.json.return_value = {"data": [{"id": "777", "type": "campaign"}]}
+    with patch("backend.app.services.patreon_resolver.requests.get", return_value=fake):
+        cid, resolved = await resolve_campaign_id_for_source(
+            "https://patreon.com/alice", None, {},
+        )
+    assert cid == "777"
+    assert resolved == "777"  # newly resolved → caller caches it
+
+
+@pytest.mark.asyncio
+async def test_for_source_unresolvable_returns_none():
+    cid, resolved = await resolve_campaign_id_for_source("not-a-patreon-url", None, {})
+    assert cid is None
+    assert resolved is None

tests/test_patreon_seen_media.py

@@ -0,0 +1,73 @@
+import pytest
+from sqlalchemy import func, select
+from sqlalchemy.exc import IntegrityError
+
+from backend.app.models import Artist, PatreonSeenMedia, Source
+
+pytestmark = pytest.mark.integration
+
+
+async def _source(db):
+    artist = Artist(name="Seen Ledger Artist", slug="seen-ledger-artist")
+    db.add(artist)
+    await db.flush()
+    source = Source(
+        artist_id=artist.id,
+        platform="patreon",
+        url="https://www.patreon.com/seenledger",
+    )
+    db.add(source)
+    await db.flush()
+    return source.id
+
+
+@pytest.mark.asyncio
+async def test_patreon_seen_media_dedup(db):
+    source_id = await _source(db)
+    db.add(
+        PatreonSeenMedia(
+            source_id=source_id,
+            filehash="0123456789abcdef0123456789abcdef",
+            post_id="123456",
+        )
+    )
+    await db.flush()
+
+    seen_at = await db.scalar(
+        select(PatreonSeenMedia.seen_at).where(
+            PatreonSeenMedia.source_id == source_id
+        )
+    )
+    assert seen_at is not None
+
+    # A second sighting of the same (source_id, filehash) is rejected by the
+    # unique constraint. Wrap in a savepoint so the IntegrityError doesn't
+    # poison the outer transaction.
+    with pytest.raises(IntegrityError):
+        async with db.begin_nested():
+            db.add(
+                PatreonSeenMedia(
+                    source_id=source_id,
+                    filehash="0123456789abcdef0123456789abcdef",
+                    post_id="654321",
+                )
+            )
+            await db.flush()
+
+    # Same filehash but a different post_id sentinel still inserts: the
+    # dedup axis is (source_id, filehash), not post_id.
+    db.add(
+        PatreonSeenMedia(
+            source_id=source_id,
+            filehash="video:123456:7890",
+            post_id="123456",
+        )
+    )
+    await db.flush()
+
+    count = await db.scalar(
+        select(func.count(PatreonSeenMedia.id)).where(
+            PatreonSeenMedia.source_id == source_id
+        )
+    )
+    assert count == 2

tests/test_source_service.py

@@ -225,6 +225,48 @@ async def test_stop_backfill_returns_to_idle(db):
     assert "_backfill_state" not in (co or {})
 
 
+@pytest.mark.asyncio
+async def test_start_recovery_arms_bypass_flag(db):
+    """Plan #697: start_recovery arms the backfill state machine PLUS the
+    _backfill_bypass_seen flag (recovery), surfaced on the record so the UI badge
+    can label it 'Recovering'. Clears any prior checkpoint state."""
+    from backend.app.services.source_service import BACKFILL_MAX_CHUNKS
+
+    artist = await _artist(db, "Alice")
+    svc = SourceService(db)
+    rec = await svc.create(
+        artist_id=artist.id, platform="patreon",
+        url="https://patreon.com/alice",
+    )
+    # Simulate a prior walk's posts counter — start must clear it (plan #704).
+    await db.execute(
+        Source.__table__.update().where(Source.id == rec.id).values(
+            config_overrides={"_backfill_posts": 42},
+        )
+    )
+    await db.commit()
+
+    updated = await svc.start_recovery(rec.id)
+    assert updated.backfill_state == "running"
+    assert updated.backfill_bypass_seen is True
+    assert updated.backfill_posts == 0  # cleared for a fresh walk
+    assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS
+
+    co = (await db.execute(
+        select(Source.config_overrides).where(Source.id == rec.id)
+    )).scalar_one()
+    assert co.get("_backfill_bypass_seen") is True
+    assert "_backfill_posts" not in co
+
+    # Stop clears the bypass flag too (shared lifecycle).
+    stopped = await svc.stop_backfill(rec.id)
+    assert stopped.backfill_bypass_seen is False
+    co2 = (await db.execute(
+        select(Source.config_overrides).where(Source.id == rec.id)
+    )).scalar_one()
+    assert "_backfill_bypass_seen" not in (co2 or {})
+
+
 @pytest.mark.asyncio
 async def test_start_backfill_raises_when_source_missing(db):
     svc = SourceService(db)