Merge pull request '#69 dev→main: release v26.06.04.0' from dev into main

Merge pull request 'dev→main: surface near-duplicate (pHash) control + reorder import tab' (#68 ) from dev into main
Merge pull request 'dev→main: post-card redesign (images→modal, in-place text expand)' (#67 ) from dev into main
2026-06-04 23:16:12 -04:00 · 2026-06-04 21:39:44 -04:00 · 2026-06-04 17:32:50 -04:00 · 2026-06-04 16:59:52 -04:00 · 2026-06-04 13:43:56 -04:00 · 2026-06-04 12:56:58 -04:00
36 changed files with 440 additions and 4129 deletions
@@ -1,53 +0,0 @@
 """patreon_seen_media: per-source ledger of already-ingested Patreon media
 Revision ID: 0037
 Revises: 0036
 Create Date: 2026-06-05
 Native Patreon ingester (build step 2a). Replaces gallery-dl's
 archive.sqlite3 with our own queryable table. The downloader upserts one
 row per (source, media) so routine walks skip media we've already
 processed; a future "recovery" mode bypasses the ledger to re-walk.
 `filehash` is a 32-hex Patreon CDN MD5, OR a video sentinel of the form
 ``video:<post_id>:<media_id>`` — hence String(128). The unique
 constraint on (source_id, filehash) is the dedup upsert key.
 """
 from typing import Sequence, Union
 import sqlalchemy as sa
 from alembic import op
 revision: str = "0037"
 down_revision: Union[str, None] = "0036"
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 def upgrade() -> None:
    op.create_table(
        "patreon_seen_media",
        sa.Column("id", sa.Integer, primary_key=True),
        sa.Column(
            "source_id",
            sa.Integer,
            sa.ForeignKey("source.id", ondelete="CASCADE"),
            nullable=False,
            index=True,
        ),
        sa.Column("filehash", sa.String(128), nullable=False),
        sa.Column("post_id", sa.String(64), nullable=True),
        sa.Column(
            "seen_at",
            sa.DateTime(timezone=True),
            nullable=False,
            server_default=sa.text("NOW()"),
        ),
        sa.UniqueConstraint(
            "source_id", "filehash", name="uq_patreon_seen_media_source_id"
        ),
    )
 def downgrade() -> None:
    op.drop_table("patreon_seen_media")
@@ -121,15 +121,13 @@ async def delete_credential(platform: str):
@credentials_bp.route("/<platform>/verify", methods=["POST"])
 async def verify_credential(platform: str):
-    """Test the stored credential against one of the platform's enabled sources,
+    """Test the stored credential by running gallery-dl --simulate
-    WITHOUT downloading. Routes through the platform's backend
+    against one of the platform's enabled sources. On success stamps
-    (download_backends.verify_credential) — native ingester for Patreon, an
+    last_verified. Returns {valid: bool|null, reason, last_verified?}.
-    authenticated API page; gallery-dl --simulate for the rest. On success
+    valid=null means "couldn't test" (no credential, or no enabled
-    stamps last_verified. Returns {valid: bool|null, reason, last_verified?};
+    source to point at)."""
    valid=null means "couldn't test" (no credential, no enabled source, or an
    inconclusive network/drift result)."""
    from ..models import Artist, Source
-    from ..services.download_backends import verify_source_credential
+    from ..services.gallery_dl import GalleryDLService, SourceConfig
    async with get_session() as session:
        if not await _ext_key_ok(session):
@@ -156,14 +154,14 @@ async def verify_credential(platform: str):
        cookies_path = await svc.get_cookies_path(platform)
        auth_token = await svc.get_token(platform)
-    ok, message = await verify_source_credential(
+    gdl = GalleryDLService(images_root=Path("/images"))
-        platform=platform,
+    ok, message = await gdl.verify(
        url=source.url,
        artist_slug=artist.slug,
-        config_overrides=source.config_overrides or {},
+        platform=platform,
        source_config=SourceConfig.from_dict(source.config_overrides or {}),
        cookies_path=str(cookies_path) if cookies_path else None,
        auth_token=auth_token,
        images_root=Path("/images"),
    )
    last_verified = None
@@ -122,32 +122,26 @@ async def delete_source(source_id: int):
@sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
 async def set_backfill(source_id: int):
-    """Plan #693/#697: start/stop a run-until-done backfill, or start a recovery.
+    """Plan #544: arm a source for backfill mode for the next N download
-    Body: `{"action": "start" | "stop" | "recover"}` (default "start"). 'start'
+    runs. Body: `{"runs": int}` (1..10, default 3). Returns the updated
-    walks the full post history in time-boxed chunks until it reaches the bottom
+    source dict. While backfill_runs_remaining > 0, downloads use
-    (then the source shows 'complete'); 'recover' is the same walk but bypasses
+    gallery-dl's full-walk config (skip: True + 30-min timeout) instead
-    the Patreon seen-ledger to re-fetch dropped-and-deleted near-dups under the
+    of the catch-up default (skip: "exit:20" + 14.5-min timeout)."""
    current pHash threshold; 'stop' cancels either back to tick mode. Returns the
    updated source dict (incl. backfill_state / backfill_chunks /
    backfill_bypass_seen)."""
    payload = await request.get_json(silent=True) or {}
-    action = payload.get("action", "start")
+    runs = payload.get("runs", 3)
-    if action not in ("start", "stop", "recover"):
+    try:
-        return _bad(
+        runs = int(runs)
-            "invalid_action",
+    except (TypeError, ValueError):
-            detail="action must be 'start', 'stop', or 'recover'",
+        return _bad("invalid_runs", detail="runs must be an integer")
        )
    async with get_session() as session:
        try:
-            svc = SourceService(session)
+            record = await SourceService(session).set_backfill_runs(
-            if action == "start":
+                source_id, runs,
-                record = await svc.start_backfill(source_id)
+            )
            elif action == "recover":
                record = await svc.start_recovery(source_id)
            else:
                record = await svc.stop_backfill(source_id)
        except LookupError:
            return _bad("not_found", status=404)
        except ValueError as exc:
            return _bad("invalid_runs", detail=str(exc))
    return jsonify(record.to_dict())
@@ -14,7 +14,6 @@ from .import_settings import ImportSettings
 from .import_task import ImportTask
 from .library_audit_run import LibraryAuditRun
 from .ml_settings import MLSettings
 from .patreon_seen_media import PatreonSeenMedia
 from .post import Post
 from .post_attachment import PostAttachment
 from .series_page import SeriesPage
@@ -34,7 +33,6 @@ __all__ = [
    "BackupRun",
    "Source",
    "Credential",
    "PatreonSeenMedia",
    "Post",
    "PostAttachment",
    "SeriesPage",
@@ -1,38 +0,0 @@
 """PatreonSeenMedia — per-source ledger of Patreon media already
 downloaded+processed.
 Replaces gallery-dl's archive.sqlite3 with our own queryable table so
 routine walks can skip media we've already ingested (and a future
 "recovery" mode can deliberately bypass the ledger to re-walk).
 `filehash` is normally a Patreon CDN MD5 (32 hex chars), but videos —
 which have no stable content hash at discovery time — use a sentinel of
 the form ``video:<post_id>:<media_id>``, hence String(128) rather than 32.
 """
 from datetime import datetime
 from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
 from sqlalchemy.orm import Mapped, mapped_column
 from sqlalchemy.types import DateTime
 from .base import Base
 class PatreonSeenMedia(Base):
    __tablename__ = "patreon_seen_media"
    __table_args__ = (
        # Dedup key the downloader upserts against: one ledger row per
        # (source, media). A second sighting of the same media is a no-op.
        UniqueConstraint("source_id", "filehash", name="uq_patreon_seen_media_source_id"),
    )
    id: Mapped[int] = mapped_column(Integer, primary_key=True)
    source_id: Mapped[int] = mapped_column(
        ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
    )
    filehash: Mapped[str] = mapped_column(String(128), nullable=False)
    post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
    seen_at: Mapped[datetime] = mapped_column(
        DateTime(timezone=True), nullable=False, server_default=func.now()
    )
@@ -1,71 +0,0 @@
 """Platform → download-backend dispatch (one place that knows which platforms
 are served by the native FC ingester vs. the gallery-dl subprocess).
 gallery-dl wasn't built to be driven by an automated scheduler — no native
 checkpoint/resume, no structured logs, per-file HEADs that dominate wall-clock.
 The native ingester (services/patreon_ingester.py, plan #697) replaces it for
 Patreon and is the path we grow as more platforms migrate. To keep that
 migration DRY, every caller that has to behave differently per backend —
 download routing, the credential-verify probe, cursor handling — asks THIS
 module instead of testing ``platform == "patreon"`` inline. When a platform gets
 a native ingester, it moves into ``NATIVE_INGESTER_PLATFORMS`` here and both the
 download path and verify switch over together.
 The backend surfaces share a UNIFORM signature so a caller invokes the same
 function regardless of platform:
  - verify_credential(...)  → (ok: bool|None, message: str)
  - (download stays in download_service for now; uses_native_ingester() is the
    shared predicate it routes on, so the decision lives here too.)
 """
 from __future__ import annotations
 from pathlib import Path
 # Platforms whose download + verify go through the native ingester rather than
 # gallery-dl. gallery-dl still serves every other platform (subscribestar,
 # hentaifoundry, discord, pixiv, deviantart) unchanged.
 NATIVE_INGESTER_PLATFORMS = frozenset({"patreon"})
 def uses_native_ingester(platform: str) -> bool:
    """True when `platform` is served by the native ingester (not gallery-dl).
    The single predicate the download path and verify both route on."""
    return platform in NATIVE_INGESTER_PLATFORMS
 async def verify_source_credential(
    *,
    platform: str,
    url: str,
    artist_slug: str,
    config_overrides: dict | None,
    cookies_path: str | None,
    auth_token: str | None,
    images_root: Path,
 ) -> tuple[bool | None, str]:
    """Uniform credential probe across backends. Returns `(ok, message)`:
    True = authenticated, False = rejected, None = inconclusive (drift /
    network / nothing to test). Callers don't branch on platform — they call
    this and render the result.
    """
    if uses_native_ingester(platform):
        # Native ingester platforms verify via their own lightweight auth probe
        # (resolve campaign id + one authenticated API page). Patreon today.
        from .patreon_ingester import verify_patreon_credential
        return await verify_patreon_credential(url, cookies_path, config_overrides)
    # gallery-dl platforms: --simulate one item; the extractor errors before it
    # can list if auth is bad.
    from .gallery_dl import GalleryDLService, SourceConfig
    gdl = GalleryDLService(images_root=images_root)
    return await gdl.verify(
        url=url,
        artist_slug=artist_slug,
        platform=platform,
        source_config=SourceConfig.from_dict(config_overrides or {}),
        cookies_path=cookies_path,
        auth_token=auth_token,
    )
@@ -13,6 +13,7 @@ from __future__ import annotations
 import asyncio
 import logging
 import re
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
@@ -24,26 +25,44 @@ from sqlalchemy.orm import joinedload
 from ..models import Artist, DownloadEvent, Source
 from .credential_service import CredentialService
 from .download_backends import uses_native_ingester
 from .gallery_dl import (
    BACKFILL_CHUNK_SECONDS,
    BACKFILL_SKIP_VALUE,
    BACKFILL_TIMEOUT_SECONDS,
    TICK_SKIP_VALUE,
    DownloadResult,
    ErrorType,
    GalleryDLService,
    SourceConfig,
    parse_last_cursor,
 )
 from .importer import Importer
-from .patreon_ingester import PatreonIngester
+from .patreon_resolver import resolve_campaign_id
 from .patreon_resolver import resolve_campaign_id_for_source
 from .platforms import auth_type_for
 from .scheduler_service import set_platform_cooldown
 log = logging.getLogger(__name__)
 _PATREON_VANITY_RE = re.compile(
    r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)",
    re.IGNORECASE,
 )
 _CAMPAIGN_ID_FAILURE_PATTERN = "failed to extract campaign id"
 def _extract_patreon_vanity(url: str) -> str | None:
    m = _PATREON_VANITY_RE.match(url)
    return m.group(1) if m else None
 def _looks_like_campaign_id_failure(stdout: str, stderr: str) -> bool:
    return _CAMPAIGN_ID_FAILURE_PATTERN in f"{stdout}\n{stderr}".lower()
 def _effective_url(platform: str, source_url: str, overrides: dict) -> str:
    if platform == "patreon" and overrides.get("patreon_campaign_id"):
        return f"https://www.patreon.com/id:{overrides['patreon_campaign_id']}"
    return source_url
 class DownloadService:
    """Async orchestrator. The Celery task runs `asyncio.run(svc.download_source(N))`.
@@ -58,18 +77,12 @@ class DownloadService:
        gdl: GalleryDLService,
        importer: Importer,
        cred_service: CredentialService,
        sync_session_factory=None,
    ):
        self.async_session = async_session
        self.sync_session = sync_session
        self.gdl = gdl
        self.importer = importer
        self.cred_service = cred_service
        # Sync sessionmaker the native Patreon ingester opens SHORT-LIVED
        # sessions from for its seen-ledger reads/writes (never one held across
        # the multi-minute walk — see PatreonIngester). Only the patreon branch
        # of phase 2 uses it; gallery-dl sources leave it None.
        self.sync_session_factory = sync_session_factory
    async def download_source(self, source_id: int) -> int:
        """Returns DownloadEvent.id. Idempotent: in-flight events are returned as-is."""
@@ -94,155 +107,63 @@ class DownloadService:
        self.sync_session.close()
        source_config = SourceConfig.from_dict(ctx["config_overrides"] or {})
-        # Backfill mode (plan #693): the source's `_backfill_state == "running"`
+        # alembic 0031 / plan #544: derive skip_value + timeout from the
-        # selects a time-boxed deep-walk chunk — skip: True (walk full history)
+        # source's backfill_runs_remaining counter. When > 0, walk the full
-        # + the BACKFILL_CHUNK_SECONDS budget, resuming from the cursor
+        # post history (skip: True + 1170s); when 0, exit gallery-dl after
-        # checkpoint (plan #689). State is the single source of truth; it stays
+        # 20 contiguous archived items (skip: "exit:20" + the default
-        # "running" across ticks until the walk reaches the bottom (phase 3
+        # 870s). Operator sets backfill via POST /api/sources/{id}/backfill.
-        # flips it to "complete"). Otherwise tick mode: exit gallery-dl after
+        backfill_remaining = ctx.get("backfill_runs_remaining", 0) or 0
-        # 20 contiguous archived items (skip: "exit:20" + the default 870s).
+        if backfill_remaining > 0:
        # Operator drives this via POST /api/sources/{id}/backfill {action}.
        overrides = ctx["config_overrides"] or {}
        in_backfill = overrides.get("_backfill_state") == "running"
        # Recovery (plan #697) reuses the entire #693 backfill state machine —
        # cursor checkpoint, time-boxed chunks, complete/stall lifecycle — and
        # differs only in bypassing the tier-1 seen-ledger so dropped-and-deleted
        # near-dups get re-fetched and re-evaluated under the CURRENT pHash
        # threshold (tier-2 disk still spares files we kept). The
        # `_backfill_bypass_seen` flag rides alongside the running backfill state;
        # download mode is "recovery" when both are set.
        bypass_seen = bool(overrides.get("_backfill_bypass_seen"))
        if in_backfill:
            skip_value: bool | str = BACKFILL_SKIP_VALUE
-            source_config.timeout = BACKFILL_CHUNK_SECONDS
+            source_config.timeout = BACKFILL_TIMEOUT_SECONDS
            pending_cursor = overrides.get("_backfill_cursor")
            if uses_native_ingester(ctx["platform"]) and pending_cursor:
                source_config.resume_cursor = pending_cursor
        else:
            skip_value = TICK_SKIP_VALUE
-        resolved_campaign_id: str | None = None
+        effective_url = _effective_url(
-        if uses_native_ingester(ctx["platform"]):
+            ctx["platform"], ctx["url"], ctx["config_overrides"] or {}
-            # Native ingester (plan #697) fully replaces gallery-dl for Patreon
+        )
            # in phase 2 — zero per-file HEADs, native cursor/resume, loud drift
            # detection. Returns a DownloadResult-shaped object so phase 3 is
            # untouched. The gallery-dl Patreon path + its campaign-id retry are
            # removed at the step-5 cutover.
            if in_backfill and bypass_seen:
                mode = "recovery"
            elif in_backfill:
                mode = "backfill"
            else:
                mode = "tick"
            dl_result, resolved_campaign_id = await self._run_patreon_ingester(
                ctx, source_config, mode,
            )
        else:
            dl_result = await self.gdl.download(
                url=ctx["url"],
                artist_slug=ctx["artist_slug"],
                platform=ctx["platform"],
                source_config=source_config,
                cookies_path=ctx["cookies_path"],
                auth_token=ctx["auth_token"],
                skip_value=skip_value,
            )
-        # A backfill chunk that hit its time-box but made forward progress is
+        dl_result = await self.gdl.download(
-        # NORMAL, not a failure — reclassify TIMEOUT → PARTIAL so it reads as
+            url=effective_url,
-        # "ok/progress" (PARTIAL maps to status "ok"), not a red error, and the
+            artist_slug=ctx["artist_slug"],
-        # next chunk just resumes from the new cursor (plan #693). A chunk that
+            platform=ctx["platform"],
-        # timed out with NO progress stays TIMEOUT and feeds phase 3's
+            source_config=source_config,
-        # stall-guard. Leave RATE_LIMITED alone so the platform-cooldown fires.
+            cookies_path=ctx["cookies_path"],
-        if in_backfill and dl_result.error_type == ErrorType.TIMEOUT:
+            auth_token=ctx["auth_token"],
-            new_cursor = parse_last_cursor(dl_result.stdout, dl_result.stderr)
+            skip_value=skip_value,
-            advanced = bool(
+        )
-                (new_cursor and new_cursor != overrides.get("_backfill_cursor"))
+
-                or dl_result.files_downloaded > 0
+        resolved_campaign_id: str | None = None
-            )
+        if (
-            if advanced:
+            ctx["platform"] == "patreon"
-                dl_result.error_type = ErrorType.PARTIAL
+            and not dl_result.success
-                dl_result.error_message = (
+            and "patreon_campaign_id" not in (ctx["config_overrides"] or {})
-                    f"Backfill chunk: {dl_result.files_downloaded} file(s) — continuing"
+            and _looks_like_campaign_id_failure(dl_result.stdout, dl_result.stderr)
        ):
            vanity = _extract_patreon_vanity(ctx["url"])
            if vanity:
                log.info(
                    "Attempting campaign-ID resolution for %s (%s)",
                    ctx["artist_slug"], vanity,
                )
                resolved_campaign_id = await resolve_campaign_id(
                    vanity, ctx["cookies_path"]
                )
                if resolved_campaign_id:
                    dl_result = await self.gdl.download(
                        url=f"https://www.patreon.com/id:{resolved_campaign_id}",
                        artist_slug=ctx["artist_slug"],
                        platform=ctx["platform"],
                        source_config=source_config,
                        cookies_path=ctx["cookies_path"],
                        auth_token=ctx["auth_token"],
                        skip_value=skip_value,
                    )
        return await self._phase3_persist(
            ctx["event_id"], ctx, dl_result, resolved_campaign_id,
        )
    async def _run_patreon_ingester(
        self, ctx: dict, source_config, mode: str,
    ) -> tuple[DownloadResult, str | None]:
        """Phase-2 Patreon branch: resolve the campaign id, then run the native
        ingester in a worker thread (it is sync requests/subprocess).
        Returns (DownloadResult, resolved_campaign_id). `resolved_campaign_id` is
        non-None only when we had to look it up from the vanity URL this run, so
        phase 3 caches it on the source the same way the old gallery-dl retry did.
        A campaign id we cannot resolve is a loud failure (NOT_FOUND) — never a
        silent empty success.
        """
        overrides = ctx["config_overrides"] or {}
        # Shared resolution path (override / id: URL / vanity lookup) — the same
        # helper the credential-verify probe uses. resolved_campaign_id is
        # non-None only when a vanity lookup ran, so phase 3 caches it.
        campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source(
            ctx["url"], ctx["cookies_path"], overrides
        )
        if not campaign_id:
            return (
                DownloadResult(
                    success=False,
                    url=ctx["url"],
                    artist_slug=ctx["artist_slug"],
                    platform="patreon",
                    error_type=ErrorType.NOT_FOUND,
                    error_message=(
                        "Could not resolve Patreon campaign id from the source "
                        "URL (vanity lookup failed — cookies expired or creator "
                        "moved?)"
                    ),
                ),
                None,
            )
        # Honor the operator's existing rate-limit knobs on the native path
        # (plan #703): the global download_rate_limit_seconds (gallery-dl's
        # `rate_limit`, here on self.gdl) paces media downloads. Page fetches use
        # the per-source sleep_request override, else `max(0.5, rate_limit/4)` —
        # the SAME API-pacing default gallery-dl applied as its `sleep-request`
        # (gallery_dl._get_default_config), so the rate-limited /api/posts
        # endpoint stays politely paced without over-throttling.
        rate_limit = self.gdl._rate_limit
        request_sleep = (
            source_config.sleep_request
            if source_config.sleep_request is not None
            else max(0.5, rate_limit / 4)
        )
        ingester = PatreonIngester(
            images_root=self.gdl.images_root,
            cookies_path=ctx["cookies_path"],
            session_factory=self.sync_session_factory,
            validate=self.gdl._validate_files,
            rate_limit=rate_limit,
            request_sleep=request_sleep,
        )
        loop = asyncio.get_running_loop()
        dl_result = await loop.run_in_executor(
            None,
            lambda: ingester.run(
                source_id=ctx["source_id"],
                campaign_id=campaign_id,
                artist_slug=ctx["artist_slug"],
                url=ctx["url"],
                mode=mode,
                resume_cursor=source_config.resume_cursor,
                time_budget_seconds=source_config.timeout,
            ),
        )
        return dl_result, resolved_campaign_id
    async def _phase1_setup(self, source_id: int) -> dict[str, Any]:
        source = (await self.async_session.execute(
            select(Source).options(joinedload(Source.artist)).where(Source.id == source_id)
@@ -474,91 +395,36 @@ class DownloadService:
            source_id=ctx["source_id"], status=status, error_message=ev.error,
            error_type=dl_result.error_type.value if dl_result.error_type else None,
        )
-        await self._apply_backfill_lifecycle(ctx, dl_result)
+        # Plan #544: backfill lifecycle — auto-complete when a clean
        # backfill run drained the queue (gallery-dl exited 0 + zero files
        # downloaded means there was nothing to fetch); otherwise decrement
        # the counter. Next tick falls back to tick mode once it hits 0.
        #
        # Audit 2026-06-02 gating: VALIDATION_FAILED also exits the
        # subprocess with return_code=0 and files_downloaded=0 (every
        # file was quarantined), which used to match the auto-complete
        # predicate exactly — zeroing the operator's armed budget on
        # the FIRST quarantine run instead of decrementing. Require
        # dl_result.success + no error_type so only genuinely-empty
        # successful runs drain the counter.
        backfill_remaining = ctx.get("backfill_runs_remaining", 0) or 0
        if backfill_remaining > 0:
            src = (await self.async_session.execute(
                select(Source).where(Source.id == ctx["source_id"])
            )).scalar_one()
            queue_drained = (
                dl_result.success
                and dl_result.error_type is None
                and dl_result.return_code == 0
                and dl_result.files_downloaded == 0
            )
            if queue_drained:
                src.backfill_runs_remaining = 0
            else:
                src.backfill_runs_remaining = max(0, backfill_remaining - 1)
        await self.async_session.commit()
        return event_id
    async def _apply_backfill_lifecycle(self, ctx: dict, dl_result) -> None:
        """Backfill state machine (plan #693, building on the cursor of #689).
        A backfill runs in time-boxed chunks while
        `config_overrides["_backfill_state"] == "running"`. Each chunk:
          - COMPLETES the walk → clean rc=0 (gallery-dl with skip:True exits 0
            only after exhausting the newest→oldest walk; a chunk cut short by
            its time-box returns success=False / rc<0 via TimeoutExpired). On
            completion: state="complete", clear the cursor, return to tick mode.
          - made PROGRESS (cursor advanced and/or files written) → stay
            "running", checkpoint the new cursor, bump the chunk counter, and
            spend one of the safety-cap chunks (backfill_runs_remaining). If the
            cap is exhausted without finishing → state="stalled".
          - made NO progress → increment the stall counter; two strikes →
            state="stalled", clear the cursor (a wedged walk can't loop).
        State is the single source of truth; the cursor lives only inside a
        running backfill. config_overrides is reassigned (not mutated in place)
        for SQLAlchemy JSON change detection.
        """
        overrides = ctx["config_overrides"] or {}
        if overrides.get("_backfill_state") != "running":
            return  # not backfilling — tick mode, nothing to do
        old_cursor = overrides.get("_backfill_cursor")
        cap_remaining = ctx.get("backfill_runs_remaining", 0) or 0
        src = (await self.async_session.execute(
            select(Source).where(Source.id == ctx["source_id"])
        )).scalar_one()
        new_overrides = dict(src.config_overrides or {})
        chunks = int(new_overrides.get("_backfill_chunks", 0)) + 1
        new_overrides["_backfill_chunks"] = chunks
        completed = (
            dl_result.success
            and dl_result.error_type is None
            and dl_result.return_code == 0
        )
        if completed:
            new_overrides["_backfill_state"] = "complete"
            new_overrides.pop("_backfill_cursor", None)
            new_overrides.pop("_backfill_cursor_stalls", None)
            # plan #697: a recovery walk shares this lifecycle; clear its bypass
            # flag on completion so the next routine tick honors the seen-ledger.
            new_overrides.pop("_backfill_bypass_seen", None)
            src.config_overrides = new_overrides
            src.backfill_runs_remaining = 0
            return
        # Did not finish. Patreon checkpoints + resumes via cursor; other
        # platforms have no resumable cursor (every chunk re-walks from the
        # top), so they advance only by the download archive growing.
        new_cursor = (
            parse_last_cursor(dl_result.stdout, dl_result.stderr)
            if uses_native_ingester(ctx["platform"]) else None
        )
        advanced = bool(
            (new_cursor and new_cursor != old_cursor)
            or dl_result.files_downloaded > 0
        )
        if advanced:
            if new_cursor:
                new_overrides["_backfill_cursor"] = new_cursor
            new_overrides.pop("_backfill_cursor_stalls", None)
            cap_remaining = max(0, cap_remaining - 1)
            src.backfill_runs_remaining = cap_remaining
            if cap_remaining == 0:
                # Safety cap hit before reaching the bottom — pause, don't loop.
                new_overrides["_backfill_state"] = "stalled"
        else:
            stalls = int(new_overrides.get("_backfill_cursor_stalls", 0)) + 1
            if stalls >= 2:
                new_overrides["_backfill_state"] = "stalled"
                new_overrides.pop("_backfill_cursor", None)
                new_overrides.pop("_backfill_cursor_stalls", None)
            else:
                new_overrides["_backfill_cursor_stalls"] = stalls
        src.config_overrides = new_overrides
    async def _update_source_health(
        self, *, source_id: int, status: str, error_message: str | None,
        error_type: str | None = None,
@@ -16,7 +16,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from ..models import Artist, Source
 from ..utils.slug import slugify
-from .source_service import BACKFILL_MAX_CHUNKS
+from .source_service import NEW_SOURCE_BACKFILL_RUNS
 class UnknownPlatformError(Exception):
@@ -205,17 +205,16 @@ class ExtensionService:
            return existing, False
        sp = await self.session.begin_nested()
        try:
-            # New subscription sources arm run-until-done backfill (plan #693)
+            # New subscription sources arm a few backfill runs so the
-            # so the first ticks walk the full history (otherwise gallery-dl's
+            # first ticks walk the full history (otherwise gallery-dl's
-            # exit:20 short-circuits before the archive is built). Mirrors
+            # exit:20 short-circuits before the archive is built).
-            # SourceService.create — without it, Firefox quick-add on a creator
+            # Mirrors SourceService.create — without it, Firefox quick-
-            # with >20 unsynced posts would surface as "check failed" with no
+            # add on a creator with >20 unsynced posts would surface
-            # diagnosis. Audit 2026-06-02.
+            # as "check failed" with no diagnosis. Audit 2026-06-02.
            src = Source(
                artist_id=artist_id, platform=platform,
                url=url, enabled=True,
-                config_overrides={"_backfill_state": "running"},
+                backfill_runs_remaining=NEW_SOURCE_BACKFILL_RUNS,
                backfill_runs_remaining=BACKFILL_MAX_CHUNKS,
            )
            self.session.add(src)
            await self.session.flush()
@@ -13,7 +13,6 @@ from __future__ import annotations
 import asyncio
 import json
 import logging
 import re
 import subprocess
 import sys
 import tempfile
@@ -40,12 +39,6 @@ class ErrorType(StrEnum):
    HTTP_ERROR = "http_error"
    UNSUPPORTED_URL = "unsupported_url"
    VALIDATION_FAILED = "validation_failed"
    # Native Patreon ingester only (plan #697): a response parsed as JSON but
    # didn't match the JSON:API shape the ingester depends on (missing `data`,
    # media lacking `file_name`/`url`, etc.). Distinct from AUTH_ERROR — the fix
    # is updating the ingester's field-set/parser, not rotating credentials. The
    # contract test guards against silently shipping this.
    API_DRIFT = "api_drift"
    # Run made real progress (downloaded ≥1 file) but did not finish in the
    # subprocess budget. Distinct from UNKNOWN_ERROR — the downstream status
    # mapping classifies this as "ok" because the next tick continues.
@@ -62,24 +55,25 @@ class ErrorType(StrEnum):
 # 20 contiguous HEADs is still negligible.
 TICK_SKIP_VALUE = "exit:20"
-# Backfill mode (operator-triggered deep scan): walk the full history,
+# Backfill mode (operator-triggered deep scan): walk the full history.
-# one TIME-BOXED CHUNK per run (plan #693). config_overrides["_backfill_state"]
+# Source.backfill_runs_remaining > 0 selects this mode; the longer
-# == "running" selects this mode; the cursor checkpoint (plan #689) lets each
+# timeout below absorbs creators with thousands of posts.
 # chunk resume where the last stopped, so the walk advances across chunks
 # until gallery-dl exits cleanly (= reached the bottom).
 #
-# The chunk budget is deliberately FAR below download_source's Celery
+# Sits below download_source's Celery soft_time_limit
-# soft_time_limit (DOWNLOAD_SOFT_TIME_LIMIT=1350, tasks/download.py), not
+# (DOWNLOAD_SOFT_TIME_LIMIT=1350, tasks/download.py) with ~180s of
-# "just under" it. Hitting this budget is the NORMAL chunk boundary, not a
+# headroom for phase-3 persist. subprocess.run MUST raise TimeoutExpired
-# failure: subprocess.run raises TimeoutExpired (which captures partial
+# before Celery raises SoftTimeLimitExceeded — that exception path
-# stdout/stderr + the last emitted cursor), and download_service reclassifies
+# captures partial stdout/stderr and finalizes the event; the soft-limit
-# a chunk that made progress as PARTIAL (status "ok"), not an error. The huge
+# path (until the 2026-06-03 fix) did not. Audit history: 1800 guaranteed
-# headroom means a stuck file or a slow chunk can never let Celery's
+# SIGKILL against the old hard limit (Knuxy #38275); 1170 was then sized
-# SoftTimeLimitExceeded preempt TimeoutExpired (the failure mode behind Knuxy
+# "30s shy of the hard limit (1200)" but still EXCEEDED the soft limit
-# #38275 / Anduo #39912/#40411). Earlier we ran one ~1170s run-to-the-wall
+# (900), so SoftTimeLimitExceeded preempted TimeoutExpired and every
-# per arming; that died as a timeout error every time on large catalogs.
+# backfill stranded empty (Anduo #39912). Raising the Celery soft/hard
 # limits to 1350/1500 (tasks/download.py) is what made 1170 safe.
 # backfill_runs_remaining=3 still gives ~58 minutes of cumulative walk
 # across three runs for prolific creators.
 BACKFILL_SKIP_VALUE = True
-BACKFILL_CHUNK_SECONDS = 600
+BACKFILL_TIMEOUT_SECONDS = 1170
 # Sits well below download_source's Celery soft_time_limit
@@ -104,17 +98,6 @@ class SourceConfig:
    Source.backfill_runs_remaining column at the download_service layer
    and is passed to _build_config_for_source as `skip_value`. Same for
    the per-run subprocess timeout.
    `resume_cursor` is RUNTIME state, not persisted operator config: the
    cursor-paged backfill checkpoint (see parse_last_cursor + the
    download_service backfill lifecycle). It is consumed only by the native
    Patreon ingester (the sole cursor-paged platform; gallery-dl's Patreon path
    was removed at the #697 cutover): on a backfill/recovery run the ingester
    resumes its newest→oldest walk from that pagination cursor instead of
    restarting from the top. download_service threads it from
    config_overrides["_backfill_cursor"]; SourceConfig deliberately does NOT read
    it in from_dict (config_overrides also holds operator config, and
    resume_cursor must only apply in backfill/recovery mode).
    """
    content_types: list[str] = field(default_factory=lambda: ["all"])
    sleep: float | None = None
@@ -123,7 +106,6 @@ class SourceConfig:
    filename_pattern: str | None = None
    save_metadata: bool = True
    timeout: int = _DEFAULT_GDL_TIMEOUT_SECONDS
    resume_cursor: str | None = None
    @classmethod
    def from_dict(cls, data: dict) -> SourceConfig:
@@ -172,22 +154,6 @@ def _summarize_validation_failures(failures: list[dict]) -> str:
    return f"{n} files quarantined ({top_count}× {top_reason}, mixed)"
 # The native Patreon ingester emits its pagination cursor as `Cursor: <token>`
 # — one line per fetched page — into the DownloadResult stdout (mirroring the
 # convention gallery-dl used before the #697 cutover, so the backfill lifecycle
 # stayed unchanged). The LAST such line is the furthest-progressed page, i.e.
 # the resume point. We scan both streams (be defensive) and take the final
 # match; download_service checkpoints it as the next chunk's resume_cursor.
 # Survives a time-boxed chunk too: the ingester emits the cursor for a page when
 # it STARTS it, so an interrupted walk still yields its last cursor.
 _CURSOR_RE = re.compile(r"Cursor:\s*(\S+)")
 def parse_last_cursor(stdout: str, stderr: str) -> str | None:
    matches = _CURSOR_RE.findall(f"{stdout or ''}\n{stderr or ''}")
    return matches[-1] if matches else None
 class GalleryDLService:
    """Service for executing gallery-dl downloads."""
@@ -221,10 +187,16 @@ class GalleryDLService:
        "permission denied", "tier required", "pledge required",
    ]
-    # Per-platform defaults for the gallery-dl-backed platforms. Patreon was
+    # Per-platform defaults. Lifted from GS — same six platforms FC supports.
    # removed at the plan-#697 cutover — it now uses the native ingester
    # (services/patreon_ingester.py), not gallery-dl.
    PLATFORM_DEFAULTS = {
        "patreon": {
            "content_types": ["images", "image_large", "attachments", "postfile", "content"],
            "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
            "filename": "{num:>02}_{filename}.{extension}",
            "videos": True,
            "embeds": True,
            "cursor": True,
        },
        "subscribestar": {
            "content_types": ["all"],
            "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
@@ -283,12 +255,7 @@ class GalleryDLService:
                "skip": True,
                "sleep": self._rate_limit,
                "sleep-request": max(0.5, self._rate_limit / 4),
-                # 2 (not 3) retries — a stuck CDN host shouldn't burn a whole
+                "retries": 3,
                # backfill chunk on one file. Anduo #40838 spent ~600s on a
                # single image (4 extractor HEAD retries × 30s + 4 downloader
                # GET retries × 120s); halving retries + the downloader
                # timeout below caps a wedged file at ~1-2 min instead.
                "retries": 2,
                "timeout": 30.0,
                "verify": True,
                "postprocessors": [
@@ -303,17 +270,27 @@ class GalleryDLService:
            "downloader": {
                "part": True,
                "part-directory": str(self._config_dir / "temp"),
-                # See the extractor retries note above (Anduo #40838): 2
+                "retries": 3,
-                # retries + a 60s read-timeout (was 120) so a stalled
+                "timeout": 120.0,
-                # connection fails fast. 60s is a per-read timeout, not a
+                # Forward Patreon as Referer/Origin to yt-dlp when it
-                # transfer cap — large GIFs that keep streaming are unaffected.
+                # fetches video manifests. Operator-flagged 2026-06-01
-                "retries": 2,
+                # (DaferQ patreon): video posts hosted on Mux carry a JWT
-                "timeout": 60.0,
+                # playback restriction that checks Referer/Origin on every
-                # NOTE: the Patreon/Mux yt-dlp Referer/Origin forwarding lived
+                # request — not just the token signature. gallery-dl's
-                # here until the plan-#697 cutover. It was Patreon-specific (and
+                # HEAD probe to stream.mux.com returns 200 (the token is
-                # would have wrongly tagged the other platforms' yt-dlp fetches);
+                # valid), but yt-dlp's actual GET-with-Range to fetch the
-                # Patreon video is now handled by the native ingester's
+                # m3u8 manifest 403s because yt-dlp sends its own default
-                # downloader (patreon_downloader._VIDEO_HEADERS), so it's gone.
+                # Referer, which Mux's policy rejects. Forcing the right
                # headers fixes the headers-only case; Mux IP-range
                # restrictions are unfixable from here.
                "ytdl": {
                    "raw-options": {
                        "http_headers": {
                            "Referer": "https://www.patreon.com/",
                            "Origin": "https://www.patreon.com",
                        },
                    },
                },
            },
            "output": {"progress": True},
        }
@@ -365,7 +342,14 @@ class GalleryDLService:
        platform_section = config["extractor"].setdefault(platform, {})
-        if platform == "hentaifoundry":
+        if platform == "patreon":
            if "all" in source_config.content_types:
                platform_section["files"] = [
                    "images", "image_large", "attachments", "postfile", "content",
                ]
            else:
                platform_section["files"] = source_config.content_types
        elif platform == "hentaifoundry":
            if "pictures" in source_config.content_types or "all" in source_config.content_types:
                platform_section["include"] = "all"
@@ -1,588 +0,0 @@
 """Native Patreon JSON:API client (build step 1 of the native ingester).
 Clean-room reimplementation of the Patreon `/api/posts` read path. This is a
 plain, synchronous client over `requests` — the orchestrator already wraps
 sync importer calls, so nothing here needs to be async (mirrors
 patreon_resolver.py, which wraps its sync lookup in run_in_executor at the
 call site).
 Scope (build step 1): fetch + page + parse only. This module is NOT wired into
 download_service yet — that is a later step. The public surface here exists so
 the later step can drive it:
  - PatreonClient(cookies_path).iter_posts(campaign_id)
        → (post, included_index, page_cursor)
  - extract_media(post, included_index) → list[MediaItem]
  - parse_cursor_from_url(url) → cursor
 Drift detection is loud on purpose: Patreon ships JSON:API and the shapes we
 depend on (top-level `data`, media resources carrying `file_name`/`url`) are
 the contract. If a response comes back as an HTML login page or a media
 resource is missing the fields we resolve against, we raise PatreonDriftError
 rather than silently yielding empty media — so the later import step surfaces
 "Patreon changed something" instead of "creator has no posts".
 FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
 """
 from __future__ import annotations
 import http.cookiejar
 import logging
 import os
 import re
 import time
 from collections.abc import Iterator
 from dataclasses import dataclass
 from html import unescape
 from pathlib import Path
 from urllib.parse import parse_qs, urlsplit
 import requests
 log = logging.getLogger(__name__)
 _POSTS_URL = "https://www.patreon.com/api/posts"
 _USER_AGENT = (
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
 )
 _TIMEOUT_SECONDS = 30.0
 # 429 backoff (plan #703): ride out a transient API rate-limit instead of
 # failing the whole walk (which would stamp RATE_LIMITED → platform-wide
 # cooldown → every Patreon source dark). Honor the server's `Retry-After`;
 # otherwise exponential base·2^(n-1), capped. Only after the retries are
 # exhausted does the 429 propagate as terminal RATE_LIMITED.
 _MAX_429_RETRIES = 3
 _BACKOFF_BASE_SECONDS = 2.0
 _BACKOFF_CAP_SECONDS = 30.0
 def _retry_after_seconds(
    resp: requests.Response,
    attempt: int,
    *,
    base: float = _BACKOFF_BASE_SECONDS,
    cap: float = _BACKOFF_CAP_SECONDS,
 ) -> float:
    """Backoff delay for a 429: the `Retry-After` seconds header if present and
    numeric, else exponential `base·2^(attempt-1)`, both capped. (HTTP-date form
    of Retry-After is rare here and falls through to exponential.)"""
    header = resp.headers.get("Retry-After")
    if header:
        try:
            return min(float(header), cap)
        except (TypeError, ValueError):
            pass
    return min(base * (2 ** max(0, attempt - 1)), cap)
 # JSON:API request contract (observed from real traffic — see module plan).
 _INCLUDE = (
    "campaign,access_rules,attachments,attachments_media,audio,images,media,"
    "native_video_insights,user,user_defined_tags,ti_checks"
 )
 _FIELDS_POST = (
    "content,post_file,image,post_type,published_at,title,url,patreon_url,"
    "current_user_can_view"
 )
 _FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name"
 _FIELDS_CAMPAIGN = "name,url"
 # A CDN download URL embeds a 32-char hex (MD5) path segment; that segment is
 # Patreon's stable per-file identity and is what we dedup + ledger against.
 # Same role gallery-dl's _filehash plays. Match the FIRST 32-hex run anywhere
 # in the URL (path or query); real Patreon CDN URLs carry exactly one.
 _FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})")
 # Inline post `content` is HTML; images are emitted as <img ... src="...">.
 # Pull every src; downstream dedup collapses any that duplicate a gallery item
 # by filehash. Tolerant of attribute ordering and single/double quotes.
 _CONTENT_IMG_RE = re.compile(r"<img\b[^>]*?\bsrc=[\"']([^\"']+)[\"']", re.IGNORECASE)
 # A bounded, sane extension parse (see importer._safe_ext for the FC gotcha:
 # URL-encoded basenames make Path.suffix return base64-ish junk). We only ever
 # use this for a fallback filename, never a network call.
 _MAX_EXT_LEN = 16
 class PatreonAPIError(Exception):
    """Base for native Patreon client failures.
    `status_code` carries the HTTP status when the failure was an HTTP response
    (None for transport-level / parse failures), so the ingester can map it to a
    DownloadResult.error_type (429 → rate_limited, 404 → not_found, …).
    """
    def __init__(self, message: str, *, status_code: int | None = None):
        super().__init__(message)
        self.status_code = status_code
 class PatreonAuthError(PatreonAPIError):
    """Authentication / authorization failure — missing or expired session
    cookies, an insufficient pledge tier, or an HTML login/challenge page served
    where JSON was expected. DISTINCT from drift: the fix is rotating the
    credential, not updating the ingester. Maps to error_type 'auth_error'.
    """
 class PatreonDriftError(PatreonAPIError):
    """A JSON response did not match the JSON:API shape we depend on.
    Raised for: a missing top-level `data` list, `data` not a list, or a media
    resource lacking the `file_name`/`url` fields we resolve against. Fail loud
    so the import step flags API drift (ingester needs update) instead of
    silently importing nothing. An HTML-login / non-JSON body is auth, not
    drift — that raises PatreonAuthError.
    """
@dataclass
 class MediaItem:
    """One resolved downloadable item belonging to a post.
    Fields:
      url       — the CDN/download URL to fetch.
      filename  — media `file_name` when present; otherwise the URL basename
                  (NEVER a network call). Bounded/sane extension.
      kind      — one of: "images", "image_large", "attachments", "postfile",
                  "content". Mirrors gallery-dl's `files` content-type names so
                  the later step can honor the same per-source content_types.
      filehash  — the 32-char hex (MD5) segment from the CDN URL, or None if
                  the URL carries no such segment. Used for in-post dedup and
                  the cross-run seen-ledger.
      post_id   — the owning post's id (so a flattened media list stays
                  traceable to its post).
    """
    url: str
    filename: str
    kind: str
    filehash: str | None
    post_id: str
 def _load_session(cookies_path: str | Path | None) -> requests.Session:
    session = requests.Session()
    session.headers.update(
        {
            "User-Agent": _USER_AGENT,
            "Accept": "application/vnd.api+json",
        }
    )
    if cookies_path and os.path.isfile(str(cookies_path)):
        try:
            jar = http.cookiejar.MozillaCookieJar(str(cookies_path))
            jar.load(ignore_discard=True, ignore_expires=True)
            session.cookies = jar  # type: ignore[assignment]
        except (OSError, http.cookiejar.LoadError) as exc:
            log.warning("Could not load Patreon cookies from %s: %s", cookies_path, exc)
    return session
 def _filehash(url: str) -> str | None:
    if not url:
        return None
    match = _FILEHASH_RE.search(url)
    return match.group(1).lower() if match else None
 def _safe_ext(name: str) -> str:
    suffix = Path(name).suffix.lower()
    if not suffix or len(suffix) > _MAX_EXT_LEN:
        return ""
    if not all(c.isalnum() for c in suffix[1:]):
        return ""
    return suffix
 def _basename_from_url(url: str) -> str:
    """Derive a sane filename from a URL when the media has no file_name.
    Strips query/fragment, takes the path basename, and drops a junk
    extension (the importer._safe_ext gotcha) so we never write base64 noise
    as a name. Falls back to the filehash, then to "file".
    """
    path = urlsplit(url).path
    base = os.path.basename(path)
    if base:
        ext = _safe_ext(base)
        stem = base[: -len(Path(base).suffix)] if Path(base).suffix else base
        # Keep the stem bounded; URL-encoded stems can be enormous.
        stem = stem[:120] or "file"
        return f"{stem}{ext}"
    fh = _filehash(url)
    return fh or "file"
 def parse_cursor_from_url(url: str | None) -> str | None:
    """Extract the `page[cursor]` query param from a links.next URL."""
    if not url:
        return None
    query = urlsplit(url).query
    values = parse_qs(query).get("page[cursor]")
    if values and values[0]:
        return values[0]
    return None
 class PatreonClient:
    """Synchronous Patreon JSON:API read client.
    Construct with a path to a Netscape cookies.txt (the same file
    CredentialService.get_cookies_path materializes). Cookies are loaded into a
    requests.Session; no secure-context APIs are used.
    """
    def __init__(
        self,
        cookies_path: str | Path | None,
        *,
        request_sleep: float = 0.0,
        max_retries: int = _MAX_429_RETRIES,
    ):
        self.cookies_path = str(cookies_path) if cookies_path else None
        self._session = _load_session(cookies_path)
        # Politeness: seconds to sleep before each /api/posts page fetch (paces
        # the rate-limited API endpoint). 0 = no pacing. plan #703.
        self._request_sleep = request_sleep or 0.0
        self._max_retries = max_retries
    # -- request -----------------------------------------------------------
    def _params(self, campaign_id: str, cursor: str | None) -> dict[str, str]:
        params = {
            "include": _INCLUDE,
            "fields[post]": _FIELDS_POST,
            "fields[media]": _FIELDS_MEDIA,
            "fields[campaign]": _FIELDS_CAMPAIGN,
            "filter[campaign_id]": campaign_id,
            "filter[contains_exclusive_posts]": "true",
            "filter[is_draft]": "false",
            "sort": "-published_at",
            "json-api-version": "1.0",
        }
        if cursor:
            params["page[cursor]"] = cursor
        return params
    def _fetch(self, campaign_id: str, cursor: str | None) -> dict:
        if self._request_sleep > 0:
            time.sleep(self._request_sleep)  # pace the API endpoint
        attempt = 0
        while True:
            try:
                resp = self._session.get(
                    _POSTS_URL,
                    params=self._params(campaign_id, cursor),
                    timeout=_TIMEOUT_SECONDS,
                )
            except requests.RequestException as exc:
                raise PatreonAPIError(
                    f"Patreon posts request failed (campaign_id={campaign_id}): {exc}"
                ) from exc
            # Transient rate-limit: back off and retry rather than failing the
            # whole walk. Only a PERSISTENT 429 (retries exhausted) falls
            # through to the terminal RATE_LIMITED raise below.
            if resp.status_code == 429 and attempt < self._max_retries:
                attempt += 1
                delay = _retry_after_seconds(resp, attempt)
                log.warning(
                    "Patreon 429 (campaign_id=%s) — backing off %.1fs (retry %d/%d)",
                    campaign_id, delay, attempt, self._max_retries,
                )
                time.sleep(delay)
                continue
            break
        if resp.status_code in (401, 403):
            # Auth rejected — expired/missing cookies or an insufficient tier.
            # Actionable as "rotate credentials", so it's auth, not drift/http.
            raise PatreonAuthError(
                f"Patreon posts API returned HTTP {resp.status_code} — auth "
                f"rejected (cookies expired or tier insufficient; "
                f"campaign_id={campaign_id})",
                status_code=resp.status_code,
            )
        if resp.status_code != 200:
            raise PatreonAPIError(
                f"Patreon posts API returned HTTP {resp.status_code} "
                f"(campaign_id={campaign_id})",
                status_code=resp.status_code,
            )
        try:
            payload = resp.json()
        except ValueError as exc:
            # A non-JSON body here is almost always the HTML login/challenge
            # page served when cookies are missing/expired — that is an AUTH
            # failure (rotate cookies), not API drift (update the ingester) and
            # not a transient network error.
            raise PatreonAuthError(
                "Patreon posts API returned a non-JSON response (likely an "
                f"HTML login/challenge page — session expired; "
                f"campaign_id={campaign_id}): {exc}"
            ) from exc
        return payload
    # -- parsing -----------------------------------------------------------
    @staticmethod
    def _transform(response: dict) -> dict:
        """Flatten the JSON:API `included` array for relationship resolution.
        Returns a dict keyed by `(type, id)` → that resource's `attributes`
        (so a post's relationships can be resolved in O(1)). Missing/oddly
        shaped `included` entries are skipped rather than fatal — drift
        detection for the top-level shape lives in _validate_response.
        """
        index: dict[tuple[str, str], dict] = {}
        for inc in response.get("included") or []:
            if not isinstance(inc, dict):
                continue
            rtype = inc.get("type")
            rid = inc.get("id")
            if rtype is None or rid is None:
                continue
            index[(str(rtype), str(rid))] = inc.get("attributes") or {}
        return index
    @staticmethod
    def _validate_response(response: dict) -> None:
        if not isinstance(response, dict):
            raise PatreonDriftError("Patreon response was not a JSON object")
        if "data" not in response:
            raise PatreonDriftError("Patreon response missing top-level 'data' key")
        data = response.get("data")
        if not isinstance(data, list):
            raise PatreonDriftError("Patreon response 'data' was not a list")
    def _related_ids(self, post: dict, rel_name: str) -> list[str]:
        rels = post.get("relationships") or {}
        rel = rels.get(rel_name) or {}
        data = rel.get("data")
        if isinstance(data, dict):  # to-one relationship
            data = [data]
        if not isinstance(data, list):
            return []
        ids: list[str] = []
        for ref in data:
            if isinstance(ref, dict) and ref.get("id") is not None:
                ids.append(str(ref["id"]))
        return ids
    @staticmethod
    def _media_url(attrs: dict) -> str | None:
        """Pick the best fetchable URL for a media resource.
        Prefer the full-size `download_url`; fall back to the largest
        `image_urls` size. gallery-dl prefers download_url too, only dipping
        into image_urls when a smaller configured size is requested — FC
        always wants the original, so download_url first.
        """
        download_url = attrs.get("download_url")
        if isinstance(download_url, str) and download_url:
            return download_url
        image_urls = attrs.get("image_urls")
        if isinstance(image_urls, dict):
            for key in ("original", "full", "large", "default"):
                candidate = image_urls.get(key)
                if isinstance(candidate, str) and candidate:
                    return candidate
            # Otherwise take any non-empty string value.
            for candidate in image_urls.values():
                if isinstance(candidate, str) and candidate:
                    return candidate
        return None
    def _media_item(
        self, attrs: dict, kind: str, post_id: str, *, require_file_name: bool
    ) -> MediaItem:
        url = self._media_url(attrs)
        if not url:
            raise PatreonDriftError(
                f"Patreon media (post {post_id}, kind={kind}) had no resolvable URL "
                f"(no download_url / image_urls)"
            )
        file_name = attrs.get("file_name")
        if require_file_name and not (isinstance(file_name, str) and file_name):
            raise PatreonDriftError(
                f"Patreon media resource (post {post_id}, kind={kind}) missing "
                f"'file_name'"
            )
        filename = file_name if isinstance(file_name, str) and file_name else _basename_from_url(url)
        return MediaItem(
            url=url,
            filename=filename,
            kind=kind,
            filehash=_filehash(url),
            post_id=post_id,
        )
    def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]:
        """Resolve all downloadable media for one post.
        Walks the kinds in the same order gallery-dl does — images,
        image_large (post cover), attachments, postfile, content (inline
        <img>) — and dedups within the post by filehash (first wins). The
        image_large cover commonly duplicates a gallery image; deduping by
        filehash collapses them to the gallery item (encountered first).
        """
        post_id = str(post.get("id") or "")
        attrs = post.get("attributes") or {}
        items: list[MediaItem] = []
        def _resolve_rel(rel_name: str, kind: str) -> None:
            for mid in self._related_ids(post, rel_name):
                media_attrs = included_index.get(("media", mid))
                if media_attrs is None:
                    # Referenced but not in `included`: a media id with no
                    # resource is drift (we asked for include=media).
                    raise PatreonDriftError(
                        f"Patreon post {post_id} references media {mid} "
                        f"({rel_name}) not present in 'included'"
                    )
                items.append(
                    self._media_item(
                        media_attrs, kind, post_id, require_file_name=True
                    )
                )
        # 1. gallery images
        _resolve_rel("images", "images")
        # 2. image_large — the post-level cover (`image.large_url`). Not a
        #    media relationship; it lives on the post attributes.
        image = attrs.get("image")
        if isinstance(image, dict):
            large_url = image.get("large_url") or image.get("url")
            if isinstance(large_url, str) and large_url:
                items.append(
                    MediaItem(
                        url=large_url,
                        filename=_basename_from_url(large_url),
                        kind="image_large",
                        filehash=_filehash(large_url),
                        post_id=post_id,
                    )
                )
        # 3. attachments
        _resolve_rel("attachments_media", "attachments")
        # 4. postfile — the post's primary attached file (`post_file`).
        post_file = attrs.get("post_file")
        if isinstance(post_file, dict):
            pf_url = post_file.get("url") or post_file.get("download_url")
            if isinstance(pf_url, str) and pf_url:
                pf_name = post_file.get("name")
                filename = (
                    pf_name
                    if isinstance(pf_name, str) and pf_name
                    else _basename_from_url(pf_url)
                )
                items.append(
                    MediaItem(
                        url=pf_url,
                        filename=filename,
                        kind="postfile",
                        filehash=_filehash(pf_url),
                        post_id=post_id,
                    )
                )
        # 5. content — inline <img> in the post HTML body.
        content = attrs.get("content")
        if isinstance(content, str) and content:
            for raw_src in _CONTENT_IMG_RE.findall(content):
                src = unescape(raw_src)
                if not src:
                    continue
                items.append(
                    MediaItem(
                        url=src,
                        filename=_basename_from_url(src),
                        kind="content",
                        filehash=_filehash(src),
                        post_id=post_id,
                    )
                )
        return _dedup_by_filehash(items)
    # -- iteration ---------------------------------------------------------
    def iter_posts(
        self, campaign_id: str, cursor: str | None = None
    ) -> Iterator[tuple[dict, dict, str | None]]:
        """Yield (post, included_index, page_cursor) for every post in the feed.
        Pages newest→oldest via `links.next`, validating each response for
        drift before yielding. The triple gives a caller everything it needs to
        resolve and checkpoint without re-fetching:
          - post           — the raw post resource.
          - included_index — the page's flattened `included` (the same object
                             for every post on a page), to pass straight to
                             extract_media(post, included_index).
          - page_cursor    — the cursor that FETCHED this post's page (None for
                             the first page). The caller checkpoints THIS value,
                             matching the existing backfill cursor logic where
                             the saved cursor re-fetches the page being
                             processed (so a chunk cut mid-page resumes the page,
                             not the one after it).
        """
        current_cursor = cursor
        while True:
            response = self._fetch(campaign_id, current_cursor)
            self._validate_response(response)
            page_cursor = current_cursor
            included_index = self._transform(response)
            for post in response.get("data") or []:
                if isinstance(post, dict):
                    yield post, included_index, page_cursor
            next_url = (response.get("links") or {}).get("next")
            next_cursor = parse_cursor_from_url(next_url)
            if not next_cursor:
                return
            current_cursor = next_cursor
    # -- verify ------------------------------------------------------------
    def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
        """Cheap auth probe: fetch the first `/api/posts` page and report whether
        the credential authenticated, WITHOUT downloading anything.
        Returns `(ok, message)` matching the credential-verify contract:
          - True  — authenticated (the feed returned a valid JSON:API page).
          - False — the credential was rejected (PatreonAuthError: 401/403, or an
                    HTML login page → cookies expired / tier insufficient).
          - None  — inconclusive: API drift (our parser is stale, not a cred
                    problem) or a transient network/HTTP error.
        """
        try:
            response = self._fetch(campaign_id, None)
            self._validate_response(response)
        except PatreonAuthError as exc:
            return False, f"Patreon rejected the credential — {exc}"
        except PatreonDriftError as exc:
            return None, f"Couldn't verify — Patreon's API shape changed: {exc}"
        except PatreonAPIError as exc:
            return None, f"Couldn't verify (network/HTTP issue): {exc}"
        return True, "Credentials valid — the Patreon feed authenticated."
 def _dedup_by_filehash(items: list[MediaItem]) -> list[MediaItem]:
    """Drop later items sharing a filehash with an earlier one (first wins).
    Items with no filehash (None) are never deduped against each other — we
    can't prove they're the same file, so keep them all.
    """
    seen: set[str] = set()
    out: list[MediaItem] = []
    for item in items:
        if item.filehash is not None:
            if item.filehash in seen:
                continue
            seen.add(item.filehash)
        out.append(item)
    return out
@@ -1,416 +0,0 @@
 """Native Patreon media downloader (build step 2b of the native ingester).
 Given a Patreon post and its already-resolved `MediaItem`s (from
 patreon_client.extract_media), download the media to the EXACT on-disk layout
 gallery-dl produces, write a sidecar JSON the existing importer consumes, and
 report per-media outcomes.
 This module is PURE: no DB. The cross-run seen-ledger and the iter_posts
 orchestration are a LATER step. The tier-1 (seen) skip is an INJECTED predicate
 (`is_seen`), so this module needs no DB and is unit-testable without network.
 On-disk layout (matches gallery-dl):
  <images_root>/<artist_slug>/patreon/<DIR>/<NN>_<filename>
  where <DIR> = "<YYYY-MM-DD>_<post_id>_<title40>" (date prefix omitted when the
  post's published_at is unparseable), <NN> is the 1-based index of the item in
  the post zero-padded to 2, and <filename> is MediaItem.filename. The sidecar
  is written next to the media as <NN>_<stem>.json (media_path.with_suffix).
 Video: a MediaItem whose URL is a Mux/HLS stream (host stream.mux.com or path
 endswith .m3u8) is fetched with yt-dlp (subprocess), passing the same
 Referer/Origin headers gallery-dl forwards for Mux playback (see gallery_dl.py
 _get_default_config). yt-dlp may remux to a container of its own choosing, so we
 accept the actual output extension and record the real path.
 FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
 """
 from __future__ import annotations
 import contextlib
 import json
 import logging
 import os
 import shutil
 import subprocess
 import time
 from collections.abc import Callable
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
 from urllib.parse import urlsplit
 import requests
 from .file_validator import is_validatable, validate_file
 from .patreon_client import _load_session, _retry_after_seconds
 log = logging.getLogger(__name__)
 _TITLE_MAX = 40
 _TIMEOUT_SECONDS = 120.0
 _CHUNK = 1 << 16
 # A CDN media GET rarely 429s, but if it does, a couple of backoff retries keep
 # one throttled file from becoming a per-item error (plan #703).
 _MAX_MEDIA_429_RETRIES = 2
 # Referer/Origin yt-dlp must send for Mux-hosted Patreon video. Mux's JWT
 # playback policy checks Referer/Origin on every request, so yt-dlp must send
 # Patreon's, not its own default. (gallery-dl forwarded the same headers before
 # the #697 cutover removed its Patreon path; this is now the only place they
 # live.)
 _VIDEO_HEADERS = {
    "Referer": "https://www.patreon.com/",
    "Origin": "https://www.patreon.com",
 }
 # Characters Windows/gallery-dl path-restrict forbids, plus path separators.
 _FORBIDDEN = set('<>:"/\\|?*')
 def _sanitize(name: str) -> str:
    """Make `name` safe for a single filesystem path segment.
    Replaces path separators, the Windows-forbidden set <>:"/\\|?* and control
    characters with `_`, then strips trailing dots/spaces (gallery-dl
    path-restrict behavior). Never returns empty (falls back to "_").
    """
    out = []
    for ch in name:
        if ch in _FORBIDDEN or ord(ch) < 32:
            out.append("_")
        else:
            out.append(ch)
    cleaned = "".join(out).rstrip(". ")
    return cleaned or "_"
 def _is_video_url(url: str) -> bool:
    parts = urlsplit(url)
    if parts.hostname and parts.hostname.lower() == "stream.mux.com":
        return True
    return parts.path.lower().endswith(".m3u8")
 def _post_dir_name(post: dict) -> str:
    """Build the post directory name matching gallery-dl's layout."""
    post_id = str(post.get("id") or "")
    attrs = post.get("attributes") or {}
    title = attrs.get("title")
    title = title if isinstance(title, str) else ""
    title40 = title[:_TITLE_MAX]
    published = attrs.get("published_at")
    date_prefix = None
    if isinstance(published, str) and published:
        s = published.strip()
        if s.endswith("Z"):
            s = s[:-1] + "+00:00"
        try:
            dt = datetime.fromisoformat(s)
        except ValueError:
            dt = None
        if dt is not None:
            date_prefix = f"{dt:%Y-%m-%d}"
    if date_prefix:
        raw = f"{date_prefix}_{post_id}_{title40}"
    else:
        raw = f"{post_id}_{title40}"
    return _sanitize(raw)
@dataclass
 class MediaOutcome:
    """Per-media result of a download_post pass.
    status is one of: "downloaded", "skipped_seen", "skipped_disk", "error".
    `path` is the final on-disk path for "downloaded" (the actual yt-dlp output
    for video), or the path that already existed for "skipped_disk"; None for
    "skipped_seen" and (usually) "error". `error` carries the failure reason for
    "error", else None.
    """
    media: object  # MediaItem (avoid importing the name for a bare annotation)
    status: str
    path: Path | None
    error: str | None
 class PatreonDownloader:
    """Download resolved Patreon media to gallery-dl's on-disk layout.
    PURE: no DB. The HTTP session and the yt-dlp invocation are injectable seams
    so tests run without network or a real subprocess:
      - pass `session=` to stub `session.get`, or monkeypatch `_fetch_to_file`.
      - monkeypatch `_run_ytdlp` to avoid spawning yt-dlp.
    """
    def __init__(
        self,
        images_root: Path,
        cookies_path: str | None = None,
        *,
        validate: bool = True,
        rate_limit: float = 0.0,
        session: requests.Session | None = None,
    ):
        self.images_root = Path(images_root)
        self.cookies_path = str(cookies_path) if cookies_path else None
        self._validate = validate
        # Politeness: seconds to sleep before each actual media download (paces
        # the CDN; honors ImportSettings.download_rate_limit_seconds, the same
        # value gallery-dl used as its between-downloads `sleep`). 0 = no pacing.
        # Applied only to real downloads, not to seen/disk skips. plan #703.
        self._rate_limit = rate_limit or 0.0
        # Build a cookie-loaded session the same way patreon_client does, so the
        # CDN GETs carry the creator's auth.
        self.session = session if session is not None else _load_session(cookies_path)
    # -- public ------------------------------------------------------------
    def download_post(
        self,
        post: dict,
        media_items: list,
        artist_slug: str,
        *,
        is_seen: Callable[[object], bool] = lambda m: False,
    ) -> list[MediaOutcome]:
        """Download every media item of one post; return per-item outcomes.
        Builds the post directory, iterates media (1-based NN), applies the
        two-tier skip (injected is_seen, then disk), downloads (plain GET or
        yt-dlp for video), writes the sidecar for each freshly-downloaded item,
        validates, and returns outcomes. Resilient: one media's failure yields
        an "error" outcome for that item; the rest proceed.
        """
        post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post)
        outcomes: list[MediaOutcome] = []
        for i, media in enumerate(media_items, start=1):
            try:
                outcomes.append(
                    self._download_one(post, media, post_dir, artist_slug, i, is_seen)
                )
            except Exception as exc:  # resilient: isolate one item's failure
                log.warning(
                    "Patreon media failed (post %s, item %d): %s",
                    post.get("id"), i, exc,
                )
                outcomes.append(
                    MediaOutcome(media=media, status="error", path=None, error=str(exc))
                )
        return outcomes
    # -- per-item ----------------------------------------------------------
    def _download_one(
        self,
        post: dict,
        media,
        post_dir: Path,
        artist_slug: str,
        index: int,
        is_seen: Callable[[object], bool],
    ) -> MediaOutcome:
        # tier-1: seen ledger (injected; no DB here).
        if is_seen(media):
            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
        nn = f"{index:02d}"
        final_name = _sanitize(f"{nn}_{media.filename}")
        media_path = post_dir / final_name
        # tier-2: already on disk.
        if media_path.exists():
            return MediaOutcome(
                media=media, status="skipped_disk", path=media_path, error=None
            )
        # Video may land at a different extension; honor a pre-existing remux.
        if _is_video_url(media.url):
            existing = self._existing_video_output(media_path)
            if existing is not None:
                return MediaOutcome(
                    media=media, status="skipped_disk", path=existing, error=None
                )
        post_dir.mkdir(parents=True, exist_ok=True)
        # Pace real downloads only (the skips above already returned). plan #703.
        if self._rate_limit > 0:
            time.sleep(self._rate_limit)
        if _is_video_url(media.url):
            out_path = self._run_ytdlp(media.url, media_path, _VIDEO_HEADERS)
            if out_path is None or not Path(out_path).exists():
                return MediaOutcome(
                    media=media,
                    status="error",
                    path=None,
                    error="yt-dlp produced no output",
                )
            out_path = Path(out_path)
        else:
            out_path = self._fetch_get(media.url, media_path)
            invalid = self._validate_path(out_path, artist_slug)
            if invalid is not None:
                return MediaOutcome(
                    media=media, status="error", path=None, error=invalid
                )
        self._write_sidecar(post, out_path)
        return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
    # -- download seams ----------------------------------------------------
    def _fetch_get(self, url: str, dest: Path) -> Path:
        """Stream `url` to a .part file then atomic-rename to `dest`.
        Thin wrapper over `_fetch_to_file` so tests can stub either the whole
        GET path (`_fetch_to_file`) or just `session.get`.
        """
        part = dest.with_name(dest.name + ".part")
        try:
            self._fetch_to_file(url, part)
        except Exception:
            with contextlib.suppress(OSError):
                part.unlink()
            raise
        os.replace(part, dest)
        return dest
    def _fetch_to_file(self, url: str, dest: Path) -> None:
        """Stream a non-video URL to `dest` via the (stubbable) session.
        Retries a transient 429 with backoff (plan #703); any other non-2xx
        falls through to raise_for_status → HTTPError, which download_post turns
        into a resilient per-item error outcome.
        """
        attempt = 0
        while True:
            resp = self.session.get(url, stream=True, timeout=_TIMEOUT_SECONDS)
            if resp.status_code == 429 and attempt < _MAX_MEDIA_429_RETRIES:
                attempt += 1
                delay = _retry_after_seconds(resp, attempt)
                log.warning(
                    "Patreon media 429 (%s) — backing off %.1fs (retry %d/%d)",
                    url, delay, attempt, _MAX_MEDIA_429_RETRIES,
                )
                time.sleep(delay)
                continue
            break
        resp.raise_for_status()
        with open(dest, "wb") as fh:
            for chunk in resp.iter_content(chunk_size=_CHUNK):
                if chunk:
                    fh.write(chunk)
    def _run_ytdlp(self, url: str, dest: Path, headers: dict) -> Path | None:
        """Invoke yt-dlp to fetch a Mux/HLS stream to (around) `dest`.
        Returns the actual output path (yt-dlp may remux to a different
        container, so we resolve the real file afterward). Overridden/
        monkeypatched in tests to avoid spawning a real subprocess.
        The output template uses `dest` without its extension; yt-dlp appends
        the chosen container extension. We pass Referer/Origin (Mux JWT policy)
        and the cookies file.
        """
        dest = Path(dest)
        out_template = str(dest.with_suffix("")) + ".%(ext)s"
        cmd = ["yt-dlp", "--no-progress", "-o", out_template]
        for key, value in headers.items():
            cmd += ["--add-header", f"{key}:{value}"]
        if self.cookies_path and os.path.isfile(self.cookies_path):
            cmd += ["--cookies", self.cookies_path]
        cmd.append(url)
        try:
            subprocess.run(
                cmd,
                check=True,
                capture_output=True,
                text=True,
                timeout=_TIMEOUT_SECONDS,
            )
        except (OSError, subprocess.SubprocessError) as exc:
            log.warning("yt-dlp failed for %s: %s", url, exc)
            return None
        return self._existing_video_output(dest)
    def _existing_video_output(self, dest: Path) -> Path | None:
        """Find a yt-dlp output for `dest` regardless of chosen extension.
        Returns `dest` itself if present, else any sibling sharing the same
        stem (the remuxed container). None if nothing matched.
        """
        dest = Path(dest)
        if dest.exists():
            return dest
        stem = dest.with_suffix("").name
        parent = dest.parent
        if not parent.is_dir():
            return None
        for cand in sorted(parent.iterdir()):
            if cand.is_file() and cand.stem == stem and cand.name != dest.name:
                return cand
        return None
    # -- validation --------------------------------------------------------
    def _validate_path(self, path: Path, artist_slug: str) -> str | None:
        """Validate a freshly-written file; quarantine + return reason if bad.
        Mirrors gallery_dl._validate_and_quarantine: fail-open for unknown
        formats, move the corrupt file to _quarantine/<slug>/patreon, and return
        the failure reason string (None when ok / not validatable / disabled).
        """
        if not self._validate or not is_validatable(path):
            return None
        try:
            result = validate_file(path)
        except Exception as exc:
            log.warning("Validator raised on %s: %s", path, exc)
            return None
        if result.ok:
            return None
        quarantine_root = self.images_root / "_quarantine" / artist_slug / "patreon"
        try:
            quarantine_root.mkdir(parents=True, exist_ok=True)
            dest = quarantine_root / path.name
            counter = 1
            while dest.exists():
                dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}"
                counter += 1
            shutil.move(str(path), str(dest))
        except OSError as exc:
            log.error("Failed to quarantine %s: %s. File left in place.", path, exc)
        return result.reason or "validation failed"
    # -- sidecar -----------------------------------------------------------
    def _write_sidecar(self, post: dict, media_path: Path) -> Path:
        """Write the importer-consumed sidecar next to `media_path`.
        Patreon uses base-default sidecar keys (parse_sidecar maps
        category->platform, id->external_post_id, title->post_title,
        content->description, published_at->post_date, url->post_url). Patreon
        registers no derive_post_url, so `url` is trusted as the permalink — we
        pass the post's attributes.url.
        """
        attrs = post.get("attributes") or {}
        title = attrs.get("title")
        content = attrs.get("content")
        published = attrs.get("published_at")
        url = attrs.get("url")
        data = {
            "category": "patreon",
            "id": str(post.get("id") or ""),
            "title": title if isinstance(title, str) else "",
            "content": content if isinstance(content, str) else "",
            "published_at": published if isinstance(published, str) else None,
            "url": url if isinstance(url, str) else None,
        }
        sidecar_path = media_path.with_suffix(".json")
        sidecar_path.write_text(json.dumps(data, indent=2))
        return sidecar_path
@@ -1,414 +0,0 @@
 """Native Patreon ingester — phase-2 orchestrator (build step 3).
 Ties build steps 1 (`patreon_client`) and 2 (`patreon_downloader` +
 `patreon_seen_media`) together into a single sync walk that REPLACES the
 gallery-dl subprocess for Patreon. `download_service.download_source` calls
 `PatreonIngester.run(...)` from phase 2 (in a thread, via run_in_executor, since
 everything here is sync `requests`/`subprocess`) and gets back a
 `DownloadResult` — the exact shape gallery-dl returns — so phase 1 (DB setup)
 and phase 3 (import → pHash dedup → thumbnails → ML) are untouched and cannot
 tell the difference.
 Three modes (selected by `download_service` from `config_overrides` state):
  - tick      — newest→oldest, skip seen (tier-1 ledger + tier-2 disk), early-out
                after N contiguous already-have-it items (the cheap native
                equivalent of gallery-dl's `exit:20`, now free of per-file HEADs).
  - backfill  — full-history walk in a time-boxed chunk, resuming from the
                pagination cursor checkpoint; reaches the bottom → "complete".
  - recovery  — like backfill but BYPASSES the tier-1 seen-ledger, so
                deliberately-dropped-and-deleted near-dups get re-fetched and
                re-evaluated under the current pHash threshold (tier-2 disk skip
                still spares files we kept). Triggered by the same #693 backfill
                state machine plus the `_backfill_bypass_seen` flag, so the whole
                cursor/chunk/complete/stall lifecycle is reused verbatim.
 Cursor contract: the ingester emits gallery-dl-style ``Cursor: <token>`` lines
 into the returned `stdout`, one per page it walks. That is exactly what
 `download_service`'s existing backfill lifecycle reads via
 `parse_last_cursor(stdout, stderr)` — so checkpointing, the TIMEOUT→PARTIAL
 reclassification, and completion detection all keep working unchanged.
 The seen-ledger lives in Postgres (`patreon_seen_media`). The ingester opens a
 SHORT-LIVED sync session per page batch (via an injected sessionmaker) — never
 held across a network fetch — so the multi-minute walk can't strand a checked-out
 connection for the server to reap ([[db-connection-held-across-subprocess]]).
 FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
 """
 from __future__ import annotations
 import asyncio
 import logging
 import time
 from collections.abc import Callable
 from pathlib import Path
 from sqlalchemy import select
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from ..models import PatreonSeenMedia
 from .gallery_dl import DownloadResult, ErrorType
 from .patreon_client import (
    MediaItem,
    PatreonAPIError,
    PatreonAuthError,
    PatreonClient,
    PatreonDriftError,
 )
 from .patreon_downloader import PatreonDownloader
 from .patreon_resolver import resolve_campaign_id_for_source
 log = logging.getLogger(__name__)
 # gallery-dl's `exit:20` default ported over: stop a tick after this many
 # CONTIGUOUS already-have-it media (seen-ledger or on-disk). Native walks have
 # zero per-file HEADs, so the only cost of a higher number is a few extra cheap
 # ledger lookups — 20 is operator-set headroom against paywalled/undownloadable
 # items interleaving with archived ones.
 _TICK_SEEN_THRESHOLD = 20
 # Ledger keys are stored in patreon_seen_media.filehash VARCHAR(128); bound any
 # synthesized key so a pathologically long file_name can't overflow the column.
 _LEDGER_KEY_MAX = 128
 def _ledger_key(media: MediaItem) -> str:
    """Stable per-media identity for the cross-run seen-ledger.
    A Patreon CDN URL carries a 32-char MD5 (`media.filehash`) — that is the
    natural key. Some media have none: Mux/HLS video (`stream.mux.com`, no
    content hash at discovery) and the odd inline-content `<img>` pointing at a
    hashless URL. The plan calls the video case the ``video:<post_id>:<media_id>``
    sentinel; `MediaItem` carries no media_id, so the post-scoped filename is the
    stable proxy. Bounded to the column width.
    """
    if media.filehash:
        return media.filehash
    return f"{media.post_id}:{media.filename}"[:_LEDGER_KEY_MAX]
 class PatreonIngester:
    """Walk a Patreon campaign's posts, download unseen media, return a
    `DownloadResult`.
    Construct with the per-source `cookies_path` and a sync sessionmaker for the
    seen-ledger. `client` / `downloader` are injectable seams so unit tests run
    without network, subprocess, or a real CDN.
    """
    def __init__(
        self,
        images_root: Path,
        cookies_path: str | None,
        session_factory: Callable[[], object],
        *,
        validate: bool = True,
        rate_limit: float = 0.0,
        request_sleep: float = 0.0,
        client: PatreonClient | None = None,
        downloader: PatreonDownloader | None = None,
    ):
        self.images_root = Path(images_root)
        self.cookies_path = str(cookies_path) if cookies_path else None
        self.session_factory = session_factory
        # Pacing (plan #703): request_sleep paces the API page fetches,
        # rate_limit paces the media downloads. Injected client/downloader (in
        # tests) already carry their own pacing, so these only apply to the
        # default-constructed ones.
        self.client = (
            client
            if client is not None
            else PatreonClient(cookies_path, request_sleep=request_sleep)
        )
        self.downloader = (
            downloader
            if downloader is not None
            else PatreonDownloader(
                self.images_root, cookies_path, validate=validate, rate_limit=rate_limit
            )
        )
    # -- public ------------------------------------------------------------
    def run(
        self,
        *,
        source_id: int,
        campaign_id: str,
        artist_slug: str,
        url: str,
        mode: str,
        resume_cursor: str | None = None,
        time_budget_seconds: float = 870.0,
        seen_threshold: int = _TICK_SEEN_THRESHOLD,
    ) -> DownloadResult:
        """Walk + download for one source, returning a gallery-dl-shaped result.
        `mode` is "tick" | "backfill" | "recovery". Recovery bypasses the tier-1
        seen-ledger (tier-2 disk still skips kept files). The walk stops on:
          - budget exhaustion (time_budget_seconds)         → TIMEOUT / PARTIAL
          - tick early-out (seen_threshold contiguous seen)  → success
          - reaching the bottom of the feed                  → success (rc 0)
        A client-level failure (drift / auth / network) fails the whole run loud.
        """
        bypass_seen = mode == "recovery"
        start = time.monotonic()
        log_lines: list[str] = []
        written: list[str] = []
        downloaded = 0
        errors = 0
        consecutive_seen = 0
        emitted_cursor: str | None = None
        reached_bottom = False
        budget_hit = False
        early_out = False
        def _result(
            *, success: bool, return_code: int,
            error_type: ErrorType | None, error_message: str | None,
        ) -> DownloadResult:
            return DownloadResult(
                success=success,
                url=url,
                artist_slug=artist_slug,
                platform="patreon",
                files_downloaded=downloaded,
                files_quarantined=0,
                quarantined_paths=[],
                written_paths=written,
                stdout="\n".join(log_lines),
                stderr="",
                return_code=return_code,
                error_type=error_type,
                error_message=error_message,
                duration_seconds=time.monotonic() - start,
            )
        try:
            for post, included, page_cursor in self.client.iter_posts(
                campaign_id, cursor=resume_cursor
            ):
                # Checkpoint: emit the cursor that FETCHED this page once, the
                # moment we START it — so a chunk cut mid-page resumes the page,
                # not the one after it (matches the gallery-dl cursor semantics
                # download_service's lifecycle already depends on).
                if page_cursor and page_cursor != emitted_cursor:
                    log_lines.append(f"Cursor: {page_cursor}")
                    emitted_cursor = page_cursor
                # Time-box check at the post boundary (coarse, like a gallery-dl
                # chunk). Backfill/recovery resume from emitted_cursor next chunk.
                if time.monotonic() - start >= time_budget_seconds:
                    budget_hit = True
                    break
                media = self.client.extract_media(post, included)
                if not media:
                    continue
                keys = [_ledger_key(m) for m in media]
                seen = (
                    set()
                    if bypass_seen
                    else self._seen_keys(source_id, keys)
                )
                def _is_seen(m: MediaItem, _seen=seen) -> bool:
                    return _ledger_key(m) in _seen
                outcomes = self.downloader.download_post(
                    post, media, artist_slug, is_seen=_is_seen
                )
                to_mark: list[tuple[str, str]] = []
                for media_item, outcome in zip(media, outcomes, strict=False):
                    key = _ledger_key(media_item)
                    if outcome.status == "downloaded":
                        downloaded += 1
                        if outcome.path is not None:
                            written.append(str(outcome.path))
                        to_mark.append((key, media_item.post_id))
                        consecutive_seen = 0
                    elif outcome.status == "skipped_disk":
                        # Already on disk (a prior run). Reconcile the ledger so a
                        # later tick skips it at tier-1 without a disk stat, but
                        # do NOT re-feed it to phase 3 — attach_in_place would see
                        # the duplicate sha256 and unlink the on-disk copy.
                        to_mark.append((key, media_item.post_id))
                        consecutive_seen += 1
                    elif outcome.status == "skipped_seen":
                        consecutive_seen += 1
                    elif outcome.status == "error":
                        errors += 1
                        # An error neither advances nor resets the run-of-seen.
                    if mode == "tick" and consecutive_seen >= seen_threshold:
                        early_out = True
                        break
                # Mark seen AFTER the network fetch, on its own short session.
                if to_mark:
                    self._mark_seen(source_id, to_mark)
                if early_out:
                    break
            else:
                reached_bottom = True
        except PatreonAPIError as exc:
            # Base of PatreonAuthError + PatreonDriftError — catches every
            # client-level failure; _failure_result maps it to a typed error.
            return self._failure_result(exc, _result)
        if errors:
            log_lines.append(f"{errors} media item(s) failed")
        log_lines.append(
            f"Patreon ingest ({mode}): {downloaded} downloaded, "
            f"{errors} error(s)"
            + (", reached end" if reached_bottom else "")
            + (", time-boxed" if budget_hit else "")
        )
        if budget_hit:
            # A chunk that hit its time-box but made forward progress is a
            # NORMAL chunk boundary, not a failure (PARTIAL → status "ok"); the
            # next chunk resumes from the emitted cursor. No progress → TIMEOUT,
            # which feeds download_service's backfill stall-guard. rc<0 mirrors
            # subprocess TimeoutExpired so completion detection stays false.
            made_progress = downloaded > 0 or emitted_cursor != resume_cursor
            if made_progress:
                return _result(
                    success=False, return_code=-1,
                    error_type=ErrorType.PARTIAL,
                    error_message=(
                        f"Patreon backfill chunk: {downloaded} file(s) — continuing"
                    ),
                )
            return _result(
                success=False, return_code=-1,
                error_type=ErrorType.TIMEOUT,
                error_message="Patreon chunk timed out with no progress",
            )
        # Normal success: reached the bottom, or a tick that early-outed. rc 0 +
        # error_type None is REQUIRED for a backfill/recovery walk that reached
        # the bottom to be marked COMPLETE by
        # download_service._apply_backfill_lifecycle — so we return None even
        # when downloaded == 0 (a re-confirming walk that found nothing new still
        # completed). NO_NEW_CONTENT would read as "not finished" there and trip
        # the stall guard. success=True maps to status "ok" regardless. A tick
        # that early-outed also returns here; ticks never set backfill state so
        # the lifecycle is a no-op for them.
        return _result(
            success=True, return_code=0,
            error_type=None, error_message=None,
        )
    # -- failure mapping ---------------------------------------------------
    def _failure_result(self, exc: Exception, _result) -> DownloadResult:
        """Map a client-level exception to a loud, typed failed DownloadResult.
        We NEVER return a silent zero-download "success" — the whole point of the
        native ingester is to fail RED when Patreon's API shape or our auth
        changes. The typed mapping lets FailingSourcesCard render the right chip
        and tells the operator what to do:
          - PatreonAuthError  → AUTH_ERROR  (rotate cookies)
          - PatreonDriftError → API_DRIFT   (ingester field-set/parser needs update)
          - HTTP 429 / 404    → RATE_LIMITED / NOT_FOUND
          - other HTTP status → HTTP_ERROR;  transport failure → NETWORK_ERROR
        PatreonAuthError and PatreonDriftError both subclass PatreonAPIError, so
        they must be matched before the generic HTTP/transport fallthrough.
        """
        message = str(exc)
        if isinstance(exc, PatreonAuthError):
            error_type = ErrorType.AUTH_ERROR
        elif isinstance(exc, PatreonDriftError):
            error_type = ErrorType.API_DRIFT
            message = f"Patreon API changed — ingester needs update: {message}"
        else:  # generic PatreonAPIError: HTTP non-2xx (status_code set) or transport
            status = getattr(exc, "status_code", None)
            if status == 429:
                error_type = ErrorType.RATE_LIMITED
            elif status == 404:
                error_type = ErrorType.NOT_FOUND
            elif status is not None:
                error_type = ErrorType.HTTP_ERROR
            else:
                error_type = ErrorType.NETWORK_ERROR
        log.warning("Patreon ingest failed (%s): %s", error_type.value, message)
        return _result(
            success=False, return_code=1,
            error_type=error_type, error_message=message,
        )
    # -- seen-ledger (short-lived sessions) --------------------------------
    def _seen_keys(self, source_id: int, keys: list[str]) -> set[str]:
        """Which of `keys` are already in the ledger for this source.
        One short SELECT on its own session — opened and closed without any
        network in between (the GETs happen after, in download_post).
        """
        if not keys:
            return set()
        with self.session_factory() as session:
            rows = session.execute(
                select(PatreonSeenMedia.filehash).where(
                    PatreonSeenMedia.source_id == source_id,
                    PatreonSeenMedia.filehash.in_(keys),
                )
            ).scalars().all()
        return set(rows)
    def _mark_seen(self, source_id: int, items: list[tuple[str, str]]) -> None:
        """Idempotent upsert of (filehash, post_id) ledger rows for a page.
        ON CONFLICT DO NOTHING against the (source_id, filehash) UNIQUE so a
        re-sighting — or a concurrent walk — is a harmless no-op
        ([[scalar_one_or_none-duplicates]]: never check-then-insert without the
        DB constraint backing it). De-dup the batch locally first so a single
        page can't present the same key twice to one INSERT.
        """
        seen_local: set[str] = set()
        values = []
        for key, post_id in items:
            if key in seen_local:
                continue
            seen_local.add(key)
            values.append(
                {"source_id": source_id, "filehash": key, "post_id": post_id}
            )
        if not values:
            return
        with self.session_factory() as session:
            stmt = pg_insert(PatreonSeenMedia).values(values)
            stmt = stmt.on_conflict_do_nothing(
                constraint="uq_patreon_seen_media_source_id"
            )
            session.execute(stmt)
            session.commit()
 async def verify_patreon_credential(
    url: str,
    cookies_path: str | None,
    overrides: dict | None,
 ) -> tuple[bool | None, str]:
    """Native Patreon credential probe — the verify counterpart to the ingester's
    download path, sharing its campaign-id resolution. Resolves the campaign id
    (override / id: URL / vanity) then does ONE authenticated `/api/posts` fetch
    via PatreonClient.verify_auth. Returns the uniform `(ok, message)` contract
    (True / False / None) so download_backends.verify_credential can treat it
    interchangeably with the gallery-dl probe. No download, no DB.
    """
    campaign_id, _ = await resolve_campaign_id_for_source(url, cookies_path, overrides)
    if not campaign_id:
        return None, (
            "Couldn't resolve the Patreon campaign id from the source URL — "
            "can't verify (cookies expired, or the creator moved/renamed?)."
        )
    client = PatreonClient(cookies_path)
    loop = asyncio.get_running_loop()
    return await loop.run_in_executor(None, client.verify_auth, campaign_id)
@@ -19,22 +19,12 @@ import asyncio
 import http.cookiejar
 import logging
 import os
 import re
 import requests
 log = logging.getLogger(__name__)
 _CAMPAIGNS_URL = "https://www.patreon.com/api/campaigns"
 # A source URL of the form `.../id:<digits>` already carries the campaign id
 # (no lookup needed). The vanity regex deliberately EXCLUDES the id: form so the
 # two paths don't overlap.
 _ID_URL_RE = re.compile(r"/id:(\d+)")
 _VANITY_RE = re.compile(
    r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)",
    re.IGNORECASE,
 )
 _USER_AGENT = (
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
@@ -110,38 +100,3 @@ async def resolve_campaign_id(
    Never raises."""
    loop = asyncio.get_running_loop()
    return await loop.run_in_executor(None, _sync_lookup, vanity, cookies_path)
 def extract_vanity(url: str) -> str | None:
    """The vanity slug from a Patreon creator URL, or None for an `id:` URL."""
    m = _VANITY_RE.match(url or "")
    return m.group(1) if m else None
 async def resolve_campaign_id_for_source(
    url: str,
    cookies_path: str | None,
    overrides: dict | None,
 ) -> tuple[str | None, str | None]:
    """Resolve a Patreon source to its campaign id — the single resolution path
    shared by the download ingester and the credential-verify probe.
    Order: cached `patreon_campaign_id` override → an `id:<digits>` URL → a
    vanity lookup against the campaigns API. Returns
    `(campaign_id, newly_resolved_id)`: `newly_resolved_id` is non-None ONLY when
    a vanity lookup actually ran, so the caller knows to cache it on the source
    (the override/id: paths needed no lookup). `(None, None)` when unresolvable.
    Never raises.
    """
    overrides = overrides or {}
    cached = overrides.get("patreon_campaign_id")
    if cached:
        return cached, None
    id_match = _ID_URL_RE.search(url or "")
    if id_match:
        return id_match.group(1), None
    vanity = extract_vanity(url)
    if vanity:
        resolved = await resolve_campaign_id(vanity, cookies_path)
        return resolved, resolved
    return None, None
@@ -64,12 +64,6 @@ class SourceRecord:
    consecutive_failures: int
    next_check_at: str | None
    backfill_runs_remaining: int
    # plan #693: derived from config_overrides for the UI badge.
    backfill_state: str | None  # "running" | "complete" | "stalled" | None (idle)
    backfill_chunks: int
    # plan #697: a running deep-walk that bypasses the Patreon seen-ledger
    # (recovery) vs. a normal backfill. Lets the badge label it "Recovering".
    backfill_bypass_seen: bool
    def to_dict(self) -> dict:
        return {
@@ -88,9 +82,6 @@ class SourceRecord:
            "consecutive_failures": self.consecutive_failures,
            "next_check_at": self.next_check_at,
            "backfill_runs_remaining": self.backfill_runs_remaining,
            "backfill_state": self.backfill_state,
            "backfill_chunks": self.backfill_chunks,
            "backfill_bypass_seen": self.backfill_bypass_seen,
        }
@@ -98,13 +89,10 @@ class SourceRecord:
 _EDITABLE = {"enabled", "url", "config_overrides", "check_interval_override", "platform"}
-# Plan #693: backfill safety cap. "Start backfill" (and a newly created
+# Plan #544 follow-up: newly created enabled sources pre-arm backfill so
-# enabled source) arms a run-until-done walk; this caps how many time-boxed
+# their first N polls walk gallery-dl's full post history with the longer
-# chunks it may spend before pausing as "stalled", so a pathological walk that
+# timeout (matches the manual "Deep scan" button's default).
-# never reaches the bottom can't run forever. Generous on purpose — at
+NEW_SOURCE_BACKFILL_RUNS = 3
 # BACKFILL_CHUNK_SECONDS (600s) per chunk this is ~33h of cumulative walk, far
 # beyond any real catalog; the cursor stall-guard is the real terminator.
 BACKFILL_MAX_CHUNKS = 200
 class SourceService:
@@ -147,7 +135,6 @@ class SourceService:
        self, source: Source, artist: Artist, settings: ImportSettings,
    ) -> SourceRecord:
        nxt = compute_next_check_at(source, artist, settings)
        co = source.config_overrides or {}
        return SourceRecord(
            id=source.id,
            artist_id=source.artist_id,
@@ -164,9 +151,6 @@ class SourceService:
            consecutive_failures=source.consecutive_failures or 0,
            next_check_at=nxt.isoformat() if nxt else None,
            backfill_runs_remaining=source.backfill_runs_remaining or 0,
            backfill_state=co.get("_backfill_state"),
            backfill_chunks=int(co.get("_backfill_chunks", 0)),
            backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")),
        )
    async def _row_to_record(self, source: Source) -> SourceRecord:
@@ -228,17 +212,16 @@ class SourceService:
            select(func.count(Source.id)).where(Source.artist_id == artist_id)
        )).scalar_one()
-        # Plan #693: a freshly added subscription has no archive yet, so it
+        # Plan #544 follow-up: a freshly added subscription has no archive
-        # should walk its full post history once. Arm run-until-done backfill
+        # yet, so the first few polls would walk the full post history in
-        # (state="running" + the chunk cap); the time-boxed chunks march to the
+        # tick mode and trip exit:20 after ~20 contiguous archive hits —
-        # bottom across ticks, then flip to "complete" and tick mode takes over.
+        # except there are none yet, so tick mode would walk forever and
-        # Disabled sources (incl. sidecar synthetics, url='sidecar:...') are
+        # blow the wall-clock cap. Pre-arm backfill so the initial syncs
-        # never polled, so leave them idle.
+        # use the longer timeout + skip:True walk. Tick mode resumes once
-        if enabled:
+        # the budget is spent or the queue drains.
-            config_overrides = {**(config_overrides or {}), "_backfill_state": "running"}
+        # Disabled sources (incl. sidecar synthetics, url='sidecar:...')
-            backfill_runs = BACKFILL_MAX_CHUNKS
+        # are never polled, so leave their counter at 0.
-        else:
+        backfill_runs = NEW_SOURCE_BACKFILL_RUNS if enabled else 0
            backfill_runs = 0
        source = Source(
            artist_id=artist_id, platform=platform, url=url,
            enabled=enabled, config_overrides=config_overrides,
@@ -303,69 +286,22 @@ class SourceService:
        await self.session.commit()
        return await self._row_to_record(source)
-    async def start_backfill(self, source_id: int) -> SourceRecord:
+    async def set_backfill_runs(
-        """Plan #693: arm a run-until-done backfill. Sets state="running" and
+        self, source_id: int, runs: int,
-        the chunk cap; download runs then walk the full post history in
+    ) -> SourceRecord:
-        time-boxed chunks (skip:True + BACKFILL_CHUNK_SECONDS), resuming from
+        """Plan #544: arm a source for backfill mode. The next `runs`
-        the cursor each chunk, until gallery-dl reaches the bottom (→ state
+        download runs will use gallery-dl's full-walk config (skip: True
-        "complete") or the cap/stall-guard pauses it (→ "stalled"). Clears any
+        + 30-min timeout) instead of the catch-up default. Runs must be
-        prior cursor/chunk/stall state so a re-start walks fresh from the top."""
+        1..10 — bigger is rejected to keep the operator from accidentally
        setting a runaway budget."""
        if not isinstance(runs, int) or runs < 1 or runs > 10:
            raise ValueError("runs must be an integer in [1, 10]")
        source = (await self.session.execute(
            select(Source).where(Source.id == source_id)
        )).scalar_one_or_none()
        if source is None:
            raise LookupError(f"source id={source_id} not found")
-        co = dict(source.config_overrides or {})
+        source.backfill_runs_remaining = runs
        co["_backfill_state"] = "running"
        for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks"):
            co.pop(k, None)
        source.config_overrides = co
        source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
        await self.session.commit()
        return await self._row_to_record(source)
    async def start_recovery(self, source_id: int) -> SourceRecord:
        """Plan #697: arm a RECOVERY walk — a backfill that bypasses the Patreon
        seen-ledger so deliberately-dropped-and-deleted near-dups get re-fetched
        and re-evaluated under the CURRENT pHash threshold (tier-2 disk still
        spares files we kept). Reuses the entire #693 backfill state machine
        (time-boxed chunks, cursor checkpoint, complete/stall lifecycle) plus the
        `_backfill_bypass_seen` flag that flips download mode to recovery. Clears
        any prior cursor/chunk/stall state so it walks fresh from the top. The
        flag is cleared on completion (download_service) and on stop.
        Recovery is Patreon-only (the seen-ledger is Patreon's); for other
        platforms the flag is inert (download_service ignores it) and the walk
        runs as a plain backfill. The UI gates the action to Patreon sources."""
        source = (await self.session.execute(
            select(Source).where(Source.id == source_id)
        )).scalar_one_or_none()
        if source is None:
            raise LookupError(f"source id={source_id} not found")
        co = dict(source.config_overrides or {})
        co["_backfill_state"] = "running"
        co["_backfill_bypass_seen"] = True
        for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks"):
            co.pop(k, None)
        source.config_overrides = co
        source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
        await self.session.commit()
        return await self._row_to_record(source)
    async def stop_backfill(self, source_id: int) -> SourceRecord:
        """Plan #693: cancel an in-progress backfill — back to idle/tick mode.
        Clears the running state + cursor/chunk/stall bookkeeping."""
        source = (await self.session.execute(
            select(Source).where(Source.id == source_id)
        )).scalar_one_or_none()
        if source is None:
            raise LookupError(f"source id={source_id} not found")
        co = dict(source.config_overrides or {})
        for k in ("_backfill_state", "_backfill_cursor", "_backfill_cursor_stalls",
                  "_backfill_chunks", "_backfill_bypass_seen"):
            co.pop(k, None)
        source.config_overrides = co
        source.backfill_runs_remaining = 0
        await self.session.commit()
        return await self._row_to_record(source)
@@ -31,7 +31,7 @@ _KEY_PATH = IMAGES_ROOT / "secrets" / "credential_key.b64"
 # SoftTimeLimitExceeded in-process, whereas the HARD limit SIGKILLs the
 # worker (no chance to finalize). Both gallery-dl subprocess budgets
 # (gallery_dl.py: _DEFAULT_GDL_TIMEOUT_SECONDS=870 tick,
-# BACKFILL_CHUNK_SECONDS=600 per backfill chunk, plan #693) MUST sit below the soft limit
+# BACKFILL_TIMEOUT_SECONDS=1170 backfill) MUST sit below the soft limit
 # so subprocess.run raises its own TimeoutExpired first — that path
 # captures partial stdout/stderr and finalizes the DownloadEvent. soft is
 # max-subprocess (1170) + ~180s phase-3 persist headroom; hard is soft +
@@ -143,11 +143,6 @@ def download_source(self, source_id: int) -> int:
                        gdl=gdl,
                        importer=importer,
                        cred_service=cred_service,
                        # The native Patreon ingester opens its own short-lived
                        # sync sessions for the seen-ledger (never held across
                        # the walk). Same factory the importer's sync session
                        # comes from — a different DB connection per checkout.
                        sync_session_factory=SyncFactory,
                    )
                    return await svc.download_source(source_id)
        finally:
@@ -203,37 +203,14 @@ function nextFrame() {
 }
@media (max-width: 900px) {
-  /* Stacked layout (operator-flagged 2026-06-05): pin the image pane and
+  .fc-viewer__body { flex-direction: column; }
-     its controls at the top while the metadata panel scrolls beneath it.
+  /* Side panel drops below the image — the next arrow uses the full width. */
-     Previously the image + a 40vh panel shared one fixed viewport and the
+  .fc-viewer__nav--next { right: 16px; }
     controls could be pushed out of view; now the body scrolls and the
     media is sticky, so the image + prev/next/close (and the integrity
     badge) stay visible no matter how far the panel is scrolled. */
  .fc-viewer__body {
    flex-direction: column;
    overflow-y: auto;
  }
  .fc-viewer__media {
    flex: none;
    height: 55vh;
    position: sticky;
    top: 0;
    z-index: 1;
    /* Opaque obsidian so the scrolling panel never bleeds through the
       haze behind the pinned image. */
    background: rgb(20, 23, 26);
  }
  .fc-viewer__side {
    width: 100%;
-    max-height: none;
+    max-height: 40vh;
    border-left: none;
    border-top: 1px solid rgb(var(--v-theme-surface-light));
  }
  /* Re-center the prev/next arrows over the 55vh image band (their base
     top:50% would land on the scrolling panel); next uses the full width
     now that the panel is below, not beside. Close + integrity badge keep
     their top:16px/72px — already within the pinned band. */
  .fc-viewer__nav { top: 27.5vh; }
  .fc-viewer__nav--next { right: 16px; }
 }
 </style>
@@ -65,17 +65,8 @@ const store = useTagStore()
 // Vuetify's v-text-field exposes .focus() on the component instance;
 // nextTick waits for the modal's mount to finish so the inner <input>
 // element exists.
 //
 // EXCEPTION — not on mobile (operator-flagged 2026-06-05): focusing the
 // field pops the soft keyboard, which shrinks the visual viewport and
 // shoves the pinned image + its nav/close controls out of view. 900px is
 // the modal's stacked-layout breakpoint (ImageViewer.vue). matchMedia is
 // safe on plain HTTP (not a secure-context API).
 const inputRef = ref(null)
-onMounted(() => {
+onMounted(() => { nextTick(() => inputRef.value?.focus?.()) })
  if (window.matchMedia?.('(max-width: 900px)')?.matches) return
  nextTick(() => inputRef.value?.focus?.())
 })
 // Single text input; no kind dropdown. Client-side mirror of the
 // backend's parse_kind_prefix lives below — kept in sync with
@@ -93,7 +93,6 @@ const ERROR_TYPE_COLOR = {
  unsupported_url: 'error',
  http_error: 'error',
  unknown_error: 'error',
  api_drift: 'error',
  partial: 'info',
  tier_limited: 'info',
  no_new_content: 'info',
@@ -109,7 +108,6 @@ const ERROR_TYPE_HINT = {
  http_error: 'Generic HTTP error — see Logs.',
  unsupported_url: 'gallery-dl does not support this URL pattern.',
  unknown_error: 'Could not classify — see Logs.',
  api_drift: 'Patreon changed its API shape — the native ingester needs a code update.',
 }
 function errorTypeColor(t) { return ERROR_TYPE_COLOR[t] || 'error' }
 function errorTypeHint(t) { return ERROR_TYPE_HINT[t] || '' }
@@ -25,18 +25,9 @@
        size="x-small" color="error" variant="tonal" label
      >{{ source.consecutive_failures }} err</v-chip>
      <v-chip
-        v-else-if="source.backfill_state === 'running'"
+        v-else-if="(source.backfill_runs_remaining || 0) > 0"
        size="x-small" color="info" variant="tonal" label
-      >{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling'
+      >backfill ({{ source.backfill_runs_remaining }}×)</v-chip>
        }}{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }}</v-chip>
      <v-chip
        v-else-if="source.backfill_state === 'complete'"
        size="x-small" color="success" variant="tonal" label
      >Backfilled</v-chip>
      <v-chip
        v-else-if="source.backfill_state === 'stalled'"
        size="x-small" color="warning" variant="tonal" label
      >Stalled</v-chip>
    </div>
    <div class="fc-source-card__actions">
@@ -49,25 +40,11 @@
      </v-btn>
      <v-btn
        size="x-small" variant="text"
-        :color="source.backfill_state === 'running' ? 'warning' : undefined"
+        :disabled="(source.backfill_runs_remaining || 0) > 0"
        @click.stop="$emit('backfill', source)"
      >
-        <v-icon>{{ source.backfill_state === 'running' ? 'mdi-stop' : 'mdi-magnify-scan' }}</v-icon>
+        <v-icon>mdi-magnify-scan</v-icon>
-        <v-tooltip activator="parent" location="top">
+        <v-tooltip activator="parent" location="top">Deep scan</v-tooltip>
          {{ source.backfill_state === 'running'
            ? (source.backfill_bypass_seen ? 'Stop recovery' : 'Stop backfill')
            : 'Backfill full history' }}
        </v-tooltip>
      </v-btn>
      <v-btn
        v-if="source.platform === 'patreon' && source.backfill_state !== 'running'"
        size="x-small" variant="text"
        @click.stop="$emit('recover', source)"
      >
        <v-icon>mdi-backup-restore</v-icon>
        <v-tooltip activator="parent" location="top">
          Recover — re-fetch dropped near-dups &amp; re-evaluate under the current threshold
        </v-tooltip>
      </v-btn>
      <v-btn size="x-small" variant="text" @click.stop="$emit('edit', source)">
        <v-icon>mdi-pencil</v-icon>
@@ -96,7 +73,7 @@ const props = defineProps({
  checking: { type: Boolean, default: false },
  warningThreshold: { type: Number, default: 5 },
 })
-const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover'])
+const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill'])
 function onToggleEnabled(value) {
  emit('toggle', { source: props.source, enabled: value })
@@ -32,18 +32,11 @@
        size="x-small" color="error" variant="tonal" label
      >{{ source.consecutive_failures }}</v-chip>
      <v-chip
-        v-else-if="source.backfill_state === 'running'"
+        v-else-if="(source.backfill_runs_remaining || 0) > 0"
        size="x-small" color="info" variant="tonal" label
-      >{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling'
+      >
-        }}{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }}</v-chip>
+        backfill ({{ source.backfill_runs_remaining }}×)
-      <v-chip
+      </v-chip>
        v-else-if="source.backfill_state === 'complete'"
        size="x-small" color="success" variant="tonal" label
      >Backfilled</v-chip>
      <v-chip
        v-else-if="source.backfill_state === 'stalled'"
        size="x-small" color="warning" variant="tonal" label
      >Stalled</v-chip>
      <span v-else class="fc-source-row__zero">0</span>
    </td>
    <td class="fc-source-row__actions">
@@ -56,25 +49,13 @@
        <v-tooltip activator="parent" location="top">Check now</v-tooltip>
      </v-btn>
      <v-btn
-        size="x-small" variant="text"
+        icon="mdi-magnify-scan" size="x-small" variant="text"
-        :color="source.backfill_state === 'running' ? 'warning' : undefined"
+        :disabled="(source.backfill_runs_remaining || 0) > 0"
        @click.stop="$emit('backfill', source)"
      >
-        <v-icon>{{ source.backfill_state === 'running' ? 'mdi-stop' : 'mdi-magnify-scan' }}</v-icon>
+        <v-icon>mdi-magnify-scan</v-icon>
        <v-tooltip activator="parent" location="top">
-          {{ source.backfill_state === 'running'
+          Deep scan — walk full history for next few runs
            ? (source.backfill_bypass_seen ? 'Stop recovery' : 'Stop backfill')
            : 'Backfill — walk the full post history until complete' }}
        </v-tooltip>
      </v-btn>
      <v-btn
        v-if="source.platform === 'patreon' && source.backfill_state !== 'running'"
        size="x-small" variant="text"
        @click.stop="$emit('recover', source)"
      >
        <v-icon>mdi-backup-restore</v-icon>
        <v-tooltip activator="parent" location="top">
          Recover — re-fetch dropped near-dups &amp; re-evaluate under the current threshold
        </v-tooltip>
      </v-btn>
      <v-btn
@@ -104,7 +85,7 @@ const props = defineProps({
  checking: { type: Boolean, default: false },
  warningThreshold: { type: Number, default: 5 },
 })
-const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover'])
+const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill'])
 function onToggleEnabled(value) {
  emit('toggle', { source: props.source, enabled: value })
@@ -175,7 +175,6 @@
                    @toggle="toggleSourceEnabled"
                    @check="onCheck"
                    @backfill="onBackfill"
                    @recover="onRecover"
                  />
                  <tr v-if="item.sources.length === 0">
                    <td colspan="8" class="fc-subs__sources-empty">
@@ -252,7 +251,6 @@
            @toggle="toggleSourceEnabled"
            @check="onCheck"
            @backfill="onBackfill"
            @recover="onRecover"
          />
          <div v-if="item.sources.length === 0" class="fc-subs__sources-empty">
            No sources yet. Tap + to add one.
@@ -534,39 +532,31 @@ async function onCheck(source) {
  }
 }
-// Plan #693: toggle a run-until-done backfill. Starting walks the full post
+// Plan #544: arm a source for backfill mode (gallery-dl walks the full
-// history in time-boxed chunks across ticks until it reaches the bottom (the
+// post history) for the next N download runs. Default 3 — enough budget
-// row badge tracks progress / completion); stopping cancels back to tick mode.
+// to finish a deep creator without re-prompting the operator across
 // timeout boundaries. The chip on the row reflects the remaining count.
 async function onBackfill(source) {
-  const running = source.backfill_state === 'running'
+  const raw = globalThis.window?.prompt(
-  try {
+    `Deep scan "${source.artist_name} (${source.platform})" — walk full history for the next how many download runs? (1–10, default 3)`,
-    if (running) {
+    '3',
-      await store.stopBackfill(source.id, source.artist_id)
+  )
-      toast({ text: `Backfill stopped for ${source.artist_name}`, type: 'success' })
+  if (raw == null) return
-    } else {
+  const runs = parseInt(raw, 10)
-      await store.startBackfill(source.id, source.artist_id)
+  if (!Number.isFinite(runs) || runs < 1 || runs > 10) {
-      toast({ text: `Backfill started for ${source.artist_name}`, type: 'success' })
+    toast({ text: 'Deep scan: runs must be 1–10', type: 'error' })
-    }
+    return
    await store.loadAll()
  } catch (e) {
    toast({
      text: `Backfill ${running ? 'stop' : 'start'} failed: ${e?.detail || e?.message || e}`,
      type: 'error',
    })
  }
 }
 // Plan #697: arm a recovery walk (Patreon-only) — re-fetches dropped-and-deleted
 // near-dups and re-evaluates them under the current pHash threshold. Reuses the
 // backfill lifecycle/badge; stop via the same Stop control (onBackfill).
 async function onRecover(source) {
  try {
-    await store.recoverSource(source.id, source.artist_id)
+    await store.setBackfill(source.id, runs, source.artist_id)
-    toast({ text: `Recovery started for ${source.artist_name}`, type: 'success' })
+    toast({
      text: `Deep scan armed for ${runs} run${runs === 1 ? '' : 's'}`,
      type: 'success',
    })
    await store.loadAll()
  } catch (e) {
    toast({
-      text: `Recovery start failed: ${e?.detail || e?.message || e}`,
+      text: `Deep scan failed: ${e?.detail || e?.message || e}`,
      type: 'error',
    })
  }
@@ -85,24 +85,11 @@ export const useSourcesStore = defineStore('sources', () => {
    }
  }
-  // Plan #693: start/stop a run-until-done backfill. 'start' walks the full
+  // Plan #544: arm a source for backfill mode. The next `runs` download
-  // post history in time-boxed chunks until it reaches the bottom (then the
+  // runs (default 3) walk gallery-dl's full post history instead of
-  // source shows backfill_state 'complete'); 'stop' cancels back to tick mode.
+  // exiting early at the first contiguous archived block.
-  async function startBackfill(id, artistIdHint = null) {
+  async function setBackfill(id, runs = 3, artistIdHint = null) {
-    const body = await api.post(`/api/sources/${id}/backfill`, { body: { action: 'start' } })
+    const body = await api.post(`/api/sources/${id}/backfill`, { body: { runs } })
    _invalidate(artistIdHint ?? body.artist_id)
    return body
  }
  async function stopBackfill(id, artistIdHint = null) {
    const body = await api.post(`/api/sources/${id}/backfill`, { body: { action: 'stop' } })
    _invalidate(artistIdHint ?? body.artist_id)
    return body
  }
  // Plan #697: arm a recovery walk — a backfill that bypasses the Patreon
  // seen-ledger to re-fetch dropped-and-deleted near-dups and re-evaluate them
  // under the current pHash threshold. Stop with stopBackfill (shared lifecycle).
  async function recoverSource(id, artistIdHint = null) {
    const body = await api.post(`/api/sources/${id}/backfill`, { body: { action: 'recover' } })
    _invalidate(artistIdHint ?? body.artist_id)
    return body
  }
@@ -133,9 +120,7 @@ export const useSourcesStore = defineStore('sources', () => {
    loadAll, loadForArtist,
    create, update, remove,
    checkNow,
-    startBackfill,
+    setBackfill,
    stopBackfill,
    recoverSource,
    findOrCreateArtist, autocompleteArtist,
    loadScheduleStatus,
    sourcesByArtistGrouped,
@@ -1,121 +0,0 @@
 {
  "data": [
    {
      "id": "1001",
      "type": "post",
      "attributes": {
        "title": "Image gallery post",
        "published_at": "2026-05-01T12:00:00.000+00:00",
        "post_type": "image_file",
        "content": "<p>just a gallery</p>",
        "url": "https://www.patreon.com/posts/1001",
        "image": {
          "large_url": "https://c10.patreonusercontent.com/4/large/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg",
          "url": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg"
        }
      },
      "relationships": {
        "images": {
          "data": [
            {"id": "9001", "type": "media"},
            {"id": "9002", "type": "media"}
          ]
        }
      }
    },
    {
      "id": "1002",
      "type": "post",
      "attributes": {
        "title": "Attachment post",
        "published_at": "2026-04-15T09:30:00.000+00:00",
        "post_type": "text_only",
        "content": "<p>here is a zip</p>",
        "url": "https://www.patreon.com/posts/1002",
        "post_file": {
          "name": "bonus-pack.zip",
          "url": "https://www.patreon.com/file?h=cccccccccccccccccccccccccccccccc&i=1002"
        }
      },
      "relationships": {
        "attachments_media": {
          "data": [
            {"id": "9003", "type": "media"}
          ]
        }
      }
    },
    {
      "id": "1003",
      "type": "post",
      "attributes": {
        "title": "Inline content post",
        "published_at": "2026-04-01T08:00:00.000+00:00",
        "post_type": "text_only",
        "content": "<p>look</p><figure><img src=\"https://c10.patreonusercontent.com/4/orig/dddddddddddddddddddddddddddddddd/inline.png?token=x&amp;v=2\" alt=\"inline\"></figure>",
        "url": "https://www.patreon.com/posts/1003"
      },
      "relationships": {}
    },
    {
      "id": "1004",
      "type": "post",
      "attributes": {
        "title": "Video post",
        "published_at": "2026-03-20T18:45:00.000+00:00",
        "post_type": "video_external_file",
        "content": "<p>watch</p>",
        "url": "https://www.patreon.com/posts/1004",
        "post_file": {
          "name": "clip.m3u8",
          "url": "https://stream.mux.com/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee.m3u8?token=jwt"
        }
      },
      "relationships": {}
    }
  ],
  "included": [
    {
      "id": "9001",
      "type": "media",
      "attributes": {
        "file_name": "gallery-one.jpg",
        "download_url": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg",
        "image_urls": {
          "original": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg",
          "default": "https://c10.patreonusercontent.com/4/default/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg"
        }
      }
    },
    {
      "id": "9002",
      "type": "media",
      "attributes": {
        "file_name": "gallery-two.jpg",
        "download_url": "https://c10.patreonusercontent.com/4/orig/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/gallery-two.jpg",
        "image_urls": {
          "original": "https://c10.patreonusercontent.com/4/orig/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/gallery-two.jpg"
        }
      }
    },
    {
      "id": "9003",
      "type": "media",
      "attributes": {
        "file_name": "bonus-pack.zip",
        "download_url": "https://www.patreon.com/file/download?h=cccccccccccccccccccccccccccccccc"
      }
    },
    {
      "id": "5555",
      "type": "campaign",
      "attributes": {
        "name": "Example Creator",
        "url": "https://www.patreon.com/example"
      }
    }
  ],
  "links": {
    "next": "https://www.patreon.com/api/posts?filter%5Bcampaign_id%5D=5555&page%5Bcursor%5D=NEXTCURSOR123&sort=-published_at"
  }
 }
@@ -155,8 +155,6 @@ async def test_verify_no_enabled_source_is_untestable(client):
@pytest.mark.asyncio
 async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypatch):
    """A gallery-dl platform routes through GalleryDLService.verify; success
    stamps last_verified (platform-agnostic endpoint behavior)."""
    from backend.app.models import Artist, Source
    from backend.app.services import gallery_dl as gdl_mod
@@ -165,45 +163,6 @@ async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypa
        return (True, "Credentials valid — the feed authenticated.")
    monkeypatch.setattr(gdl_mod.GalleryDLService, "verify", _fake_verify)
    await client.post("/api/credentials", json={
        "platform": "subscribestar", "credential_type": "cookies", "data": _NETSCAPE,
    })
    artist = Artist(name="Maewix", slug="maewix")
    db.add(artist)
    await db.flush()
    db.add(Source(
        artist_id=artist.id, platform="subscribestar",
        url="https://www.subscribestar.com/maewix", enabled=True, config_overrides={},
    ))
    await db.commit()
    resp = await client.post("/api/credentials/subscribestar/verify")
    body = await resp.get_json()
    assert body["valid"] is True
    assert body["last_verified"] is not None
    # The stamp is persisted on the credential record.
    rec = await (await client.get("/api/credentials/subscribestar")).get_json()
    assert rec["last_verified"] is not None
@pytest.mark.asyncio
 async def test_verify_patreon_uses_native_ingester_not_gallery_dl(client, db, monkeypatch):
    """plan #697 cutover: Patreon credential verify routes through the native
    ingester (verify_patreon_credential), NOT gallery-dl --simulate."""
    from backend.app.models import Artist, Source
    from backend.app.services import gallery_dl as gdl_mod
    from backend.app.services import patreon_ingester as pi_mod
    async def _native_verify(url, cookies_path, overrides):
        return (True, "Credentials valid — the Patreon feed authenticated.")
    monkeypatch.setattr(pi_mod, "verify_patreon_credential", _native_verify)
    # gallery-dl must NOT be consulted for Patreon.
    async def _boom(self, *args, **kwargs):
        raise AssertionError("gallery-dl verify must not run for patreon")
    monkeypatch.setattr(gdl_mod.GalleryDLService, "verify", _boom)
    await client.post("/api/credentials", json={
        "platform": "patreon", "credential_type": "cookies", "data": _NETSCAPE,
    })
@@ -221,6 +180,10 @@ async def test_verify_patreon_uses_native_ingester_not_gallery_dl(client, db, mo
    assert body["valid"] is True
    assert body["last_verified"] is not None
    # The stamp is persisted on the credential record.
    rec = await (await client.get("/api/credentials/patreon")).get_json()
    assert rec["last_verified"] is not None
@pytest.mark.asyncio
 async def test_verify_reports_auth_failure(client, db, monkeypatch):
@@ -199,7 +199,7 @@ async def test_list_derives_next_check_at_when_last_checked_set(
@pytest.mark.asyncio
-async def test_backfill_endpoint_start_and_stop(client, artist, db):
+async def test_backfill_endpoint_arms_source(client, artist, db):
    src = Source(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/alice-backfill", enabled=True,
@@ -208,21 +208,18 @@ async def test_backfill_endpoint_start_and_stop(client, artist, db):
    await db.commit()
    sid = src.id
-    resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "start"})
+    resp = await client.post(f"/api/sources/{sid}/backfill", json={"runs": 5})
    assert resp.status_code == 200
    body = await resp.get_json()
-    assert body["backfill_state"] == "running"
+    assert body["backfill_runs_remaining"] == 5
-    # GET reflects the running state.
+    # GET reflects the new state.
    one = await client.get(f"/api/sources/{sid}")
-    assert (await one.get_json())["backfill_state"] == "running"
+    assert (await one.get_json())["backfill_runs_remaining"] == 5
    stopped = await client.post(f"/api/sources/{sid}/backfill", json={"action": "stop"})
    assert (await stopped.get_json())["backfill_state"] is None
@pytest.mark.asyncio
-async def test_backfill_endpoint_defaults_to_start(client, artist, db):
+async def test_backfill_endpoint_defaults_to_three(client, artist, db):
    src = Source(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/alice-backfill-default", enabled=True,
@@ -231,47 +228,26 @@ async def test_backfill_endpoint_defaults_to_start(client, artist, db):
    await db.commit()
    resp = await client.post(f"/api/sources/{src.id}/backfill", json={})
    body = await resp.get_json()
-    assert body["backfill_state"] == "running"
+    assert body["backfill_runs_remaining"] == 3
@pytest.mark.asyncio
-async def test_backfill_endpoint_rejects_bad_action(client, artist, db):
+async def test_backfill_endpoint_rejects_out_of_range(client, artist, db):
    src = Source(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/alice-backfill-bad", enabled=True,
    )
    db.add(src)
    await db.commit()
-    bad = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "nope"})
+    bad = await client.post(f"/api/sources/{src.id}/backfill", json={"runs": 0})
    assert bad.status_code == 400
    too_big = await client.post(
        f"/api/sources/{src.id}/backfill", json={"runs": 99}
    )
    assert too_big.status_code == 400
@pytest.mark.asyncio
 async def test_backfill_endpoint_404_when_source_missing(client):
-    resp = await client.post("/api/sources/999999/backfill", json={"action": "start"})
+    resp = await client.post("/api/sources/999999/backfill", json={"runs": 3})
    assert resp.status_code == 404
@pytest.mark.asyncio
 async def test_backfill_endpoint_recover_arms_bypass(client, artist, db):
    """Plan #697: action='recover' arms a backfill that bypasses the Patreon
    seen-ledger; the response exposes backfill_bypass_seen=True so the UI badge
    can label it 'Recovering'. Stop clears it."""
    src = Source(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/alice-recover", enabled=True,
    )
    db.add(src)
    await db.commit()
    sid = src.id
    resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "recover"})
    assert resp.status_code == 200
    body = await resp.get_json()
    assert body["backfill_state"] == "running"
    assert body["backfill_bypass_seen"] is True
    stopped = await client.post(f"/api/sources/{sid}/backfill", json={"action": "stop"})
    stopped_body = await stopped.get_json()
    assert stopped_body["backfill_state"] is None
    assert stopped_body["backfill_bypass_seen"] is False
@@ -1,23 +0,0 @@
 """download_backends — the single predicate that routes a platform to the
 native ingester vs. gallery-dl. Pure, no DB."""
 from backend.app.services.download_backends import (
    NATIVE_INGESTER_PLATFORMS,
    uses_native_ingester,
 )
 def test_patreon_is_native():
    assert uses_native_ingester("patreon") is True
    assert "patreon" in NATIVE_INGESTER_PLATFORMS
 def test_gallery_dl_platforms_are_not_native():
    # The five platforms still served by gallery-dl must NOT route to the
    # native ingester — guards an accidental over-broad migration.
    for platform in ("subscribestar", "hentaifoundry", "discord", "pixiv", "deviantart"):
        assert uses_native_ingester(platform) is False
 def test_unknown_platform_is_not_native():
    assert uses_native_ingester("nonsense") is False
@@ -94,23 +94,6 @@ def _fake_gdl_with_result(result):
    return fake
 def _stub_patreon_ingester(svc, result, resolved_campaign_id=None):
    """Route the Patreon phase-2 branch (plan #697 native ingester) to a canned
    DownloadResult so these tests exercise download_service's
    phase-2-result→phase-3 handling (status mapping, backfill lifecycle, health)
    without the real ingester's network/DB. The ingester itself is covered by
    test_patreon_ingester.py. Returns a list capturing (ctx, source_config, mode)
    per call so a test can assert mode / resume_cursor threading."""
    calls = []
    async def fake(ctx, source_config, mode):
        calls.append({"ctx": ctx, "source_config": source_config, "mode": mode})
        return result, resolved_campaign_id
    svc._run_patreon_ingester = fake
    return calls
@pytest.mark.asyncio
 async def test_download_source_attaches_written_files(
    db, db_sync, tmp_path, seed_artist_and_source,
@@ -129,13 +112,12 @@ async def test_download_source_attaches_written_files(
    _make_jpg(f1, split="h")
    _make_jpg(f2, split="v")
-    result = _make_fake_dl_result(
+    fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
        success=True,
        written_paths=[str(f1), str(f2)],
        files_downloaded=2,
        stdout=f"{f1}\n{f2}\n",
-    )
+    ))
    fake_gdl = _fake_gdl_with_result(result)
    sync_settings = db_sync.execute(
        select(ImportSettings).where(ImportSettings.id == 1)
@@ -155,7 +137,6 @@ async def test_download_source_attaches_written_files(
        async_session=db, sync_session=db_sync,
        gdl=fake_gdl, importer=importer, cred_service=cred_service,
    )
    _stub_patreon_ingester(svc, result)
    event_id = await svc.download_source(source.id)
    ev = (await db.execute(
@@ -233,19 +214,40 @@ async def test_in_flight_idempotency_returns_existing_event(
@pytest.mark.asyncio
-async def test_patreon_resolved_campaign_id_is_cached(
+async def test_patreon_retry_caches_campaign_id(
-    db, db_sync, tmp_path, seed_artist_and_source,
+    db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
 ):
    """plan #697: the native ingester resolves a Patreon vanity → campaign id
    on the fly (replacing gallery-dl's reactive campaign-id retry). When phase 2
    reports a freshly-resolved id, phase 3 caches it on the source so later runs
    skip the lookup. Stub the ingester to report a resolved id; assert it lands
    in config_overrides."""
    from backend.app.services.download_service import DownloadService
    from backend.app.services.importer import Importer
    _artist, source = seed_artist_and_source
    failed = _make_fake_dl_result(
        success=False, written_paths=[], stdout="",
        stderr="[patreon][error] Failed to extract campaign ID for alice\n",
    )
    succeeded = _make_fake_dl_result(success=True, written_paths=[])
    download_calls = []
    async def fake_download(url, **k):
        download_calls.append(url)
        return failed if len(download_calls) == 1 else succeeded
    fake_gdl = MagicMock()
    fake_gdl.download = fake_download
    fake_gdl._compute_run_stats = lambda *a, **k: {
        "exit_code": 0, "downloaded_count": 0, "skipped_count": 0,
        "per_item_failures": 0, "warning_count": 0, "tier_gated_count": 0,
    }
    fake_gdl._extract_errors_warnings = lambda *a, **k: ""
    fake_gdl._truncate_log = lambda x, **k: x
    monkeypatch.setattr(
        "backend.app.services.download_service.resolve_campaign_id",
        AsyncMock(return_value="99"),
    )
    sync_settings = db_sync.execute(
        select(ImportSettings).where(ImportSettings.id == 1)
    ).scalar_one()
@@ -259,104 +261,19 @@ async def test_patreon_resolved_campaign_id_is_cached(
    svc = DownloadService(
        async_session=db, sync_session=db_sync,
-        gdl=_fake_gdl_with_result(_make_fake_dl_result(success=True)),
+        gdl=fake_gdl, importer=importer, cred_service=cred_service,
        importer=importer, cred_service=cred_service,
    )
    _stub_patreon_ingester(
        svc, _make_fake_dl_result(success=True, written_paths=[]),
        resolved_campaign_id="99",
    )
    await svc.download_source(source.id)
    assert len(download_calls) == 2
    assert "id:99" in download_calls[1]
    overrides = db_sync.execute(
        select(Source.config_overrides).where(Source.id == source.id)
    ).scalar_one()
    assert overrides.get("patreon_campaign_id") == "99"
@pytest.mark.asyncio
 async def test_run_patreon_ingester_resolves_vanity_and_runs(
    db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
 ):
    """_run_patreon_ingester: with no cached campaign id, resolve the vanity and
    pass the resolved id straight to the ingester; report it back so phase 3 can
    cache it."""
    from backend.app.services import download_service as dl_mod
    from backend.app.services.download_service import DownloadService
    from backend.app.services.gallery_dl import SourceConfig
    _artist, source = seed_artist_and_source
    monkeypatch.setattr(
        dl_mod, "resolve_campaign_id_for_source",
        AsyncMock(return_value=("4242", "4242")),
    )
    run_kwargs = {}
    class _FakeIngester:
        def __init__(self, **kw):
            pass
        def run(self, **kw):
            run_kwargs.update(kw)
            return _make_fake_dl_result(success=True, written_paths=[])
    monkeypatch.setattr(dl_mod, "PatreonIngester", _FakeIngester)
    svc = DownloadService(
        async_session=db, sync_session=db_sync,
        gdl=_fake_gdl_with_result(_make_fake_dl_result(success=True)),
        importer=MagicMock(), cred_service=MagicMock(),
        sync_session_factory=MagicMock(),
    )
    ctx = {
        "source_id": source.id, "url": "https://patreon.com/alice",
        "artist_slug": "alice", "cookies_path": None,
        "config_overrides": {},
    }
    result, resolved = await svc._run_patreon_ingester(
        ctx, SourceConfig.from_dict({}), "tick",
    )
    assert result.success is True
    assert resolved == "4242"
    assert run_kwargs["campaign_id"] == "4242"
    assert run_kwargs["mode"] == "tick"
@pytest.mark.asyncio
 async def test_run_patreon_ingester_unresolvable_fails_loud(
    db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
 ):
    """A campaign id we can't resolve is a loud NOT_FOUND failure, never a
    silent empty success."""
    from backend.app.services import download_service as dl_mod
    from backend.app.services.download_service import DownloadService
    from backend.app.services.gallery_dl import ErrorType, SourceConfig
    _artist, source = seed_artist_and_source
    monkeypatch.setattr(
        dl_mod, "resolve_campaign_id_for_source",
        AsyncMock(return_value=(None, None)),
    )
    svc = DownloadService(
        async_session=db, sync_session=db_sync,
        gdl=MagicMock(), importer=MagicMock(), cred_service=MagicMock(),
        sync_session_factory=MagicMock(),
    )
    ctx = {
        "source_id": source.id, "url": "https://patreon.com/alice",
        "artist_slug": "alice", "cookies_path": None,
        "config_overrides": {},
    }
    result, resolved = await svc._run_patreon_ingester(
        ctx, SourceConfig.from_dict({}), "tick",
    )
    assert result.success is False
    assert result.error_type == ErrorType.NOT_FOUND
    assert resolved is None
 # --- FC-3d: finalize hook updates Source health columns -------------------
@@ -455,20 +372,32 @@ async def test_finalize_skipped_preserves_failures_clears_error(db):
    assert row.last_checked_at is not None
-# --- Plan #693: backfill state machine (time-boxed chunks, run-until-done) ---
+# --- Plan #544: backfill lifecycle + PARTIAL → status=ok -------------------
-def _backfill_svc(db, db_sync, tmp_path, result):
+@pytest.mark.asyncio
-    """DownloadService wired with a real importer (so empty written_paths just
+async def test_backfill_decrements_after_run(
-    attaches nothing) whose Patreon phase-2 branch is stubbed to return `result`.
+    db, db_sync, tmp_path, seed_artist_and_source,
-
+):
-    The seeded source is Patreon, so phase 2 routes to the native ingester
+    """When backfill_runs_remaining > 0 going in, a non-clean / non-empty
-    (plan #697); the stub returns the canned result + captures the per-call
+    run decrements by 1 — operator gets N runs to complete the deep scan
-    (ctx, source_config, mode). Returns (svc, ingester_calls)."""
+    before tick mode resumes."""
    from backend.app.services.download_service import DownloadService
    from backend.app.services.importer import Importer
    _artist, source = seed_artist_and_source
    source.backfill_runs_remaining = 3
    await db.commit()
    images_root = tmp_path / "images"
    f1 = images_root / "alice" / "patreon" / "post" / "a.jpg"
    _make_jpg(f1, split="h")
    fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
        success=True, written_paths=[str(f1)], files_downloaded=1,
        stdout=f"{f1}\n",
    ))
    sync_settings = db_sync.execute(
        select(ImportSettings).where(ImportSettings.id == 1)
    ).scalar_one()
@@ -477,243 +406,95 @@ def _backfill_svc(db, db_sync, tmp_path, result):
        thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
    )
    cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
    fake_gdl = _fake_gdl_with_result(result)
    svc = DownloadService(
        async_session=db, sync_session=db_sync,
        gdl=fake_gdl, importer=importer, cred_service=cred_service,
    )
    calls = _stub_patreon_ingester(svc, result)
    return svc, calls
@pytest.mark.asyncio
 async def test_backfill_chunk_progress_advances_cursor(
    db, db_sync, tmp_path, seed_artist_and_source,
 ):
    """A running backfill chunk that didn't finish but advanced (new cursor)
    stays 'running', checkpoints the cursor, bumps the chunk counter, and
    spends one safety-cap chunk."""
    _artist, source = seed_artist_and_source
    source.config_overrides = {"_backfill_state": "running"}
    source.backfill_runs_remaining = 5
    await db.commit()
    svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
        success=False, written_paths=[], files_downloaded=0,
        stderr="[patreon][debug] Cursor: 03:PAGE2:xyz\n",
    ))
    await svc.download_source(source.id)
-    remaining, co = (await db.execute(
+    remaining = (await db.execute(
-        select(Source.backfill_runs_remaining, Source.config_overrides)
+        select(Source.backfill_runs_remaining).where(Source.id == source.id)
        .where(Source.id == source.id)
    )).one()
    assert co.get("_backfill_state") == "running"
    assert co.get("_backfill_cursor") == "03:PAGE2:xyz"
    assert co.get("_backfill_chunks") == 1
    assert remaining == 4
@pytest.mark.asyncio
 async def test_backfill_state_running_selects_backfill_mode_and_resumes(
    db, db_sync, tmp_path, seed_artist_and_source,
 ):
    """state=='running' drives backfill mode (chunk budget) and threads the
    stored cursor to the native ingester as the resume point."""
    from backend.app.services.gallery_dl import BACKFILL_CHUNK_SECONDS
    _artist, source = seed_artist_and_source
    source.config_overrides = {"_backfill_state": "running", "_backfill_cursor": "03:RESUME:here"}
    source.backfill_runs_remaining = 5
    await db.commit()
    svc, calls = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
        success=False, written_paths=[],
        stderr="[patreon][debug] Cursor: 03:RESUME2:next\n",
    ))
    await svc.download_source(source.id)
    assert len(calls) == 1
    assert calls[0]["mode"] == "backfill"
    assert calls[0]["source_config"].resume_cursor == "03:RESUME:here"
    assert calls[0]["source_config"].timeout == BACKFILL_CHUNK_SECONDS
    co = (await db.execute(
        select(Source.config_overrides).where(Source.id == source.id)
    )).scalar_one()
-    assert co.get("_backfill_cursor") == "03:RESUME2:next"
+    assert remaining == 2
@pytest.mark.asyncio
-async def test_backfill_clean_exit_marks_complete(
+async def test_backfill_auto_resets_on_clean_zero_files(
    db, db_sync, tmp_path, seed_artist_and_source,
 ):
-    """A clean rc=0 chunk = gallery-dl reached the bottom → state 'complete',
+    """A clean run (rc=0) that downloaded zero files means the backfill
-    cursor cleared, returns to tick mode."""
+    queue drained — reset to 0 immediately instead of wasting the rest of
-    _artist, source = seed_artist_and_source
+    the N-run budget on no-op walks."""
    source.config_overrides = {"_backfill_state": "running", "_backfill_cursor": "03:NEAR:bottom"}
    source.backfill_runs_remaining = 5
    await db.commit()
    svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
        success=True, written_paths=[], files_downloaded=0,
    ))
    await svc.download_source(source.id)
    remaining, co = (await db.execute(
        select(Source.backfill_runs_remaining, Source.config_overrides)
        .where(Source.id == source.id)
    )).one()
    assert co.get("_backfill_state") == "complete"
    assert "_backfill_cursor" not in co
    assert remaining == 0
@pytest.mark.asyncio
 async def test_backfill_cap_exhaustion_stalls(
    db, db_sync, tmp_path, seed_artist_and_source,
 ):
    """A progressing chunk that spends the last safety-cap chunk without
    reaching the bottom pauses as 'stalled' (doesn't loop forever)."""
    _artist, source = seed_artist_and_source
    source.config_overrides = {"_backfill_state": "running"}
    source.backfill_runs_remaining = 1
    await db.commit()
    svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
        success=False, written_paths=[],
        stderr="[patreon][debug] Cursor: 03:MORE:left\n",
    ))
    await svc.download_source(source.id)
    remaining, co = (await db.execute(
        select(Source.backfill_runs_remaining, Source.config_overrides)
        .where(Source.id == source.id)
    )).one()
    assert co.get("_backfill_state") == "stalled"
    assert remaining == 0
@pytest.mark.asyncio
 async def test_backfill_stuck_guard_stalls_after_two_no_advance(
    db, db_sync, tmp_path, seed_artist_and_source,
 ):
    """Two consecutive chunks that fail to advance the cursor → 'stalled',
    cursor cleared, so a wedged walk can't re-strand forever."""
    from unittest.mock import MagicMock
    from backend.app.services.download_service import DownloadService
    from backend.app.services.importer import Importer
    _artist, source = seed_artist_and_source
-    source.config_overrides = {"_backfill_state": "running", "_backfill_cursor": "stuck"}
+    source.backfill_runs_remaining = 3
    source.backfill_runs_remaining = 5
    await db.commit()
    fake_result = _make_fake_dl_result(
        success=True, written_paths=[], files_downloaded=0,
    )
    fake_gdl = _fake_gdl_with_result(fake_result)
    sync_settings = db_sync.execute(
        select(ImportSettings).where(ImportSettings.id == 1)
    ).scalar_one()
    importer = Importer(
        session=db_sync, images_root=tmp_path / "images",
        import_root=tmp_path / "images",
        thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
        settings=sync_settings,
    )
    cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
    svc = DownloadService(
        async_session=db, sync_session=db_sync,
-        gdl=MagicMock(), importer=MagicMock(), cred_service=MagicMock(),
+        gdl=fake_gdl, importer=importer, cred_service=cred_service,
    )
    ctx = {
        "source_id": source.id, "platform": "patreon",
        "config_overrides": {"_backfill_state": "running", "_backfill_cursor": "stuck"},
        "backfill_runs_remaining": 5,
    }
    stuck = _make_fake_dl_result(success=False, stderr="Cursor: stuck\n")
    await svc._apply_backfill_lifecycle(ctx, stuck)
    await db.commit()
    co = (await db.execute(
        select(Source.config_overrides).where(Source.id == source.id)
    )).scalar_one()
    assert co.get("_backfill_state") == "running"
    assert co.get("_backfill_cursor_stalls") == 1
    ctx["config_overrides"] = dict(co)
    await svc._apply_backfill_lifecycle(ctx, stuck)
    await db.commit()
    co2 = (await db.execute(
        select(Source.config_overrides).where(Source.id == source.id)
    )).scalar_one()
    assert co2.get("_backfill_state") == "stalled"
    assert "_backfill_cursor" not in co2
@pytest.mark.asyncio
 async def test_backfill_timeout_chunk_reclassified_to_ok(
    db, db_sync, tmp_path, seed_artist_and_source,
 ):
    """A backfill chunk that hit its time-box (TIMEOUT) but advanced is
    reclassified to PARTIAL → event status 'ok' (progress, not error), and
    consecutive_failures is not bumped."""
    from backend.app.services.gallery_dl import ErrorType
    _artist, source = seed_artist_and_source
    source.config_overrides = {"_backfill_state": "running"}
    source.backfill_runs_remaining = 5
    await db.commit()
    svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
        success=False, files_downloaded=3,
        error_type=ErrorType.TIMEOUT,
        error_message="Download timed out",
        stderr="[patreon][debug] Cursor: 03:NEXT:page\n",
    ))
    event_id = await svc.download_source(source.id)
    ev = (await db.execute(
        select(DownloadEvent).where(DownloadEvent.id == event_id)
    )).scalar_one()
    assert ev.status == "ok"
    failures = (await db.execute(
        select(Source.consecutive_failures).where(Source.id == source.id)
    )).scalar_one()
    assert failures == 0
@pytest.mark.asyncio
 async def test_tick_mode_when_not_running_leaves_state_untouched(
    db, db_sync, tmp_path, seed_artist_and_source,
 ):
    """No _backfill_state → not backfilling; the lifecycle is a no-op and the
    counter/state aren't mutated."""
    _artist, source = seed_artist_and_source
    assert (source.config_overrides or {}).get("_backfill_state") is None
    svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
        success=True, written_paths=[], files_downloaded=0,
    ))
    await svc.download_source(source.id)
-    co = (await db.execute(
+    remaining = (await db.execute(
-        select(Source.config_overrides).where(Source.id == source.id)
+        select(Source.backfill_runs_remaining).where(Source.id == source.id)
    )).scalar_one()
-    assert (co or {}).get("_backfill_state") is None
+    assert remaining == 0
@pytest.mark.asyncio
-async def test_recovery_mode_selected_and_flag_cleared_on_complete(
+async def test_tick_mode_does_not_touch_backfill_counter(
    db, db_sync, tmp_path, seed_artist_and_source,
 ):
-    """plan #697: `_backfill_bypass_seen` alongside a running backfill selects
+    """When backfill_runs_remaining is already 0, downloads don't go
-    recovery mode (ingester bypasses the seen-ledger). A clean rc=0 walk
+    negative or otherwise mutate the counter."""
-    completes the shared lifecycle AND clears the bypass flag so the next tick
+    from backend.app.services.download_service import DownloadService
-    honors the ledger again."""
+    from backend.app.services.importer import Importer
    _artist, source = seed_artist_and_source
    source.config_overrides = {"_backfill_state": "running", "_backfill_bypass_seen": True}
    source.backfill_runs_remaining = 5
    await db.commit()
-    svc, calls = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
+    _artist, source = seed_artist_and_source
    assert source.backfill_runs_remaining == 0
    fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
        success=True, written_paths=[], files_downloaded=0,
    ))
    sync_settings = db_sync.execute(
        select(ImportSettings).where(ImportSettings.id == 1)
    ).scalar_one()
    importer = Importer(
        session=db_sync, images_root=tmp_path / "images",
        import_root=tmp_path / "images",
        thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
        settings=sync_settings,
    )
    cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
    svc = DownloadService(
        async_session=db, sync_session=db_sync,
        gdl=fake_gdl, importer=importer, cred_service=cred_service,
    )
    await svc.download_source(source.id)
-    assert calls[0]["mode"] == "recovery"
+    remaining = (await db.execute(
-    co = (await db.execute(
+        select(Source.backfill_runs_remaining).where(Source.id == source.id)
        select(Source.config_overrides).where(Source.id == source.id)
    )).scalar_one()
-    assert co.get("_backfill_state") == "complete"
+    assert remaining == 0
    assert "_backfill_bypass_seen" not in co
@pytest.mark.asyncio
@@ -753,7 +534,6 @@ async def test_partial_error_type_maps_to_ok_status(
        async_session=db, sync_session=db_sync,
        gdl=fake_gdl, importer=importer, cred_service=cred_service,
    )
    _stub_patreon_ingester(svc, fake_result)
    event_id = await svc.download_source(source.id)
    ev = (await db.execute(
@@ -789,11 +569,10 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image(
    _make_jpg(f1, split="h")
    _make_jpg(f2, split="v")
-    result = _make_fake_dl_result(
+    fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
        success=True, written_paths=[str(f1), str(f2)],
        files_downloaded=2, stdout=f"{f1}\n{f2}\n",
-    )
+    ))
    fake_gdl = _fake_gdl_with_result(result)
    sync_settings = db_sync.execute(
        select(ImportSettings).where(ImportSettings.id == 1)
@@ -826,7 +605,6 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image(
        async_session=db, sync_session=db_sync,
        gdl=fake_gdl, importer=importer, cred_service=cred_service,
    )
    _stub_patreon_ingester(svc, result)
    await svc.download_source(source.id)
    # Two files attached → two thumbnail enqueues + two ML enqueues, IDs
@@ -841,11 +619,11 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image(
 async def test_releases_db_connections_before_subprocess(
    db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
 ):
-    """Phase 1's DB connections must be released BEFORE the (multi-minute)
+    """Phase 1's DB connections must be released BEFORE the (up to ~19.5-min
-    phase-2 fetch, so they don't idle-die and strand phase 3 with
+    in backfill) gallery-dl subprocess, so they don't idle-die and strand
-    ConnectionDoesNotExistError (Anduo #40014). Spy on the async session close
+    phase 3 with ConnectionDoesNotExistError (Anduo #40014). Spy on the
-    and assert it happened before the phase-2 work (here the native Patreon
+    async session close and assert it happened before gdl.download runs;
-    ingester) runs; phase 3 must still finalize the event."""
+    phase 3 must still finalize the event."""
    from backend.app.services.download_service import DownloadService
    from backend.app.services.importer import Importer
@@ -861,9 +639,19 @@ async def test_releases_db_connections_before_subprocess(
    monkeypatch.setattr(db, "close", spy_close)
    seen = {}
-    fake_gdl = _fake_gdl_with_result(
+
-        _make_fake_dl_result(success=True, written_paths=[], stdout="")
+    async def fake_download(url, **k):
-    )
+        seen["closed_before_download"] = closed["async"]
        return _make_fake_dl_result(success=True, written_paths=[], stdout="")
    fake_gdl = MagicMock()
    fake_gdl.download = fake_download
    fake_gdl._compute_run_stats = lambda *a, **k: {
        "exit_code": 0, "downloaded_count": 0, "skipped_count": 0,
        "per_item_failures": 0, "warning_count": 0, "tier_gated_count": 0,
    }
    fake_gdl._extract_errors_warnings = lambda *a, **k: ""
    fake_gdl._truncate_log = lambda x, **k: x
    sync_settings = db_sync.execute(
        select(ImportSettings).where(ImportSettings.id == 1)
@@ -880,15 +668,9 @@ async def test_releases_db_connections_before_subprocess(
        async_session=db, sync_session=db_sync,
        gdl=fake_gdl, importer=importer, cred_service=cred_service,
    )
    async def fake_ingest(ctx, source_config, mode):
        seen["closed_before_phase2"] = closed["async"]
        return _make_fake_dl_result(success=True, written_paths=[], stdout=""), None
    svc._run_patreon_ingester = fake_ingest
    event_id = await svc.download_source(source.id)
-    assert seen["closed_before_phase2"] is True
+    assert seen["closed_before_download"] is True
    ev = (await db.execute(
        select(DownloadEvent).where(DownloadEvent.id == event_id)
    )).scalar_one()
@@ -34,7 +34,7 @@ def test_timeout_ladder_keeps_subprocess_budgets_under_soft_limit():
    preempts it. soft must in turn sit below the hard SIGKILL cap."""
    from backend.app.services.gallery_dl import (
        _DEFAULT_GDL_TIMEOUT_SECONDS,
-        BACKFILL_CHUNK_SECONDS,
+        BACKFILL_TIMEOUT_SECONDS,
    )
    from backend.app.tasks.download import (
        DOWNLOAD_HARD_TIME_LIMIT,
@@ -42,7 +42,7 @@ def test_timeout_ladder_keeps_subprocess_budgets_under_soft_limit():
    )
    assert _DEFAULT_GDL_TIMEOUT_SECONDS < DOWNLOAD_SOFT_TIME_LIMIT
-    assert BACKFILL_CHUNK_SECONDS < DOWNLOAD_SOFT_TIME_LIMIT
+    assert BACKFILL_TIMEOUT_SECONDS < DOWNLOAD_SOFT_TIME_LIMIT
    assert DOWNLOAD_SOFT_TIME_LIMIT < DOWNLOAD_HARD_TIME_LIMIT
@@ -8,7 +8,6 @@ from backend.app.services.gallery_dl import (
    ErrorType,
    GalleryDLService,
    SourceConfig,
    parse_last_cursor,
 )
@@ -260,7 +259,7 @@ def test_build_config_emits_tick_skip_value(gdl):
    scans once 20 contiguous archived items are seen."""
    from backend.app.services.gallery_dl import TICK_SKIP_VALUE
    cfg = gdl._build_config_for_source(
-        platform="subscribestar",
+        platform="patreon",
        source_config=SourceConfig(),
        artist_slug="alice",
        skip_value=TICK_SKIP_VALUE,
@@ -273,7 +272,7 @@ def test_build_config_emits_backfill_skip_value(gdl):
    full post history."""
    from backend.app.services.gallery_dl import BACKFILL_SKIP_VALUE
    cfg = gdl._build_config_for_source(
-        platform="subscribestar",
+        platform="patreon",
        source_config=SourceConfig(),
        artist_slug="alice",
        skip_value=BACKFILL_SKIP_VALUE,
@@ -318,26 +317,12 @@ def test_categorize_tier_limited_wins_over_partial(gdl):
    assert etype == ErrorType.TIER_LIMITED
-# --- Cursor parsing (plan #689 / native ingester #697) ---------------------
+def test_default_config_forwards_patreon_referer_to_ytdl(gdl):
-# parse_last_cursor is retained post-cutover: the native Patreon ingester emits
+    """Operator-flagged 2026-06-01: Mux video playback restrictions reject
-# `Cursor: <token>` per page into DownloadResult.stdout and download_service
+    yt-dlp's manifest fetch when Referer/Origin don't match Patreon. The
-# checkpoints the last one. (The gallery-dl Patreon `files`/`cursor` config and
+    fix is a static `downloader.ytdl.raw-options.http_headers` block, so
-# the Mux yt-dlp Referer block were removed at the #697 cutover, along with the
+    it's enough to assert the keys land in the default config."""
-# tests that pinned them.)
+    cfg = gdl._get_default_config()
-
+    headers = cfg["downloader"]["ytdl"]["raw-options"]["http_headers"]
-
+    assert headers["Referer"] == "https://www.patreon.com/"
-def test_parse_last_cursor_returns_last_match_across_streams():
+    assert headers["Origin"] == "https://www.patreon.com"
    # One `Cursor: <token>` line per page; the last is the furthest-progressed
    # resume point.
    stderr = (
        "[patreon][debug] Cursor: 03:AAAA:bbb\n"
        "[urllib3] GET ...\n"
        "[patreon][debug] Cursor: 03:CCCC:ddd\n"
    )
    assert parse_last_cursor("", stderr) == "03:CCCC:ddd"
 def test_parse_last_cursor_scans_stdout_too_and_none_when_absent():
    assert parse_last_cursor("Cursor: 99:ZZZ", "") == "99:ZZZ"
    assert parse_last_cursor("no marker here", "still nothing") is None
    assert parse_last_cursor("", "") is None
@@ -1,339 +0,0 @@
 """PatreonClient tests — parsing only, no network.
 The fixture is loaded from disk and fed directly to the pure parsing methods
 (_transform / extract_media / _validate_response / parse_cursor_from_url); no
 live requests call is exercised here.
 """
 import json
 from pathlib import Path
 import pytest
 from backend.app.services import patreon_client as pc_mod
 from backend.app.services.patreon_client import (
    MediaItem,
    PatreonAPIError,
    PatreonAuthError,
    PatreonClient,
    PatreonDriftError,
    _retry_after_seconds,
    parse_cursor_from_url,
 )
 _FIXTURE = Path(__file__).parent / "fixtures" / "patreon_posts_page1.json"
@pytest.fixture
 def response():
    return json.loads(_FIXTURE.read_text())
@pytest.fixture
 def client():
    # No cookies file → empty session, but we never make a request in tests.
    return PatreonClient(cookies_path=None)
 def _post_by_id(response, post_id):
    for post in response["data"]:
        if post["id"] == post_id:
            return post
    raise AssertionError(f"no post {post_id} in fixture")
 def test_transform_indexes_included(client, response):
    index = client._transform(response)
    assert index[("media", "9001")]["file_name"] == "gallery-one.jpg"
    assert index[("media", "9003")]["file_name"] == "bonus-pack.zip"
    assert index[("campaign", "5555")]["name"] == "Example Creator"
    # Unknown keys are absent, not errors.
    assert ("media", "0000") not in index
 def test_extract_gallery_dedups_image_large(client, response):
    index = client._transform(response)
    post = _post_by_id(response, "1001")
    items = client.extract_media(post, index)
    # Two gallery images; the image_large cover shares filehash aaaa... with
    # gallery image 9001 → collapses to one, leaving exactly 2 items.
    assert len(items) == 2
    assert all(isinstance(m, MediaItem) for m in items)
    kinds = [m.kind for m in items]
    assert kinds == ["images", "images"]  # image_large deduped away
    filenames = {m.filename for m in items}
    assert filenames == {"gallery-one.jpg", "gallery-two.jpg"}
    hashes = {m.filehash for m in items}
    assert hashes == {"a" * 32, "b" * 32}
    assert all(m.post_id == "1001" for m in items)
 def test_extract_attachment_postfile_same_file_collapses(client, response):
    index = client._transform(response)
    post = _post_by_id(response, "1002")
    items = client.extract_media(post, index)
    # The attachment (attachments_media 9003) and the post_file are the SAME
    # zip — both carry filehash c…. attachments resolve before postfile, so the
    # postfile collapses into the attachment by filehash → exactly one survives.
    # (postfile-as-a-kind is exercised separately by the video post test.)
    assert len(items) == 1
    item = items[0]
    assert item.kind == "attachments"
    assert item.filename == "bonus-pack.zip"
    assert item.filehash == "c" * 32
 def test_extract_inline_content_img(client, response):
    index = client._transform(response)
    post = _post_by_id(response, "1003")
    items = client.extract_media(post, index)
    assert len(items) == 1
    item = items[0]
    assert item.kind == "content"
    assert item.filehash == "d" * 32
    # &amp; in the src must be unescaped to & .
    assert "&amp;" not in item.url
    assert "token=x&v=2" in item.url
 def test_extract_video_postfile(client, response):
    index = client._transform(response)
    post = _post_by_id(response, "1004")
    items = client.extract_media(post, index)
    assert len(items) == 1
    item = items[0]
    assert item.kind == "postfile"
    assert item.url.startswith("https://stream.mux.com/")
    assert item.filehash == "e" * 32
 def test_parse_cursor_from_url_extracts_cursor(response):
    next_url = response["links"]["next"]
    assert parse_cursor_from_url(next_url) == "NEXTCURSOR123"
 def test_parse_cursor_from_url_none_when_absent():
    assert parse_cursor_from_url("https://www.patreon.com/api/posts?sort=-x") is None
    assert parse_cursor_from_url(None) is None
 def test_iter_posts_yields_triples_and_paginates(client, response, monkeypatch):
    # Page 1 = the fixture (4 posts, links.next → NEXTCURSOR123); page 2 = one
    # post, no links.next → stop. Monkeypatch _fetch so no network is touched.
    page2 = {
        "data": [{"id": "1005", "type": "post", "attributes": {"title": "older"}}],
        "included": [],
        "links": {},
    }
    def fake_fetch(campaign_id, cursor):
        return page2 if cursor == "NEXTCURSOR123" else response
    monkeypatch.setattr(client, "_fetch", fake_fetch)
    seen = list(client.iter_posts("5555"))
    # 4 posts on page 1 + 1 on page 2.
    assert [p["id"] for p, _idx, _cur in seen] == ["1001", "1002", "1003", "1004", "1005"]
    # page_cursor is the cursor that FETCHED the post's page.
    cursors = {p["id"]: cur for p, _idx, cur in seen}
    assert cursors["1001"] is None and cursors["1005"] == "NEXTCURSOR123"
    # included_index is the page's flattened resources, ready for extract_media.
    page1_index = next(idx for p, idx, _cur in seen if p["id"] == "1001")
    assert page1_index[("media", "9001")]["file_name"] == "gallery-one.jpg"
 def test_missing_data_raises_drift(client):
    with pytest.raises(PatreonDriftError):
        client._validate_response({"included": []})
 def test_non_list_data_raises_drift(client):
    with pytest.raises(PatreonDriftError):
        client._validate_response({"data": {"id": "1"}})
 def test_media_missing_file_name_raises_drift(client):
    # A media resource referenced by a gallery relationship but lacking
    # file_name is drift from the contract.
    post = {
        "id": "2000",
        "type": "post",
        "attributes": {"title": "broken"},
        "relationships": {"images": {"data": [{"id": "7777", "type": "media"}]}},
    }
    index = {
        ("media", "7777"): {
            "download_url": "https://c10.patreonusercontent.com/4/orig/"
            + "f" * 32
            + "/x.jpg"
        }
    }
    with pytest.raises(PatreonDriftError):
        client.extract_media(post, index)
 def test_media_referenced_but_absent_raises_drift(client):
    post = {
        "id": "2001",
        "type": "post",
        "attributes": {"title": "dangling"},
        "relationships": {"images": {"data": [{"id": "8888", "type": "media"}]}},
    }
    with pytest.raises(PatreonDriftError):
        client.extract_media(post, {})
 # -- HTTP status classification (plan #697 step 4) -------------------------
 class _FakeResp:
    def __init__(self, status_code, *, json_data=None, raise_json=False, headers=None):
        self.status_code = status_code
        self._json_data = json_data
        self._raise_json = raise_json
        self.headers = headers or {}
    def json(self):
        if self._raise_json:
            raise ValueError("Expecting value: line 1 column 1")
        return self._json_data
 def _client_returning(monkeypatch, resp):
    client = PatreonClient(cookies_path=None)
    monkeypatch.setattr(client._session, "get", lambda *a, **k: resp)
    return client
@pytest.mark.parametrize("status", [401, 403])
 def test_fetch_auth_status_raises_auth_error(monkeypatch, status):
    client = _client_returning(monkeypatch, _FakeResp(status))
    with pytest.raises(PatreonAuthError) as ei:
        client._fetch("5555", None)
    assert ei.value.status_code == status
 def test_fetch_non_json_body_is_auth_not_drift(monkeypatch):
    # An HTML login/challenge page (non-JSON) means the session expired — auth,
    # actionable as "rotate cookies", NOT API drift.
    client = _client_returning(monkeypatch, _FakeResp(200, raise_json=True))
    with pytest.raises(PatreonAuthError):
        client._fetch("5555", None)
@pytest.mark.parametrize("status", [404, 429, 500])
 def test_fetch_other_status_raises_api_error_with_code(monkeypatch, status):
    client = _client_returning(monkeypatch, _FakeResp(status))
    with pytest.raises(PatreonAPIError) as ei:
        client._fetch("5555", None)
    assert ei.value.status_code == status
    # Not auth — the ingester maps these by status, not to auth_error.
    assert not isinstance(ei.value, PatreonAuthError)
 def test_fetch_ok_returns_payload(monkeypatch):
    client = _client_returning(monkeypatch, _FakeResp(200, json_data={"data": []}))
    assert client._fetch("5555", None) == {"data": []}
 # -- verify_auth (credential probe; (True/False/None, message) contract) ---
 def test_verify_auth_ok_when_page_authenticates(monkeypatch):
    client = _client_returning(monkeypatch, _FakeResp(200, json_data={"data": []}))
    ok, msg = client.verify_auth("5555")
    assert ok is True
    assert "valid" in msg.lower()
@pytest.mark.parametrize("status", [401, 403])
 def test_verify_auth_false_when_rejected(monkeypatch, status):
    client = _client_returning(monkeypatch, _FakeResp(status))
    ok, _msg = client.verify_auth("5555")
    assert ok is False
 def test_verify_auth_inconclusive_on_drift(monkeypatch):
    # 200 + JSON but missing top-level 'data' → drift → inconclusive, NOT a
    # credential verdict (our parser is stale, the cookie may be fine).
    client = _client_returning(monkeypatch, _FakeResp(200, json_data={"included": []}))
    ok, msg = client.verify_auth("5555")
    assert ok is None
    assert "api shape changed" in msg.lower()
 def test_verify_auth_inconclusive_on_network(monkeypatch):
    import requests
    client = PatreonClient(cookies_path=None)
    def _raise(*a, **k):
        raise requests.ConnectionError("boom")
    monkeypatch.setattr(client._session, "get", _raise)
    ok, _msg = client.verify_auth("5555")
    assert ok is None
 # -- 429 backoff + pacing (plan #703) --------------------------------------
 def test_retry_after_seconds_honors_numeric_header():
    assert _retry_after_seconds(_FakeResp(429, headers={"Retry-After": "7"}), 1) == 7.0
 def test_retry_after_seconds_exponential_without_header():
    resp = _FakeResp(429)
    assert _retry_after_seconds(resp, 1) == 2.0
    assert _retry_after_seconds(resp, 2) == 4.0
    assert _retry_after_seconds(resp, 3) == 8.0
 def test_retry_after_seconds_caps():
    assert _retry_after_seconds(_FakeResp(429), 10) == 30.0
    assert _retry_after_seconds(_FakeResp(429, headers={"Retry-After": "999"}), 1) == 30.0
 def _client_with_sequence(monkeypatch, responses):
    """A client whose session yields `responses` in order; time.sleep stubbed so
    backoff never really sleeps. Returns (client, slept_list)."""
    client = PatreonClient(cookies_path=None)
    it = iter(responses)
    monkeypatch.setattr(client._session, "get", lambda *a, **k: next(it))
    slept: list[float] = []
    monkeypatch.setattr(pc_mod.time, "sleep", lambda s: slept.append(s))
    return client, slept
 def test_fetch_retries_then_succeeds_on_429(monkeypatch):
    client, slept = _client_with_sequence(monkeypatch, [
        _FakeResp(429, headers={"Retry-After": "5"}),
        _FakeResp(200, json_data={"data": []}),
    ])
    assert client._fetch("5555", None) == {"data": []}
    assert 5.0 in slept  # backed off once before the retry
 def test_fetch_persistent_429_raises_after_retries(monkeypatch):
    client, _slept = _client_with_sequence(monkeypatch, [_FakeResp(429)] * 10)
    with pytest.raises(PatreonAPIError) as ei:
        client._fetch("5555", None)
    assert ei.value.status_code == 429
    assert not isinstance(ei.value, PatreonAuthError)
 def test_request_sleep_paces_before_fetch(monkeypatch):
    client = PatreonClient(cookies_path=None, request_sleep=1.5)
    monkeypatch.setattr(
        client._session, "get",
        lambda *a, **k: _FakeResp(200, json_data={"data": []}),
    )
    slept: list[float] = []
    monkeypatch.setattr(pc_mod.time, "sleep", lambda s: slept.append(s))
    client._fetch("5555", None)
    assert slept == [1.5]
@@ -1,72 +0,0 @@
 """Patreon API contract test (native ingester build step 4).
 The drift tripwire the plan calls for: if the field-set we SEND or the parser we
 resolve responses against drifts from what real Patreon returns, this build goes
 RED — instead of the ingester silently importing nothing in production (the exact
 failure mode the native ingester exists to escape).
 Two halves:
  - the recorded `/api/posts` fixture must still parse end-to-end (no drift), and
  - the request params must still carry every field the parser depends on (the
    fixture already has the data, so trimming the request would NOT trip the
    parse half — this half guards the request shape directly).
 Pure: no network, no DB.
 """
 import json
 from pathlib import Path
 from backend.app.services.patreon_client import MediaItem, PatreonClient
 _FIXTURE = Path(__file__).parent / "fixtures" / "patreon_posts_page1.json"
 # Pinned media total across the 4 fixture posts: 1001→2 (gallery, image_large
 # cover deduped), 1002→1 (attachment+postfile collapse), 1003→1 (inline content
 # img), 1004→1 (video postfile). A parser change that drops or doubles media
 # trips this.
 _EXPECTED_MEDIA_TOTAL = 5
 _KNOWN_KINDS = {"images", "image_large", "attachments", "postfile", "content"}
 def test_recorded_fixture_parses_end_to_end():
    response = json.loads(_FIXTURE.read_text())
    client = PatreonClient(cookies_path=None)
    client._validate_response(response)  # top-level shape must hold
    index = client._transform(response)
    total = 0
    for post in response["data"]:
        items = client.extract_media(post, index)  # must not raise drift
        for m in items:
            assert isinstance(m, MediaItem)
            assert m.url, f"empty url in post {post['id']}"
            assert m.filename, f"empty filename in post {post['id']}"
            assert m.kind in _KNOWN_KINDS, f"unknown kind {m.kind!r}"
            assert m.post_id == post["id"]
        total += len(items)
    assert total == _EXPECTED_MEDIA_TOTAL
 def test_request_field_set_carries_parser_dependencies():
    """The params we SEND must keep requesting the fields the parser resolves
    against. Trimming `fields[media]=file_name`, an `include` relationship, or a
    `fields[post]` key would make real responses parse to nothing — caught here,
    not by the fixture parse above."""
    params = PatreonClient(cookies_path=None)._params("5555", None)
    media_fields = params["fields[media]"]
    assert "file_name" in media_fields
    assert "image_urls" in media_fields or "download_url" in media_fields
    includes = params["include"]
    for rel in ("images", "attachments_media", "media"):
        assert rel in includes, f"missing include relationship {rel}"
    post_fields = params["fields[post]"]
    for field in ("content", "post_file", "image"):
        assert field in post_fields, f"missing post field {field}"
    assert params["filter[campaign_id]"] == "5555"
@@ -1,313 +0,0 @@
 """Unit tests for PatreonDownloader (build step 2b).
 No network, no real subprocess. The HTTP layer is stubbed via the `session`
 seam (a fake session whose `.get` returns a fake streaming response) and yt-dlp
 via monkeypatching `_run_ytdlp`. PURE module → no DB, so unmarked (fast lane).
 """
 from __future__ import annotations
 from pathlib import Path
 import pytest
 import requests
 from backend.app.services import patreon_downloader as pd_mod
 from backend.app.services.patreon_client import MediaItem
 from backend.app.services.patreon_downloader import (
    MediaOutcome,
    PatreonDownloader,
    _sanitize,
 )
 from backend.app.utils.sidecar import find_sidecar
 # Minimal valid PNG so the file_validator passes on the happy path.
 _PNG_HEAD = b"\x89PNG\r\n\x1a\n"
 _PNG_TAIL = b"\x00\x00\x00\x00IEND\xaeB`\x82"
 _PNG_BYTES = _PNG_HEAD + (b"\x00" * 16) + _PNG_TAIL
 class _FakeResponse:
    def __init__(self, payload: bytes, status_code: int = 200, headers=None):
        self._payload = payload
        self.status_code = status_code
        self.headers = headers or {}
    def raise_for_status(self):
        if self.status_code >= 400:
            raise requests.HTTPError(f"HTTP {self.status_code}")
        return None
    def iter_content(self, chunk_size=65536):
        for i in range(0, len(self._payload), chunk_size):
            yield self._payload[i : i + chunk_size]
 class _FakeSession:
    """Records GETs and returns canned bytes (per-URL, default _PNG_BYTES)."""
    def __init__(self, payloads: dict[str, bytes] | None = None):
        self.payloads = payloads or {}
        self.calls: list[str] = []
    def get(self, url, stream=False, timeout=None):
        self.calls.append(url)
        return _FakeResponse(self.payloads.get(url, _PNG_BYTES))
 class _SeqSession:
    """Returns the queued responses in order (for retry tests)."""
    def __init__(self, responses: list[_FakeResponse]):
        self._responses = list(responses)
        self.calls: list[str] = []
    def get(self, url, stream=False, timeout=None):
        self.calls.append(url)
        return self._responses.pop(0)
 def _post(post_id="1001", title="My Post Title", published="2026-05-01T12:00:00Z"):
    return {
        "id": post_id,
        "attributes": {
            "title": title,
            "content": "<p>body</p>",
            "published_at": published,
            "url": f"https://www.patreon.com/posts/{post_id}",
        },
    }
 def _img(name, post_id="1001", url=None):
    return MediaItem(
        url=url or f"https://cdn.patreon.com/{name}",
        filename=name,
        kind="images",
        filehash=None,
        post_id=post_id,
    )
 def _downloader(tmp_path: Path, session=None, validate=True):
    return PatreonDownloader(
        images_root=tmp_path,
        cookies_path=None,
        validate=validate,
        session=session or _FakeSession(),
    )
 def test_path_layout_two_images(tmp_path):
    dl = _downloader(tmp_path)
    items = [_img("media1.png"), _img("media2.png")]
    outcomes = dl.download_post(_post(), items, "artist-x")
    post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title"
    p1 = post_dir / "01_media1.png"
    p2 = post_dir / "02_media2.png"
    assert p1.is_file()
    assert p2.is_file()
    assert [o.status for o in outcomes] == ["downloaded", "downloaded"]
    assert outcomes[0].path == p1
    assert p1.read_bytes() == _PNG_BYTES
 def test_dir_sanitized(tmp_path):
    dl = _downloader(tmp_path)
    post = _post(title='bad/name:with*chars?')
    dl.download_post(post, [_img("a.png")], "artist-x")
    patreon_dir = tmp_path / "artist-x" / "patreon"
    children = list(patreon_dir.iterdir())
    assert len(children) == 1
    name = children[0].name
    assert "/" not in name
    assert not any(c in name for c in '<>:"\\|?*')
 def test_no_date_prefix_when_unparseable(tmp_path):
    dl = _downloader(tmp_path)
    post = _post(published="not-a-date")
    dl.download_post(post, [_img("a.png")], "artist-x")
    patreon_dir = tmp_path / "artist-x" / "patreon"
    children = [c.name for c in patreon_dir.iterdir()]
    assert children == ["1001_My Post Title"]
 def test_sidecar_written_and_findable(tmp_path):
    dl = _downloader(tmp_path)
    outcomes = dl.download_post(_post(), [_img("media1.png")], "artist-x")
    media_path = outcomes[0].path
    sidecar = find_sidecar(media_path)
    assert sidecar is not None
    assert sidecar.name == "01_media1.json"
    import json
    data = json.loads(sidecar.read_text())
    assert data["category"] == "patreon"
    assert data["id"] == "1001"
    assert data["title"] == "My Post Title"
    assert data["url"] == "https://www.patreon.com/posts/1001"
 def test_skip_seen(tmp_path):
    session = _FakeSession()
    dl = _downloader(tmp_path, session=session)
    outcomes = dl.download_post(
        _post(), [_img("media1.png")], "artist-x", is_seen=lambda m: True
    )
    assert outcomes[0].status == "skipped_seen"
    assert outcomes[0].path is None
    assert session.calls == []
    # No file written anywhere.
    assert not (tmp_path / "artist-x").exists()
 def test_skip_disk(tmp_path):
    dl = _downloader(tmp_path)
    post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title"
    post_dir.mkdir(parents=True)
    existing = post_dir / "01_media1.png"
    existing.write_bytes(b"already here")
    outcomes = dl.download_post(_post(), [_img("media1.png")], "artist-x")
    assert outcomes[0].status == "skipped_disk"
    assert outcomes[0].path == existing
    assert existing.read_bytes() == b"already here"  # untouched
 def test_video_routes_to_ytdlp(tmp_path, monkeypatch):
    session = _FakeSession()
    dl = _downloader(tmp_path, session=session)
    captured = {}
    def fake_ytdlp(url, dest, headers):
        captured["url"] = url
        captured["headers"] = headers
        out = Path(dest).with_suffix(".mp4")
        out.parent.mkdir(parents=True, exist_ok=True)
        out.write_bytes(b"video-bytes")
        return out
    monkeypatch.setattr(dl, "_run_ytdlp", fake_ytdlp)
    video = MediaItem(
        url="https://stream.mux.com/abc123.m3u8",
        filename="clip.mp4",
        kind="postfile",
        filehash=None,
        post_id="1001",
    )
    outcomes = dl.download_post(_post(), [video], "artist-x")
    assert captured["url"] == "https://stream.mux.com/abc123.m3u8"
    assert captured["headers"]["Referer"] == "https://www.patreon.com/"
    assert captured["headers"]["Origin"] == "https://www.patreon.com"
    assert session.calls == []  # NOT the plain-GET path
    assert outcomes[0].status == "downloaded"
    assert outcomes[0].path.read_bytes() == b"video-bytes"
    # Sidecar written next to the actual (remuxed) output.
    assert find_sidecar(outcomes[0].path) is not None
 def test_one_failure_isolated(tmp_path):
    class _BoomSession(_FakeSession):
        def get(self, url, stream=False, timeout=None):
            self.calls.append(url)
            if "bad" in url:
                raise RuntimeError("network boom")
            return _FakeResponse(_PNG_BYTES)
    dl = _downloader(tmp_path, session=_BoomSession())
    items = [
        _img("good.png", url="https://cdn.patreon.com/good.png"),
        _img("bad.png", url="https://cdn.patreon.com/bad.png"),
    ]
    outcomes = dl.download_post(_post(), items, "artist-x")
    assert outcomes[0].status == "downloaded"
    assert outcomes[1].status == "error"
    assert "boom" in outcomes[1].error
 def test_validation_quarantines_corrupt(tmp_path):
    # A .png whose bytes are not a valid PNG → validator fails → error +
    # quarantine move; with validate=False it would pass.
    bad = b"not a real png"
    session = _FakeSession({"https://cdn.patreon.com/corrupt.png": bad})
    dl = _downloader(tmp_path, session=session, validate=True)
    item = _img("corrupt.png", url="https://cdn.patreon.com/corrupt.png")
    outcomes = dl.download_post(_post(), [item], "artist-x")
    assert outcomes[0].status == "error"
    post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title"
    assert not (post_dir / "01_corrupt.png").exists()
    quarantine = tmp_path / "_quarantine" / "artist-x" / "patreon"
    assert quarantine.is_dir()
    assert any(quarantine.iterdir())
 def test_validation_off_keeps_bytes(tmp_path):
    bad = b"not a real png"
    session = _FakeSession({"https://cdn.patreon.com/x.png": bad})
    dl = _downloader(tmp_path, session=session, validate=False)
    item = _img("x.png", url="https://cdn.patreon.com/x.png")
    outcomes = dl.download_post(_post(), [item], "artist-x")
    assert outcomes[0].status == "downloaded"
    assert outcomes[0].path.read_bytes() == bad
 def test_sanitize_helper():
    assert _sanitize('a/b') == "a_b"
    assert _sanitize('x<>:"|?*y') == "x_______y"
    assert _sanitize("trail.  ") == "trail"
    assert _sanitize("") == "_"
 def test_media_outcome_shape():
    o = MediaOutcome(media=None, status="downloaded", path=Path("/tmp/x"), error=None)
    assert o.status == "downloaded"
    assert o.path == Path("/tmp/x")
 # -- pacing + 429 backoff (plan #703) --------------------------------------
 def test_rate_limit_paces_each_real_download(tmp_path, monkeypatch):
    slept: list[float] = []
    monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s))
    dl = PatreonDownloader(
        images_root=tmp_path, cookies_path=None, validate=False,
        rate_limit=2.0, session=_FakeSession(),
    )
    dl.download_post(_post(), [_img("a.png"), _img("b.png")], "artist-x")
    # One pacing sleep per actual download (two media downloaded).
    assert slept == [2.0, 2.0]
 def test_skips_do_not_pace(tmp_path, monkeypatch):
    slept: list[float] = []
    monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s))
    dl = PatreonDownloader(
        images_root=tmp_path, cookies_path=None, validate=False,
        rate_limit=2.0, session=_FakeSession(),
    )
    # is_seen → skipped_seen, never reaches the download/pacing path.
    dl.download_post(_post(), [_img("a.png")], "artist-x", is_seen=lambda m: True)
    assert slept == []
 def test_media_429_retried_then_succeeds(tmp_path, monkeypatch):
    slept: list[float] = []
    monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s))
    session = _SeqSession([
        _FakeResponse(b"", status_code=429, headers={"Retry-After": "4"}),
        _FakeResponse(_PNG_BYTES, status_code=200),
    ])
    dl = PatreonDownloader(
        images_root=tmp_path, cookies_path=None, validate=False, session=session,
    )
    outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
    assert outcomes[0].status == "downloaded"
    assert 4.0 in slept  # backed off before the retry
    assert len(session.calls) == 2
@@ -1,401 +0,0 @@
 """PatreonIngester tests (native ingester build step 3).
 The HTTP client and the media downloader are stubbed (injected seams), so these
 exercise the ingester's own logic — mode behavior, the two-tier skip, cursor
 emission, the budget cutoff, and the Postgres seen-ledger — without network or a
 real CDN. The ledger is real (a sync sessionmaker bound to the test engine), so
 the tier-1 skip and the idempotent mark-seen run against actual rows.
 """
 import pytest
 from sqlalchemy import func, select
 from sqlalchemy.orm import sessionmaker
 from backend.app.models import Artist, PatreonSeenMedia, Source
 from backend.app.services.gallery_dl import ErrorType
 from backend.app.services.patreon_client import (
    MediaItem,
    PatreonAPIError,
    PatreonAuthError,
    PatreonDriftError,
 )
 from backend.app.services.patreon_downloader import MediaOutcome
 from backend.app.services.patreon_ingester import PatreonIngester, _ledger_key
 pytestmark = pytest.mark.integration
 # --- fakes ----------------------------------------------------------------
 def _media(post_id, n, *, filehash=None, kind="images"):
    # Deterministic 32-hex-ish key, unique per (post_id, n).
    fh = filehash if filehash is not None else f"{post_id}{n}".encode().hex().ljust(32, "0")[:32]
    return MediaItem(
        url=f"https://cdn.patreon.com/{fh}/{n}.jpg",
        filename=f"{n}.jpg",
        kind=kind,
        filehash=fh,
        post_id=post_id,
    )
 class _FakeClient:
    """Stub PatreonClient. `pages` is a list of (page_cursor, [posts]); each post
    is (post_id, [MediaItem]). `raise_on_first` lets a test trip drift."""
    def __init__(self, pages, raise_exc=None):
        self._pages = pages
        self._raise_exc = raise_exc
        self.consumed_posts = 0
    def iter_posts(self, campaign_id, cursor=None):
        if self._raise_exc is not None:
            raise self._raise_exc
        for page_cursor, posts in self._pages:
            for post_id, media in posts:
                self.consumed_posts += 1
                yield {"id": post_id, "_media": media}, {}, page_cursor
    def extract_media(self, post, included_index):
        return post["_media"]
 class _FakeDownloader:
    """Stub PatreonDownloader. Honors the injected is_seen (tier-1); media in
    `on_disk` report skipped_disk (tier-2); everything else downloads."""
    def __init__(self, tmp_path, on_disk=None):
        self.tmp_path = tmp_path
        self.on_disk = set(on_disk or ())
        self.download_calls = 0
    def download_post(self, post, media_items, artist_slug, *, is_seen):
        outcomes = []
        for m in media_items:
            if is_seen(m):
                outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
            elif _ledger_key(m) in self.on_disk:
                p = self.tmp_path / f"{m.post_id}_{m.filename}"
                outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None))
            else:
                self.download_calls += 1
                p = self.tmp_path / f"{m.post_id}_{m.filename}"
                p.write_bytes(b"x")
                outcomes.append(MediaOutcome(media=m, status="downloaded", path=p, error=None))
        return outcomes
@pytest.fixture
 async def source_id(db):
    artist = Artist(name="Ingest", slug="ingest")
    db.add(artist)
    await db.flush()
    source = Source(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/ingest", enabled=True, config_overrides={},
    )
    db.add(source)
    await db.commit()
    return source.id
 def _ingester(sync_engine, tmp_path, client, downloader):
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    return PatreonIngester(
        images_root=tmp_path, cookies_path=None, session_factory=factory,
        client=client, downloader=downloader,
    )
 def _count_ledger(sync_engine, source_id):
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    with factory() as s:
        return s.execute(
            select(func.count(PatreonSeenMedia.id)).where(
                PatreonSeenMedia.source_id == source_id
            )
        ).scalar_one()
 # --- tick -----------------------------------------------------------------
@pytest.mark.asyncio
 async def test_tick_downloads_unseen_and_marks_seen(source_id, sync_engine, tmp_path):
    m1, m2 = _media("p1", 1), _media("p1", 2)
    client = _FakeClient([(None, [("p1", [m1, m2])])])
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)
    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick",
    )
    assert result.success is True
    assert result.return_code == 0
    assert result.files_downloaded == 2
    assert len(result.written_paths) == 2
    assert _count_ledger(sync_engine, source_id) == 2
@pytest.mark.asyncio
 async def test_tick_skips_seen_via_ledger(source_id, sync_engine, tmp_path):
    m1 = _media("p1", 1)
    # Pre-seed the ledger with m1's key.
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    with factory() as s:
        s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
        s.commit()
    client = _FakeClient([(None, [("p1", [m1])])])
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)
    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick",
    )
    assert result.files_downloaded == 0
    assert downloader.download_calls == 0
@pytest.mark.asyncio
 async def test_tick_early_out_after_threshold(source_id, sync_engine, tmp_path):
    # Three posts, one already-seen media each; threshold 2 → stop before the 3rd.
    seen_media = [_media(f"p{i}", 1) for i in range(1, 4)]
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    with factory() as s:
        for m in seen_media:
            s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m), post_id=m.post_id))
        s.commit()
    pages = [(None, [(m.post_id, [m]) for m in seen_media])]
    client = _FakeClient(pages)
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)
    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick", seen_threshold=2,
    )
    assert result.success is True
    # Stopped after the 2nd contiguous seen item — never consumed the 3rd post.
    assert client.consumed_posts == 2
 # --- backfill -------------------------------------------------------------
@pytest.mark.asyncio
 async def test_backfill_reaches_bottom_signals_complete(source_id, sync_engine, tmp_path):
    pages = [
        (None, [("p1", [_media("p1", 1)])]),
        ("CUR2", [("p2", [_media("p2", 1)])]),
    ]
    client = _FakeClient(pages)
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)
    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="backfill",
    )
    # rc 0 + error_type None is what _apply_backfill_lifecycle reads as "complete".
    assert result.return_code == 0
    assert result.error_type is None
    assert result.files_downloaded == 2
    # The page-2 cursor was emitted for checkpointing.
    assert "Cursor: CUR2" in result.stdout
@pytest.mark.asyncio
 async def test_backfill_budget_cut_returns_partial_with_progress(
    source_id, sync_engine, tmp_path, monkeypatch,
 ):
    # Deterministic clock: start=0, post1 check=10 (ok), post2 check=200 (>budget).
    import backend.app.services.patreon_ingester as mod
    ticks = iter([0.0, 10.0, 200.0, 250.0])
    last = [0.0]
    def fake_monotonic():
        try:
            last[0] = next(ticks)
        except StopIteration:
            pass
        return last[0]
    monkeypatch.setattr(mod.time, "monotonic", fake_monotonic)
    pages = [
        ("CUR1", [("p1", [_media("p1", 1)])]),
        ("CUR2", [("p2", [_media("p2", 1)])]),
    ]
    client = _FakeClient(pages)
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)
    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="backfill",
        resume_cursor=None, time_budget_seconds=100.0,
    )
    assert result.error_type == ErrorType.PARTIAL
    assert result.return_code == -1
    assert result.files_downloaded == 1  # post1 downloaded before the cut
    assert downloader.download_calls == 1  # post2's body never ran
    assert "Cursor: CUR1" in result.stdout
 # --- recovery -------------------------------------------------------------
@pytest.mark.asyncio
 async def test_recovery_bypasses_seen_ledger(source_id, sync_engine, tmp_path):
    m1 = _media("p1", 1)
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    with factory() as s:
        s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
        s.commit()
    client = _FakeClient([(None, [("p1", [m1])])])
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)
    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="recovery",
    )
    # Ledger says seen, but recovery bypasses tier-1 → re-downloaded for a fresh
    # pHash evaluation under the current threshold.
    assert result.files_downloaded == 1
    assert downloader.download_calls == 1
@pytest.mark.asyncio
 async def test_recovery_tier2_disk_still_skips(source_id, sync_engine, tmp_path):
    m1 = _media("p1", 1)
    client = _FakeClient([(None, [("p1", [m1])])])
    # File still on disk (a kept image) → tier-2 spares it even under recovery.
    downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
    ing = _ingester(sync_engine, tmp_path, client, downloader)
    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="recovery",
    )
    assert result.files_downloaded == 0
    assert downloader.download_calls == 0
    # Disk-skip still reconciles the ledger so a later tick skips at tier-1.
    assert _count_ledger(sync_engine, source_id) == 1
 # --- ledger + drift -------------------------------------------------------
@pytest.mark.asyncio
 async def test_mark_seen_is_idempotent(source_id, sync_engine, tmp_path):
    m1 = _media("p1", 1)
    client = _FakeClient([(None, [("p1", [m1])])])
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)
    # Same media downloaded twice across two runs — the second marks seen again.
    ing._mark_seen(source_id, [(_ledger_key(m1), "p1")])
    ing._mark_seen(source_id, [(_ledger_key(m1), "p1")])
    assert _count_ledger(sync_engine, source_id) == 1
 def _run_with_client_error(source_id, sync_engine, tmp_path, exc):
    client = _FakeClient([], raise_exc=exc)
    ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path))
    return ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick",
    )
@pytest.mark.asyncio
 async def test_drift_maps_to_api_drift_and_fails_loud(source_id, sync_engine, tmp_path):
    result = _run_with_client_error(
        source_id, sync_engine, tmp_path,
        PatreonDriftError("missing top-level 'data' key"),
    )
    assert result.success is False
    assert result.error_type == ErrorType.API_DRIFT
    assert "Patreon API changed" in (result.error_message or "")
@pytest.mark.asyncio
 async def test_auth_error_maps_to_auth_error(source_id, sync_engine, tmp_path):
    result = _run_with_client_error(
        source_id, sync_engine, tmp_path,
        PatreonAuthError("HTTP 403 — auth rejected", status_code=403),
    )
    assert result.success is False
    assert result.error_type == ErrorType.AUTH_ERROR
@pytest.mark.asyncio
 async def test_rate_limit_status_maps_to_rate_limited(source_id, sync_engine, tmp_path):
    result = _run_with_client_error(
        source_id, sync_engine, tmp_path,
        PatreonAPIError("HTTP 429", status_code=429),
    )
    assert result.error_type == ErrorType.RATE_LIMITED
@pytest.mark.asyncio
 async def test_not_found_status_maps_to_not_found(source_id, sync_engine, tmp_path):
    result = _run_with_client_error(
        source_id, sync_engine, tmp_path,
        PatreonAPIError("HTTP 404", status_code=404),
    )
    assert result.error_type == ErrorType.NOT_FOUND
@pytest.mark.asyncio
 async def test_transport_error_maps_to_network_error(source_id, sync_engine, tmp_path):
    result = _run_with_client_error(
        source_id, sync_engine, tmp_path,
        PatreonAPIError("connection reset"),  # no status_code → transport
    )
    assert result.error_type == ErrorType.NETWORK_ERROR
 def test_ledger_key_video_has_no_filehash():
    video = MediaItem(
        url="https://stream.mux.com/abc.m3u8", filename="clip.mp4",
        kind="postfile", filehash=None, post_id="p9",
    )
    assert _ledger_key(video) == "p9:clip.mp4"
 # --- verify_patreon_credential (the verify counterpart, plan #697) ---------
@pytest.mark.asyncio
 async def test_verify_patreon_credential_unresolvable_is_inconclusive():
    from backend.app.services.patreon_ingester import verify_patreon_credential
    # No campaign id resolvable from a non-Patreon URL → can't test → None.
    ok, msg = await verify_patreon_credential("not-a-patreon-url", None, {})
    assert ok is None
    assert "campaign id" in msg.lower()
@pytest.mark.asyncio
 async def test_verify_patreon_credential_delegates_to_client(monkeypatch):
    import backend.app.services.patreon_ingester as mod
    async def _fake_resolve(url, cookies_path, overrides):
        return "123", None
    monkeypatch.setattr(mod, "resolve_campaign_id_for_source", _fake_resolve)
    monkeypatch.setattr(
        mod.PatreonClient, "verify_auth",
        lambda self, campaign_id: (True, f"ok:{campaign_id}"),
    )
    ok, msg = await mod.verify_patreon_credential("https://patreon.com/x", None, {})
    assert ok is True
    assert msg == "ok:123"
@@ -2,10 +2,7 @@ from unittest.mock import MagicMock, patch
 import pytest
-from backend.app.services.patreon_resolver import (
+from backend.app.services.patreon_resolver import resolve_campaign_id
    resolve_campaign_id,
    resolve_campaign_id_for_source,
 )
@pytest.mark.asyncio
@@ -89,48 +86,3 @@ async def test_loads_cookies_file_when_present(tmp_path):
    assert result == "9"
    _, kwargs = mock_get.call_args
    assert kwargs.get("cookies") is not None
 # -- resolve_campaign_id_for_source (shared by download + verify) ----------
@pytest.mark.asyncio
 async def test_for_source_uses_cached_override_without_lookup():
    with patch("backend.app.services.patreon_resolver.requests.get") as mock_get:
        cid, resolved = await resolve_campaign_id_for_source(
            "https://patreon.com/alice", None, {"patreon_campaign_id": "999"},
        )
    assert cid == "999"
    assert resolved is None  # cached → no newly-resolved id to cache
    mock_get.assert_not_called()
@pytest.mark.asyncio
 async def test_for_source_extracts_id_url_without_lookup():
    with patch("backend.app.services.patreon_resolver.requests.get") as mock_get:
        cid, resolved = await resolve_campaign_id_for_source(
            "https://www.patreon.com/id:4242", None, {},
        )
    assert cid == "4242"
    assert resolved is None
    mock_get.assert_not_called()
@pytest.mark.asyncio
 async def test_for_source_resolves_vanity_and_reports_it():
    fake = MagicMock()
    fake.status_code = 200
    fake.json.return_value = {"data": [{"id": "777", "type": "campaign"}]}
    with patch("backend.app.services.patreon_resolver.requests.get", return_value=fake):
        cid, resolved = await resolve_campaign_id_for_source(
            "https://patreon.com/alice", None, {},
        )
    assert cid == "777"
    assert resolved == "777"  # newly resolved → caller caches it
@pytest.mark.asyncio
 async def test_for_source_unresolvable_returns_none():
    cid, resolved = await resolve_campaign_id_for_source("not-a-patreon-url", None, {})
    assert cid is None
    assert resolved is None
@@ -1,73 +0,0 @@
 import pytest
 from sqlalchemy import func, select
 from sqlalchemy.exc import IntegrityError
 from backend.app.models import Artist, PatreonSeenMedia, Source
 pytestmark = pytest.mark.integration
 async def _source(db):
    artist = Artist(name="Seen Ledger Artist", slug="seen-ledger-artist")
    db.add(artist)
    await db.flush()
    source = Source(
        artist_id=artist.id,
        platform="patreon",
        url="https://www.patreon.com/seenledger",
    )
    db.add(source)
    await db.flush()
    return source.id
@pytest.mark.asyncio
 async def test_patreon_seen_media_dedup(db):
    source_id = await _source(db)
    db.add(
        PatreonSeenMedia(
            source_id=source_id,
            filehash="0123456789abcdef0123456789abcdef",
            post_id="123456",
        )
    )
    await db.flush()
    seen_at = await db.scalar(
        select(PatreonSeenMedia.seen_at).where(
            PatreonSeenMedia.source_id == source_id
        )
    )
    assert seen_at is not None
    # A second sighting of the same (source_id, filehash) is rejected by the
    # unique constraint. Wrap in a savepoint so the IntegrityError doesn't
    # poison the outer transaction.
    with pytest.raises(IntegrityError):
        async with db.begin_nested():
            db.add(
                PatreonSeenMedia(
                    source_id=source_id,
                    filehash="0123456789abcdef0123456789abcdef",
                    post_id="654321",
                )
            )
            await db.flush()
    # Same filehash but a different post_id sentinel still inserts: the
    # dedup axis is (source_id, filehash), not post_id.
    db.add(
        PatreonSeenMedia(
            source_id=source_id,
            filehash="video:123456:7890",
            post_id="123456",
        )
    )
    await db.flush()
    count = await db.scalar(
        select(func.count(PatreonSeenMedia.id)).where(
            PatreonSeenMedia.source_id == source_id
        )
    )
    assert count == 2
@@ -175,107 +175,58 @@ async def test_list_hides_sidecar_synthetic_anchors(db):
@pytest.mark.asyncio
-async def test_start_backfill_arms_run_until_done(db):
+async def test_set_backfill_runs_arms_source(db):
-    """start_backfill sets state=running + the chunk cap and clears any prior
+    """The service method overrides backfill_runs_remaining (regardless of
-    cursor/chunk state, returning the updated record for the API to echo."""
+    the auto-arm-on-create starting value) and returns the updated record
-    from backend.app.services.source_service import BACKFILL_MAX_CHUNKS
+    so the API can echo it back."""
    artist = await _artist(db, "Alice")
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/alice",
    )
-    # Simulate a prior, finished walk leaving stale checkpoint state.
+    updated = await svc.set_backfill_runs(rec.id, 5)
-    await db.execute(
+    assert updated.backfill_runs_remaining == 5
        Source.__table__.update().where(Source.id == rec.id).values(
            config_overrides={"_backfill_state": "complete", "_backfill_cursor": "old",
                              "_backfill_chunks": 7},
        )
    )
    await db.commit()
-    updated = await svc.start_backfill(rec.id)
+    db_value = (await db.execute(
-    assert updated.backfill_state == "running"
+        select(Source.backfill_runs_remaining).where(Source.id == rec.id)
    assert updated.backfill_chunks == 0
    assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS
    co = (await db.execute(
        select(Source.config_overrides).where(Source.id == rec.id)
    )).scalar_one()
-    assert co.get("_backfill_state") == "running"
+    assert db_value == 5
    assert "_backfill_cursor" not in co
@pytest.mark.asyncio
-async def test_stop_backfill_returns_to_idle(db):
+async def test_set_backfill_runs_rejects_out_of_range(db):
    artist = await _artist(db, "Alice")
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/alice",
    )
-    await svc.start_backfill(rec.id)
+    with pytest.raises(ValueError):
-    updated = await svc.stop_backfill(rec.id)
+        await svc.set_backfill_runs(rec.id, 0)
-    assert updated.backfill_state is None
+    with pytest.raises(ValueError):
-    assert updated.backfill_runs_remaining == 0
+        await svc.set_backfill_runs(rec.id, 11)
    co = (await db.execute(
        select(Source.config_overrides).where(Source.id == rec.id)
    )).scalar_one()
    assert "_backfill_state" not in (co or {})
@pytest.mark.asyncio
-async def test_start_recovery_arms_bypass_flag(db):
+async def test_set_backfill_runs_raises_when_source_missing(db):
    """Plan #697: start_recovery arms the backfill state machine PLUS the
    _backfill_bypass_seen flag (recovery), surfaced on the record so the UI badge
    can label it 'Recovering'. Clears any prior checkpoint state."""
    from backend.app.services.source_service import BACKFILL_MAX_CHUNKS
    artist = await _artist(db, "Alice")
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/alice",
    )
    updated = await svc.start_recovery(rec.id)
    assert updated.backfill_state == "running"
    assert updated.backfill_bypass_seen is True
    assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS
    co = (await db.execute(
        select(Source.config_overrides).where(Source.id == rec.id)
    )).scalar_one()
    assert co.get("_backfill_bypass_seen") is True
    # Stop clears the bypass flag too (shared lifecycle).
    stopped = await svc.stop_backfill(rec.id)
    assert stopped.backfill_bypass_seen is False
    co2 = (await db.execute(
        select(Source.config_overrides).where(Source.id == rec.id)
    )).scalar_one()
    assert "_backfill_bypass_seen" not in (co2 or {})
@pytest.mark.asyncio
 async def test_start_backfill_raises_when_source_missing(db):
    svc = SourceService(db)
    with pytest.raises(LookupError):
-        await svc.start_backfill(99999)
+        await svc.set_backfill_runs(99999, 3)
@pytest.mark.asyncio
 async def test_new_enabled_source_starts_in_backfill_mode(db):
-    """Plan #693: freshly added enabled sources have no archive yet, so they
+    """Plan #544 follow-up: freshly added enabled sources have no archive
-    arm run-until-done backfill — state 'running' — to walk the full history
+    yet, so the first few polls would blow the wall-clock cap in tick
-    on the first ticks instead of blowing the wall-clock cap in tick mode."""
+    mode. Pre-arm backfill so the initial walks use the longer timeout."""
    artist = await _artist(db, "Alice")
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/alice-new",
    )
-    assert rec.backfill_state == "running"
+    assert rec.backfill_runs_remaining == 3
@pytest.mark.asyncio