feat(download): cursor-paged Patreon backfill for large catalogs

Large Patreon creators (Anduo: weekly 50-120-image Reports back months =
thousands of files) couldn't backfill: each run re-walked newest→oldest
from the top, and gallery-dl's polite ~0.75s/request HEAD walk alone
exceeded the 1170s subprocess budget, so the run died during enumeration
with 0 files written and NO forward progress — re-stranding every time
(event #40411).

Checkpoint gallery-dl's pagination cursor so each backfill window advances
the frontier:

- gallery_dl.py: SourceConfig.resume_cursor; _build_config_for_source sets
  extractor.patreon.cursor=<resume> (PLATFORM_DEFAULTS leave log-only True
  for a fresh run); parse_last_cursor() pulls the last emitted
  'Cursor: <token>' from stdout+stderr — survives a timed-out run since the
  TimeoutExpired path returns partial output.
- download_service.py: phase2 stays in BACKFILL mode while a cursor is
  pending (even after the run budget drains) and threads resume_cursor;
  _apply_backfill_lifecycle() checkpoints the advancing cursor each
  non-completing run, completes on a clean rc=0 finish (walk reached
  bottom), and a stuck-guard clears the cursor after 2 non-advancing runs
  so a wedged walk can't re-strand forever.

patreon-only (sole platform with a resumable cursor); other platforms keep
the simple counter semantics. Cursor state lives in config_overrides JSON
(patreon_campaign_id precedent) — no migration. Time-budget ladder
(1170/1350/1500) unchanged.

Plan #689.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/test_download_service.py

@@ -376,12 +376,13 @@ async def test_finalize_skipped_preserves_failures_clears_error(db):
 
 
 @pytest.mark.asyncio
-async def test_backfill_decrements_after_run(
+async def test_backfill_decrements_and_checkpoints_cursor(
     db, db_sync, tmp_path, seed_artist_and_source,
 ):
-    """When backfill_runs_remaining > 0 going in, a non-clean / non-empty
-    run decrements by 1 — operator gets N runs to complete the deep scan
-    before tick mode resumes."""
+    """A backfill run that did NOT finish the walk (subprocess budget cut it
+    short → success=False / rc!=0) decrements the run budget AND checkpoints
+    gallery-dl's last-emitted cursor, so the next run resumes instead of
+    re-walking from the top (plan #689, Anduo #40411)."""
     from backend.app.services.download_service import DownloadService
     from backend.app.services.importer import Importer
 
@@ -390,12 +391,9 @@ async def test_backfill_decrements_after_run(
     await db.commit()
 
     images_root = tmp_path / "images"
-    f1 = images_root / "alice" / "patreon" / "post" / "a.jpg"
-    _make_jpg(f1, split="h")
-
     fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
-        success=True, written_paths=[str(f1)], files_downloaded=1,
-        stdout=f"{f1}\n",
+        success=False, written_paths=[], files_downloaded=0,
+        stderr="[patreon][debug] Cursor: 03:PAGE2:xyz\n",
     ))
 
     sync_settings = db_sync.execute(
@@ -412,10 +410,148 @@ async def test_backfill_decrements_after_run(
     )
     await svc.download_source(source.id)
 
-    remaining = (await db.execute(
-        select(Source.backfill_runs_remaining).where(Source.id == source.id)
-    )).scalar_one()
+    remaining, overrides = (await db.execute(
+        select(Source.backfill_runs_remaining, Source.config_overrides)
+        .where(Source.id == source.id)
+    )).one()
     assert remaining == 2
+    assert overrides.get("_backfill_cursor") == "03:PAGE2:xyz"
+
+
+@pytest.mark.asyncio
+async def test_backfill_cursor_keeps_backfill_mode_after_budget_drained(
+    db, db_sync, tmp_path, seed_artist_and_source,
+):
+    """A pending cursor with runs_remaining == 0 still runs in BACKFILL mode
+    (skip:True + the resume cursor threaded to gallery-dl), so a large walk
+    finishes across ticks without the operator re-triggering."""
+    from backend.app.services.download_service import DownloadService
+    from backend.app.services.gallery_dl import BACKFILL_SKIP_VALUE
+    from backend.app.services.importer import Importer
+
+    _artist, source = seed_artist_and_source
+    source.backfill_runs_remaining = 0
+    source.config_overrides = {"_backfill_cursor": "03:RESUME:here"}
+    await db.commit()
+
+    fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
+        success=False, written_paths=[],
+        stderr="[patreon][debug] Cursor: 03:RESUME2:next\n",
+    ))
+    sync_settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    importer = Importer(
+        session=db_sync, images_root=tmp_path / "images",
+        import_root=tmp_path / "images",
+        thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
+        settings=sync_settings,
+    )
+    cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
+    svc = DownloadService(
+        async_session=db, sync_session=db_sync,
+        gdl=fake_gdl, importer=importer, cred_service=cred_service,
+    )
+    await svc.download_source(source.id)
+
+    kwargs = fake_gdl.download.call_args.kwargs
+    assert kwargs["skip_value"] is BACKFILL_SKIP_VALUE
+    assert kwargs["source_config"].resume_cursor == "03:RESUME:here"
+
+    overrides = (await db.execute(
+        select(Source.config_overrides).where(Source.id == source.id)
+    )).scalar_one()
+    assert overrides.get("_backfill_cursor") == "03:RESUME2:next"
+
+
+@pytest.mark.asyncio
+async def test_backfill_completes_and_clears_cursor(
+    db, db_sync, tmp_path, seed_artist_and_source,
+):
+    """A clean rc=0 finish means gallery-dl walked to the bottom — clear the
+    checkpoint and drop back to tick mode, regardless of files this run."""
+    from backend.app.services.download_service import DownloadService
+    from backend.app.services.importer import Importer
+
+    _artist, source = seed_artist_and_source
+    source.backfill_runs_remaining = 2
+    source.config_overrides = {"_backfill_cursor": "03:NEAR:bottom"}
+    await db.commit()
+
+    fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
+        success=True, written_paths=[], files_downloaded=0,
+        stderr="[patreon][debug] Cursor: 03:LAST:page\n",
+    ))
+    sync_settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    importer = Importer(
+        session=db_sync, images_root=tmp_path / "images",
+        import_root=tmp_path / "images",
+        thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
+        settings=sync_settings,
+    )
+    cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
+    svc = DownloadService(
+        async_session=db, sync_session=db_sync,
+        gdl=fake_gdl, importer=importer, cred_service=cred_service,
+    )
+    await svc.download_source(source.id)
+
+    remaining, overrides = (await db.execute(
+        select(Source.backfill_runs_remaining, Source.config_overrides)
+        .where(Source.id == source.id)
+    )).one()
+    assert remaining == 0
+    assert "_backfill_cursor" not in (overrides or {})
+
+
+@pytest.mark.asyncio
+async def test_backfill_cursor_stuck_guard_gives_up(
+    db, db_sync, tmp_path, seed_artist_and_source,
+):
+    """If two consecutive non-completing runs fail to advance the cursor,
+    drop the checkpoint + zero the budget so a wedged walk can't re-strand
+    forever (cf. download soft-limit ladder)."""
+    from unittest.mock import MagicMock
+
+    from backend.app.services.download_service import DownloadService
+
+    _artist, source = seed_artist_and_source
+    source.backfill_runs_remaining = 0
+    source.config_overrides = {"_backfill_cursor": "stuck"}
+    await db.commit()
+
+    svc = DownloadService(
+        async_session=db, sync_session=db_sync,
+        gdl=MagicMock(), importer=MagicMock(), cred_service=MagicMock(),
+    )
+    ctx = {
+        "source_id": source.id, "platform": "patreon",
+        "config_overrides": {"_backfill_cursor": "stuck"},
+        "backfill_runs_remaining": 0,
+    }
+    stuck = _make_fake_dl_result(success=False, stderr="Cursor: stuck\n")
+
+    # Run 1: cursor unchanged → 1 stall, checkpoint retained.
+    await svc._apply_backfill_lifecycle(ctx, stuck)
+    await db.commit()
+    overrides = (await db.execute(
+        select(Source.config_overrides).where(Source.id == source.id)
+    )).scalar_one()
+    assert overrides.get("_backfill_cursor") == "stuck"
+    assert overrides.get("_backfill_cursor_stalls") == 1
+
+    # Run 2: still stuck → give up.
+    ctx["config_overrides"] = dict(overrides)
+    await svc._apply_backfill_lifecycle(ctx, stuck)
+    await db.commit()
+    remaining, overrides2 = (await db.execute(
+        select(Source.backfill_runs_remaining, Source.config_overrides)
+        .where(Source.id == source.id)
+    )).one()
+    assert remaining == 0
+    assert "_backfill_cursor" not in (overrides2 or {})
 
 
 @pytest.mark.asyncio

tests/test_gallery_dl_service.py

@@ -8,6 +8,7 @@ from backend.app.services.gallery_dl import (
     ErrorType,
     GalleryDLService,
     SourceConfig,
+    parse_last_cursor,
 )
 
 
@@ -326,3 +327,38 @@ def test_default_config_forwards_patreon_referer_to_ytdl(gdl):
     headers = cfg["downloader"]["ytdl"]["raw-options"]["http_headers"]
     assert headers["Referer"] == "https://www.patreon.com/"
     assert headers["Origin"] == "https://www.patreon.com"
+
+
+# --- Cursor-paged backfill (plan #689) -------------------------------------
+
+
+def test_parse_last_cursor_returns_last_match_across_streams():
+    # gallery-dl logs `Cursor: <token>` per page (debug → stderr); the last
+    # is the furthest-progressed resume point.
+    stderr = (
+        "[patreon][debug] Cursor: 03:AAAA:bbb\n"
+        "[urllib3] GET ...\n"
+        "[patreon][debug] Cursor: 03:CCCC:ddd\n"
+    )
+    assert parse_last_cursor("", stderr) == "03:CCCC:ddd"
+
+
+def test_parse_last_cursor_scans_stdout_too_and_none_when_absent():
+    assert parse_last_cursor("Cursor: 99:ZZZ", "") == "99:ZZZ"
+    assert parse_last_cursor("no marker here", "still nothing") is None
+    assert parse_last_cursor("", "") is None
+
+
+def test_build_config_patreon_resume_cursor_overrides_default(gdl):
+    # No resume cursor → PLATFORM_DEFAULTS leaves the log-only `cursor: True`.
+    fresh = gdl._build_config_for_source(
+        "patreon", SourceConfig(), "alice", skip_value=True,
+    )
+    assert fresh["extractor"]["patreon"]["cursor"] is True
+
+    # Resume cursor set → gallery-dl restarts the walk from that page.
+    resumed = gdl._build_config_for_source(
+        "patreon", SourceConfig(resume_cursor="03:CCCC:ddd"), "alice",
+        skip_value=True,
+    )
+    assert resumed["extractor"]["patreon"]["cursor"] == "03:CCCC:ddd"