feat(download): tick/backfill modes + partial-success classifier (plan #544)

Routine subscription polls walked the entire post history every tick
even when nothing had changed, because gallery-dl's default `skip: True`
continues iterating archived posts. A creator with ~550 archived posts
(Knuxy patreon) saturates the 870s wall-clock cap before completing,
even with zero downloads needed. Plus, a tier-limited run that
downloaded hundreds of files but ran out the clock should be a
warning, not an error.

Two coupled changes, both operator-flagged 2026-06-01:

* **Tick mode (default, cron polls).** New `TICK_SKIP_VALUE = "exit:20"`
  asks gallery-dl to exit after 20 contiguous archived items. Fresh
  subscriptions + new-content cases still walk normally; established
  subscription with zero new content exits in ~30s of HEAD requests
  instead of pegging the timeout. 20 (not 5) gives headroom against
  paywall warnings interleaving with archived items.
* **Backfill mode (explicit, operator-triggered).** Sticky for N runs
  via new `Source.backfill_runs_remaining` (alembic 0031). While > 0,
  downloads use `skip: True` + 1800s timeout. Auto-decrements per run
  with early-reset to 0 when a clean run finds zero files (queue
  drained). N defaults to 3 — multiple runs give the system enough
  budget to finish a deep walk across timeout boundaries. New
  `POST /api/sources/{id}/backfill` arms the source; "Deep scan"
  button on each SourceRow (chip shows remaining count) wires it.

Plus partial-success classifier: non-zero gallery-dl exit + ≥1 file
downloaded + no source-level error fires `ErrorType.PARTIAL`, which
download_service maps to `status=\"ok\"`. The run did real work; the
next tick continues via gallery-dl's archive. No more red events for
"timed out mid-walk after downloading 300 files."

Retires `SourceConfig.skip_existing` — skip value is now derived from
the source state and passed as a separate `skip_value` parameter
through download() / _build_config_for_source(). `GD_DEFAULTS` drops
the now-dead key (was inert data after this refactor).

Tests cover:
* tick + backfill skip-value emission in _build_config_for_source
* PARTIAL classifier branch + TIER_LIMITED-wins-over-PARTIAL ordering
* SourceService.set_backfill_runs validation + persistence
* /api/sources/{id}/backfill 200/400/404 paths
* download_service auto-decrement / auto-reset / tick-mode-no-touch
* PARTIAL → status=ok in the orchestrator (no consecutive_failures bump)

tests/test_api_sources.py

@@ -193,3 +193,61 @@ async def test_list_derives_next_check_at_when_last_checked_set(
     )
     assert target["next_check_at"] is not None
     assert "T" in target["next_check_at"]  # ISO 8601
+
+
+# --- Plan #544: POST /api/sources/{id}/backfill ----------------------------
+
+
+@pytest.mark.asyncio
+async def test_backfill_endpoint_arms_source(client, artist, db):
+    src = Source(
+        artist_id=artist.id, platform="patreon",
+        url="https://patreon.com/alice-backfill", enabled=True,
+    )
+    db.add(src)
+    await db.commit()
+    sid = src.id
+
+    resp = await client.post(f"/api/sources/{sid}/backfill", json={"runs": 5})
+    assert resp.status_code == 200
+    body = await resp.get_json()
+    assert body["backfill_runs_remaining"] == 5
+
+    # GET reflects the new state.
+    one = await client.get(f"/api/sources/{sid}")
+    assert (await one.get_json())["backfill_runs_remaining"] == 5
+
+
+@pytest.mark.asyncio
+async def test_backfill_endpoint_defaults_to_three(client, artist, db):
+    src = Source(
+        artist_id=artist.id, platform="patreon",
+        url="https://patreon.com/alice-backfill-default", enabled=True,
+    )
+    db.add(src)
+    await db.commit()
+    resp = await client.post(f"/api/sources/{src.id}/backfill", json={})
+    body = await resp.get_json()
+    assert body["backfill_runs_remaining"] == 3
+
+
+@pytest.mark.asyncio
+async def test_backfill_endpoint_rejects_out_of_range(client, artist, db):
+    src = Source(
+        artist_id=artist.id, platform="patreon",
+        url="https://patreon.com/alice-backfill-bad", enabled=True,
+    )
+    db.add(src)
+    await db.commit()
+    bad = await client.post(f"/api/sources/{src.id}/backfill", json={"runs": 0})
+    assert bad.status_code == 400
+    too_big = await client.post(
+        f"/api/sources/{src.id}/backfill", json={"runs": 99}
+    )
+    assert too_big.status_code == 400
+
+
+@pytest.mark.asyncio
+async def test_backfill_endpoint_404_when_source_missing(client):
+    resp = await client.post("/api/sources/999999/backfill", json={"runs": 3})
+    assert resp.status_code == 404

tests/test_download_service.py

@@ -370,3 +370,179 @@ async def test_finalize_skipped_preserves_failures_clears_error(db):
     assert row.consecutive_failures == 4
     assert row.last_error is None
     assert row.last_checked_at is not None
+
+
+# --- Plan #544: backfill lifecycle + PARTIAL → status=ok -------------------
+
+
+@pytest.mark.asyncio
+async def test_backfill_decrements_after_run(
+    db, db_sync, tmp_path, seed_artist_and_source,
+):
+    """When backfill_runs_remaining > 0 going in, a non-clean / non-empty
+    run decrements by 1 — operator gets N runs to complete the deep scan
+    before tick mode resumes."""
+    from backend.app.services.download_service import DownloadService
+    from backend.app.services.importer import Importer
+
+    _artist, source = seed_artist_and_source
+    source.backfill_runs_remaining = 3
+    await db.commit()
+
+    images_root = tmp_path / "images"
+    f1 = images_root / "alice" / "patreon" / "post" / "a.jpg"
+    _make_jpg(f1, split="h")
+
+    fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
+        success=True, written_paths=[str(f1)], files_downloaded=1,
+        stdout=f"{f1}\n",
+    ))
+
+    sync_settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    importer = Importer(
+        session=db_sync, images_root=images_root, import_root=images_root,
+        thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
+    )
+    cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64"))
+    svc = DownloadService(
+        async_session=db, sync_session=db_sync,
+        gdl=fake_gdl, importer=importer, cred_service=cred_service,
+    )
+    await svc.download_source(source.id)
+
+    remaining = (await db.execute(
+        select(Source.backfill_runs_remaining).where(Source.id == source.id)
+    )).scalar_one()
+    assert remaining == 2
+
+
+@pytest.mark.asyncio
+async def test_backfill_auto_resets_on_clean_zero_files(
+    db, db_sync, tmp_path, seed_artist_and_source,
+):
+    """A clean run (rc=0) that downloaded zero files means the backfill
+    queue drained — reset to 0 immediately instead of wasting the rest of
+    the N-run budget on no-op walks."""
+    from backend.app.services.download_service import DownloadService
+    from backend.app.services.importer import Importer
+
+    _artist, source = seed_artist_and_source
+    source.backfill_runs_remaining = 3
+    await db.commit()
+
+    fake_result = _make_fake_dl_result(
+        success=True, written_paths=[], files_downloaded=0,
+    )
+    fake_gdl = _fake_gdl_with_result(fake_result)
+
+    sync_settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    importer = Importer(
+        session=db_sync, images_root=tmp_path / "images",
+        import_root=tmp_path / "images",
+        thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
+        settings=sync_settings,
+    )
+    cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64"))
+    svc = DownloadService(
+        async_session=db, sync_session=db_sync,
+        gdl=fake_gdl, importer=importer, cred_service=cred_service,
+    )
+    await svc.download_source(source.id)
+
+    remaining = (await db.execute(
+        select(Source.backfill_runs_remaining).where(Source.id == source.id)
+    )).scalar_one()
+    assert remaining == 0
+
+
+@pytest.mark.asyncio
+async def test_tick_mode_does_not_touch_backfill_counter(
+    db, db_sync, tmp_path, seed_artist_and_source,
+):
+    """When backfill_runs_remaining is already 0, downloads don't go
+    negative or otherwise mutate the counter."""
+    from backend.app.services.download_service import DownloadService
+    from backend.app.services.importer import Importer
+
+    _artist, source = seed_artist_and_source
+    assert source.backfill_runs_remaining == 0
+
+    fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
+        success=True, written_paths=[], files_downloaded=0,
+    ))
+
+    sync_settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    importer = Importer(
+        session=db_sync, images_root=tmp_path / "images",
+        import_root=tmp_path / "images",
+        thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
+        settings=sync_settings,
+    )
+    cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64"))
+    svc = DownloadService(
+        async_session=db, sync_session=db_sync,
+        gdl=fake_gdl, importer=importer, cred_service=cred_service,
+    )
+    await svc.download_source(source.id)
+
+    remaining = (await db.execute(
+        select(Source.backfill_runs_remaining).where(Source.id == source.id)
+    )).scalar_one()
+    assert remaining == 0
+
+
+@pytest.mark.asyncio
+async def test_partial_error_type_maps_to_ok_status(
+    db, db_sync, tmp_path, seed_artist_and_source,
+):
+    """A non-zero gallery-dl exit accompanied by PARTIAL error_type
+    (real files were downloaded before the run was cut short) is treated
+    as status=ok by the orchestrator — next tick continues."""
+    from backend.app.services.download_service import DownloadService
+    from backend.app.services.gallery_dl import ErrorType
+    from backend.app.services.importer import Importer
+
+    _artist, source = seed_artist_and_source
+
+    images_root = tmp_path / "images"
+    f1 = images_root / "alice" / "patreon" / "post" / "a.jpg"
+    _make_jpg(f1, split="h")
+
+    fake_result = _make_fake_dl_result(
+        success=False, written_paths=[str(f1)], files_downloaded=1,
+        error_type=ErrorType.PARTIAL,
+        error_message="Downloaded 1 file; run did not complete in budget",
+        stdout=f"{f1}\n",
+    )
+    fake_gdl = _fake_gdl_with_result(fake_result)
+
+    sync_settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    importer = Importer(
+        session=db_sync, images_root=images_root, import_root=images_root,
+        thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
+    )
+    cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64"))
+    svc = DownloadService(
+        async_session=db, sync_session=db_sync,
+        gdl=fake_gdl, importer=importer, cred_service=cred_service,
+    )
+    event_id = await svc.download_source(source.id)
+
+    ev = (await db.execute(
+        select(DownloadEvent).where(DownloadEvent.id == event_id)
+    )).scalar_one()
+    assert ev.status == "ok"
+    assert ev.error is None
+    # And the source's failure counter isn't bumped — PARTIAL isn't a failure.
+    src_after = (await db.execute(
+        select(Source.consecutive_failures).where(Source.id == source.id)
+    )).scalar_one()
+    assert src_after == 0

tests/test_gallery_dl_service.py

@@ -249,3 +249,69 @@ def test_truncate_log_caps_large_text(gdl):
 def test_truncate_log_passes_short_text(gdl):
     text = "small\n"
     assert gdl._truncate_log(text) == text
+
+
+# --- Plan #544: tick/backfill skip emission + PARTIAL classifier ------------
+
+
+def test_build_config_emits_tick_skip_value(gdl):
+    """Tick mode emits gallery-dl's exit:20 to short-circuit catch-up
+    scans once 20 contiguous archived items are seen."""
+    from backend.app.services.gallery_dl import TICK_SKIP_VALUE
+    cfg = gdl._build_config_for_source(
+        platform="patreon",
+        source_config=SourceConfig(),
+        artist_slug="alice",
+        skip_value=TICK_SKIP_VALUE,
+    )
+    assert cfg["extractor"]["skip"] == "exit:20"
+
+
+def test_build_config_emits_backfill_skip_value(gdl):
+    """Backfill mode keeps gallery-dl's default skip=True so it walks the
+    full post history."""
+    from backend.app.services.gallery_dl import BACKFILL_SKIP_VALUE
+    cfg = gdl._build_config_for_source(
+        platform="patreon",
+        source_config=SourceConfig(),
+        artist_slug="alice",
+        skip_value=BACKFILL_SKIP_VALUE,
+    )
+    assert cfg["extractor"]["skip"] is True
+
+
+def test_categorize_partial_when_files_downloaded_then_nonzero_exit(gdl):
+    """Non-zero exit + ≥1 file downloaded + no source-level error →
+    PARTIAL. Downstream maps to status=ok because the next tick continues."""
+    # stdout has a single downloaded file (line starting with `/`); no
+    # error-class indicators in stderr; non-zero return code.
+    etype, msg = gdl._categorize_error(
+        return_code=1,
+        stdout="/images/alice/patreon/post1/img.jpg\n",
+        stderr="",
+    )
+    assert etype == ErrorType.PARTIAL
+    assert "Downloaded 1 file" in msg
+
+
+def test_categorize_unknown_when_no_files_and_no_pattern(gdl):
+    """PARTIAL only kicks in when files were actually written. A non-zero
+    exit with no downloads and no recognized pattern still falls through
+    to UNKNOWN_ERROR — distinct from the partial-success case."""
+    etype, _ = gdl._categorize_error(
+        return_code=2,
+        stdout="",
+        stderr="some unrecognized noise\n",
+    )
+    assert etype == ErrorType.UNKNOWN_ERROR
+
+
+def test_categorize_tier_limited_wins_over_partial(gdl):
+    """A run that hit tier-limited warnings AND downloaded some files
+    classifies as TIER_LIMITED (the more specific category), not PARTIAL."""
+    stdout = "/images/alice/patreon/post1/img.jpg\n"
+    stderr = "[patreon][warning] Not allowed to view post 123\n"
+    etype, _ = gdl._categorize_error(
+        return_code=1, stdout=stdout, stderr=stderr,
+    )
+    assert etype == ErrorType.TIER_LIMITED

tests/test_source_service.py

@@ -169,3 +169,47 @@ async def test_list_hides_sidecar_synthetic_anchors(db):
     assert {s.url for s in everything} >= {
         "https://patreon.com/alice", "sidecar:patreon:alice",
     }
+
+
+# --- Plan #544: backfill counter -------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_set_backfill_runs_arms_source(db):
+    """The service method sets backfill_runs_remaining and returns the
+    updated record so the API can echo it back."""
+    artist = await _artist(db, "Alice")
+    svc = SourceService(db)
+    rec = await svc.create(
+        artist_id=artist.id, platform="patreon",
+        url="https://patreon.com/alice",
+    )
+    assert rec.backfill_runs_remaining == 0
+    updated = await svc.set_backfill_runs(rec.id, 5)
+    assert updated.backfill_runs_remaining == 5
+
+    db_value = (await db.execute(
+        select(Source.backfill_runs_remaining).where(Source.id == rec.id)
+    )).scalar_one()
+    assert db_value == 5
+
+
+@pytest.mark.asyncio
+async def test_set_backfill_runs_rejects_out_of_range(db):
+    artist = await _artist(db, "Alice")
+    svc = SourceService(db)
+    rec = await svc.create(
+        artist_id=artist.id, platform="patreon",
+        url="https://patreon.com/alice",
+    )
+    with pytest.raises(ValueError):
+        await svc.set_backfill_runs(rec.id, 0)
+    with pytest.raises(ValueError):
+        await svc.set_backfill_runs(rec.id, 11)
+
+
+@pytest.mark.asyncio
+async def test_set_backfill_runs_raises_when_source_missing(db):
+    svc = SourceService(db)
+    with pytest.raises(LookupError):
+        await svc.set_backfill_runs(99999, 3)