feat(download): tick/backfill modes + partial-success classifier (plan #544)

Routine subscription polls walked the entire post history every tick
even when nothing had changed, because gallery-dl's default `skip: True`
continues iterating archived posts. A creator with ~550 archived posts
(Knuxy patreon) saturates the 870s wall-clock cap before completing,
even with zero downloads needed. Plus, a tier-limited run that
downloaded hundreds of files but ran out the clock should be a
warning, not an error.

Two coupled changes, both operator-flagged 2026-06-01:

* **Tick mode (default, cron polls).** New `TICK_SKIP_VALUE = "exit:20"`
  asks gallery-dl to exit after 20 contiguous archived items. Fresh
  subscriptions + new-content cases still walk normally; established
  subscription with zero new content exits in ~30s of HEAD requests
  instead of pegging the timeout. 20 (not 5) gives headroom against
  paywall warnings interleaving with archived items.
* **Backfill mode (explicit, operator-triggered).** Sticky for N runs
  via new `Source.backfill_runs_remaining` (alembic 0031). While > 0,
  downloads use `skip: True` + 1800s timeout. Auto-decrements per run
  with early-reset to 0 when a clean run finds zero files (queue
  drained). N defaults to 3 — multiple runs give the system enough
  budget to finish a deep walk across timeout boundaries. New
  `POST /api/sources/{id}/backfill` arms the source; "Deep scan"
  button on each SourceRow (chip shows remaining count) wires it.

Plus partial-success classifier: non-zero gallery-dl exit + ≥1 file
downloaded + no source-level error fires `ErrorType.PARTIAL`, which
download_service maps to `status=\"ok\"`. The run did real work; the
next tick continues via gallery-dl's archive. No more red events for
"timed out mid-walk after downloading 300 files."

Retires `SourceConfig.skip_existing` — skip value is now derived from
the source state and passed as a separate `skip_value` parameter
through download() / _build_config_for_source(). `GD_DEFAULTS` drops
the now-dead key (was inert data after this refactor).

Tests cover:
* tick + backfill skip-value emission in _build_config_for_source
* PARTIAL classifier branch + TIER_LIMITED-wins-over-PARTIAL ordering
* SourceService.set_backfill_runs validation + persistence
* /api/sources/{id}/backfill 200/400/404 paths
* download_service auto-decrement / auto-reset / tick-mode-no-touch
* PARTIAL → status=ok in the orchestrator (no consecutive_failures bump)

backend/app/api/sources.py

@@ -120,6 +120,31 @@ async def delete_source(source_id: int):
     return "", 204
 
 
+@sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
+async def set_backfill(source_id: int):
+    """Plan #544: arm a source for backfill mode for the next N download
+    runs. Body: `{"runs": int}` (1..10, default 3). Returns the updated
+    source dict. While backfill_runs_remaining > 0, downloads use
+    gallery-dl's full-walk config (skip: True + 30-min timeout) instead
+    of the catch-up default (skip: "exit:20" + 14.5-min timeout)."""
+    payload = await request.get_json(silent=True) or {}
+    runs = payload.get("runs", 3)
+    try:
+        runs = int(runs)
+    except (TypeError, ValueError):
+        return _bad("invalid_runs", detail="runs must be an integer")
+    async with get_session() as session:
+        try:
+            record = await SourceService(session).set_backfill_runs(
+                source_id, runs,
+            )
+        except LookupError:
+            return _bad("not_found", status=404)
+        except ValueError as exc:
+            return _bad("invalid_runs", detail=str(exc))
+    return jsonify(record.to_dict())
+
+
 @sources_bp.route("/<int:source_id>/check", methods=["POST"])
 async def check_source(source_id: int):
     """FC-3c: enqueue a download for this source.

backend/app/models/source.py

@@ -29,4 +29,13 @@ class Source(Base):
     check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True)
     consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 
+    # alembic 0031: sticky deep-scan budget. When > 0, the next N download
+    # runs use gallery-dl's full-walk config (skip: True + 1800s timeout);
+    # when 0, runs use tick mode (skip: "exit:20" + 870s, exits early once
+    # 20 contiguous archived items are seen). Auto-decrements per run, with
+    # an auto-reset to 0 on clean exit + zero downloads (queue drained).
+    backfill_runs_remaining: Mapped[int] = mapped_column(
+        Integer, nullable=False, default=0, server_default="0",
+    )
+
     artist = relationship("Artist", back_populates="sources")

backend/app/services/download_service.py

@@ -25,7 +25,14 @@ from sqlalchemy.orm import joinedload
 
 from ..models import Artist, DownloadEvent, Source
 from .credential_service import CredentialService
-from .gallery_dl import GalleryDLService, SourceConfig
+from .gallery_dl import (
+    BACKFILL_SKIP_VALUE,
+    BACKFILL_TIMEOUT_SECONDS,
+    TICK_SKIP_VALUE,
+    ErrorType,
+    GalleryDLService,
+    SourceConfig,
+)
 from .importer import Importer
 from .patreon_resolver import resolve_campaign_id
 from .scheduler_service import set_platform_cooldown
@@ -84,6 +91,18 @@ class DownloadService:
         ctx = setup
 
         source_config = SourceConfig.from_dict(ctx["config_overrides"] or {})
+        # alembic 0031 / plan #544: derive skip_value + timeout from the
+        # source's backfill_runs_remaining counter. When > 0, walk the full
+        # post history (skip: True + 1800s); when 0, exit gallery-dl after
+        # 20 contiguous archived items (skip: "exit:20" + the default
+        # 870s). Operator sets backfill via POST /api/sources/{id}/backfill.
+        backfill_remaining = ctx.get("backfill_runs_remaining", 0) or 0
+        if backfill_remaining > 0:
+            skip_value: bool | str = BACKFILL_SKIP_VALUE
+            source_config.timeout = BACKFILL_TIMEOUT_SECONDS
+        else:
+            skip_value = TICK_SKIP_VALUE
+
         effective_url = _effective_url(
             ctx["platform"], ctx["url"], ctx["config_overrides"] or {}
         )
@@ -95,6 +114,7 @@ class DownloadService:
             source_config=source_config,
             cookies_path=ctx["cookies_path"],
             auth_token=ctx["auth_token"],
+            skip_value=skip_value,
         )
 
         resolved_campaign_id: str | None = None
@@ -121,6 +141,7 @@ class DownloadService:
                         source_config=source_config,
                         cookies_path=ctx["cookies_path"],
                         auth_token=ctx["auth_token"],
+                        skip_value=skip_value,
                     )
 
         return await self._phase3_persist(
@@ -185,6 +206,7 @@ class DownloadService:
             "config_overrides": dict(source.config_overrides or {}),
             "cookies_path": cookies_path,
             "auth_token": auth_token,
+            "backfill_runs_remaining": source.backfill_runs_remaining or 0,
         }
 
     async def _phase3_persist(
@@ -259,12 +281,21 @@ class DownloadService:
         run_stats["quarantined_count"] = dl_result.files_quarantined
         stderr_summary = self.gdl._extract_errors_warnings(dl_result.stderr)
 
-        status = "ok" if (dl_result.success and import_summary["errors"] == 0) else "error"
+        # Plan #544: PARTIAL means the run downloaded ≥1 file but the
+        # subprocess didn't finish in budget (typically wall-clock timeout
+        # mid-walk). Real work happened; the next tick continues via
+        # gallery-dl's archive. NOT a failure for status purposes.
+        if dl_result.success and import_summary["errors"] == 0:
+            status = "ok"
+        elif dl_result.error_type == ErrorType.PARTIAL and import_summary["errors"] == 0:
+            status = "ok"
+        else:
+            status = "error"
         ev.status = status
         ev.finished_at = datetime.now(UTC)
         ev.files_count = import_summary["attached"]
         ev.bytes_downloaded = bytes_downloaded
-        ev.error = dl_result.error_message if not dl_result.success else None
+        ev.error = dl_result.error_message if status == "error" else None
         ev.metadata_ = {
             "run_stats": run_stats,
             "error_type": dl_result.error_type.value if dl_result.error_type else None,
@@ -279,6 +310,19 @@ class DownloadService:
             source_id=ctx["source_id"], status=status, error_message=ev.error,
             error_type=dl_result.error_type.value if dl_result.error_type else None,
         )
+        # Plan #544: backfill lifecycle — auto-complete when a clean
+        # backfill run drained the queue (gallery-dl exited 0 + zero files
+        # downloaded means there was nothing to fetch); otherwise decrement
+        # the counter. Next tick falls back to tick mode once it hits 0.
+        backfill_remaining = ctx.get("backfill_runs_remaining", 0) or 0
+        if backfill_remaining > 0:
+            src = (await self.async_session.execute(
+                select(Source).where(Source.id == ctx["source_id"])
+            )).scalar_one()
+            if dl_result.return_code == 0 and dl_result.files_downloaded == 0:
+                src.backfill_runs_remaining = 0
+            else:
+                src.backfill_runs_remaining = max(0, backfill_remaining - 1)
         await self.async_session.commit()
         return event_id
 

backend/app/services/gallery_dl.py

@@ -39,9 +39,29 @@ class ErrorType(StrEnum):
     HTTP_ERROR = "http_error"
     UNSUPPORTED_URL = "unsupported_url"
     VALIDATION_FAILED = "validation_failed"
+    # Run made real progress (downloaded ≥1 file) but did not finish in the
+    # subprocess budget. Distinct from UNKNOWN_ERROR — the downstream status
+    # mapping classifies this as "ok" because the next tick continues.
+    PARTIAL = "partial"
     UNKNOWN_ERROR = "unknown_error"
 
 
+# Tick mode (routine cron polls): skip ≤20 contiguous already-archived
+# items, then exit gallery-dl. Established subscription with zero new
+# content exits in ~30s of HEAD requests instead of walking to the bottom
+# of the post history (which can be hours for prolific creators). 20 (not
+# 5) is operator-set headroom against any edge case where paywalled or
+# otherwise-non-downloadable items might interleave with archived ones —
+# 20 contiguous HEADs is still negligible.
+TICK_SKIP_VALUE = "exit:20"
+
+# Backfill mode (operator-triggered deep scan): walk the full history.
+# Source.backfill_runs_remaining > 0 selects this mode; the longer
+# timeout below absorbs creators with thousands of posts.
+BACKFILL_SKIP_VALUE = True
+BACKFILL_TIMEOUT_SECONDS = 1800
+
+
 # 30 seconds shy of download_source's Celery soft_time_limit (900s, see
 # tasks/download.py:32). subprocess.run MUST raise TimeoutExpired before
 # Celery raises SoftTimeLimitExceeded — otherwise Celery wins the race,
@@ -56,12 +76,19 @@ _DEFAULT_GDL_TIMEOUT_SECONDS = 870
 
 @dataclass
 class SourceConfig:
+    """Per-source overrides loaded from Source.config_overrides JSON.
+
+    Note: the gallery-dl `skip` value (tick vs backfill, see TICK_SKIP_VALUE /
+    BACKFILL_SKIP_VALUE) is NOT carried here — it derives from the
+    Source.backfill_runs_remaining column at the download_service layer
+    and is passed to _build_config_for_source as `skip_value`. Same for
+    the per-run subprocess timeout.
+    """
     content_types: list[str] = field(default_factory=lambda: ["all"])
     sleep: float | None = None
     sleep_request: float | None = None
     directory_pattern: str | None = None
     filename_pattern: str | None = None
-    skip_existing: bool = True
     save_metadata: bool = True
     timeout: int = _DEFAULT_GDL_TIMEOUT_SECONDS
 
@@ -73,7 +100,6 @@ class SourceConfig:
             sleep_request=data.get("sleep_request"),
             directory_pattern=data.get("directory_pattern"),
             filename_pattern=data.get("filename_pattern"),
-            skip_existing=data.get("skip_existing", True),
             save_metadata=data.get("save_metadata", True),
             timeout=data.get("timeout", _DEFAULT_GDL_TIMEOUT_SECONDS),
         )
@@ -245,7 +271,18 @@ class GalleryDLService:
         platform: str,
         source_config: SourceConfig,
         artist_slug: str,
+        skip_value: bool | str = BACKFILL_SKIP_VALUE,
     ) -> dict:
+        """`skip_value` controls gallery-dl's archive-walk behavior:
+            - True (BACKFILL_SKIP_VALUE): walk full post history, skipping
+              archived items but continuing past them. Used in backfill mode.
+            - "exit:20" (TICK_SKIP_VALUE): exit gallery-dl after 20
+              contiguous archived items. Used in tick (routine catch-up)
+              mode for fast no-op syncs on creators with deep history.
+            - False: don't skip — redownload everything (not used in FC).
+        The caller (download_service) chooses based on
+        Source.backfill_runs_remaining.
+        """
         config = json.loads(json.dumps(self._get_default_config()))  # deep copy
 
         destination = str(self.images_root / artist_slug / platform)
@@ -255,7 +292,7 @@ class GalleryDLService:
             config["extractor"]["sleep"] = source_config.sleep
         if source_config.sleep_request is not None:
             config["extractor"]["sleep-request"] = source_config.sleep_request
-        config["extractor"]["skip"] = source_config.skip_existing
+        config["extractor"]["skip"] = skip_value
 
         if source_config.save_metadata:
             config["extractor"]["postprocessors"] = [
@@ -394,6 +431,22 @@ class GalleryDLService:
                     f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}",
                 )
 
+        # Partial-success: the subprocess exited non-zero (typically because
+        # the wall-clock timeout fired mid-walk), but it had downloaded ≥1
+        # file by then and no source-level error category fired. The work
+        # the run DID do is real; gallery-dl's archive will pick up where
+        # it left off on the next tick. Mapped to status="ok" downstream
+        # (download_service.py) so this doesn't flag the source as
+        # "needs attention." Operator-flagged 2026-06-01 after a Knuxy
+        # patreon run downloaded hundreds of files then ran red on timeout.
+        files_downloaded = self._count_downloaded_files(stdout)
+        if not has_actual_error and files_downloaded > 0:
+            return (
+                ErrorType.PARTIAL,
+                f"Downloaded {files_downloaded} file{'s' if files_downloaded != 1 else ''}; "
+                "run did not complete in budget — next tick will continue",
+            )
+
         return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
 
     def _count_downloaded_files(self, stdout: str) -> int:
@@ -545,6 +598,7 @@ class GalleryDLService:
         source_config: SourceConfig | None = None,
         cookies_path: str | None = None,
         auth_token: str | None = None,
+        skip_value: bool | str = BACKFILL_SKIP_VALUE,
     ) -> DownloadResult:
         start_time = time.time()
         started_at = datetime.now(UTC).isoformat()
@@ -552,7 +606,9 @@ class GalleryDLService:
         if source_config is None:
             source_config = SourceConfig()
 
-        config = self._build_config_for_source(platform, source_config, artist_slug)
+        config = self._build_config_for_source(
+            platform, source_config, artist_slug, skip_value=skip_value,
+        )
 
         if cookies_path:
             config["extractor"]["cookies"] = cookies_path

backend/app/services/platforms/base.py

@@ -96,10 +96,12 @@ def str_field(v) -> str | None:
 # Shared gallery-dl invocation defaults. Embedded in each platform's
 # default_config (with platform-specific overrides) so per-platform
 # choices stay explicit.
+# Note: the gallery-dl `skip` value (tick "exit:20" vs backfill True) is
+# NOT here — it's derived from Source.backfill_runs_remaining at download
+# time. See plan #544 / gallery_dl.TICK_SKIP_VALUE,BACKFILL_SKIP_VALUE.
 GD_DEFAULTS = {
     "sleep": 3.0,
     "sleep_request": 1.5,
-    "skip_existing": True,
     "save_metadata": True,
     "timeout": 3600,
 }

backend/app/services/refetch_service.py

@@ -86,7 +86,7 @@ def attempt_refetch(
     if src is None:
         return {"status": "no_source"}
 
-    # Remove the bad copy so gallery-dl (skip_existing) re-fetches it on
+    # Remove the bad copy so gallery-dl's archive-skip re-fetches it on
     # the source re-check instead of skipping the still-present corrupt
     # file.
     try:

backend/app/services/source_service.py

@@ -62,6 +62,7 @@ class SourceRecord:
     check_interval_override: int | None
     consecutive_failures: int
     next_check_at: str | None
+    backfill_runs_remaining: int
 
     def to_dict(self) -> dict:
         return {
@@ -78,6 +79,7 @@ class SourceRecord:
             "check_interval_override": self.check_interval_override,
             "consecutive_failures": self.consecutive_failures,
             "next_check_at": self.next_check_at,
+            "backfill_runs_remaining": self.backfill_runs_remaining,
         }
 
 
@@ -140,6 +142,7 @@ class SourceService:
             check_interval_override=source.check_interval_override,
             consecutive_failures=source.consecutive_failures or 0,
             next_check_at=nxt.isoformat() if nxt else None,
+            backfill_runs_remaining=source.backfill_runs_remaining or 0,
         )
 
     async def _row_to_record(self, source: Source) -> SourceRecord:
@@ -264,6 +267,25 @@ class SourceService:
         await self.session.commit()
         return await self._row_to_record(source)
 
+    async def set_backfill_runs(
+        self, source_id: int, runs: int,
+    ) -> SourceRecord:
+        """Plan #544: arm a source for backfill mode. The next `runs`
+        download runs will use gallery-dl's full-walk config (skip: True
+        + 30-min timeout) instead of the catch-up default. Runs must be
+        1..10 — bigger is rejected to keep the operator from accidentally
+        setting a runaway budget."""
+        if not isinstance(runs, int) or runs < 1 or runs > 10:
+            raise ValueError("runs must be an integer in [1, 10]")
+        source = (await self.session.execute(
+            select(Source).where(Source.id == source_id)
+        )).scalar_one_or_none()
+        if source is None:
+            raise LookupError(f"source id={source_id} not found")
+        source.backfill_runs_remaining = runs
+        await self.session.commit()
+        return await self._row_to_record(source)
+
     async def delete(self, source_id: int) -> None:
         source = (await self.session.execute(
             select(Source).where(Source.id == source_id)