From 19aece1fc43202548a5fe1649dc5ff242eaea2da Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 1 Jun 2026 18:23:28 -0400 Subject: [PATCH] feat(download): tick/backfill modes + partial-success classifier (plan #544) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Routine subscription polls walked the entire post history every tick even when nothing had changed, because gallery-dl's default `skip: True` continues iterating archived posts. A creator with ~550 archived posts (Knuxy patreon) saturates the 870s wall-clock cap before completing, even with zero downloads needed. Plus, a tier-limited run that downloaded hundreds of files but ran out the clock should be a warning, not an error. Two coupled changes, both operator-flagged 2026-06-01: * **Tick mode (default, cron polls).** New `TICK_SKIP_VALUE = "exit:20"` asks gallery-dl to exit after 20 contiguous archived items. Fresh subscriptions + new-content cases still walk normally; established subscription with zero new content exits in ~30s of HEAD requests instead of pegging the timeout. 20 (not 5) gives headroom against paywall warnings interleaving with archived items. * **Backfill mode (explicit, operator-triggered).** Sticky for N runs via new `Source.backfill_runs_remaining` (alembic 0031). While > 0, downloads use `skip: True` + 1800s timeout. Auto-decrements per run with early-reset to 0 when a clean run finds zero files (queue drained). N defaults to 3 — multiple runs give the system enough budget to finish a deep walk across timeout boundaries. New `POST /api/sources/{id}/backfill` arms the source; "Deep scan" button on each SourceRow (chip shows remaining count) wires it. Plus partial-success classifier: non-zero gallery-dl exit + ≥1 file downloaded + no source-level error fires `ErrorType.PARTIAL`, which download_service maps to `status=\"ok\"`. The run did real work; the next tick continues via gallery-dl's archive. No more red events for "timed out mid-walk after downloading 300 files." Retires `SourceConfig.skip_existing` — skip value is now derived from the source state and passed as a separate `skip_value` parameter through download() / _build_config_for_source(). `GD_DEFAULTS` drops the now-dead key (was inert data after this refactor). Tests cover: * tick + backfill skip-value emission in _build_config_for_source * PARTIAL classifier branch + TIER_LIMITED-wins-over-PARTIAL ordering * SourceService.set_backfill_runs validation + persistence * /api/sources/{id}/backfill 200/400/404 paths * download_service auto-decrement / auto-reset / tick-mode-no-touch * PARTIAL → status=ok in the orchestrator (no consecutive_failures bump) --- .../0031_source_backfill_runs_remaining.py | 45 +++++ backend/app/api/sources.py | 25 +++ backend/app/models/source.py | 9 + backend/app/services/download_service.py | 50 ++++- backend/app/services/gallery_dl.py | 64 ++++++- backend/app/services/platforms/base.py | 4 +- backend/app/services/refetch_service.py | 2 +- backend/app/services/source_service.py | 22 +++ .../components/subscriptions/SourceRow.vue | 18 +- .../subscriptions/SubscriptionsTab.vue | 31 +++ frontend/src/stores/sources.js | 10 + tests/test_api_sources.py | 58 ++++++ tests/test_download_service.py | 176 ++++++++++++++++++ tests/test_gallery_dl_service.py | 66 +++++++ tests/test_source_service.py | 44 +++++ 15 files changed, 614 insertions(+), 10 deletions(-) create mode 100644 alembic/versions/0031_source_backfill_runs_remaining.py diff --git a/alembic/versions/0031_source_backfill_runs_remaining.py b/alembic/versions/0031_source_backfill_runs_remaining.py new file mode 100644 index 0000000..ed140fc --- /dev/null +++ b/alembic/versions/0031_source_backfill_runs_remaining.py @@ -0,0 +1,45 @@ +"""source.backfill_runs_remaining: sticky deep-scan mode + +Revision ID: 0031 +Revises: 0030 +Create Date: 2026-06-01 + +Tick vs backfill mode for subscription downloads. When +`backfill_runs_remaining > 0`, the next N download runs use +`skip: True` + 30-min timeout (walk full history). When 0, runs use +`skip: "exit:20"` + 14.5-min timeout (catch-up mode, exits early once +20 contiguous archived items are seen). + +Operator-flagged 2026-06-01 (Knuxy run #38887): a creator with ~550 +archived posts saturates the 870s catch-up timeout even when there is +no new content, because gallery-dl's default `skip: True` keeps walking. +Tick mode short-circuits that; backfill mode is the explicit opt-in for +deep history scans. + +Default 0 (all existing subscriptions start in tick mode). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0031" +down_revision: Union[str, None] = "0030" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "source", + sa.Column( + "backfill_runs_remaining", + sa.Integer, + nullable=False, + server_default="0", + ), + ) + + +def downgrade() -> None: + op.drop_column("source", "backfill_runs_remaining") diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index a1312ba..fe234ee 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -120,6 +120,31 @@ async def delete_source(source_id: int): return "", 204 +@sources_bp.route("//backfill", methods=["POST"]) +async def set_backfill(source_id: int): + """Plan #544: arm a source for backfill mode for the next N download + runs. Body: `{"runs": int}` (1..10, default 3). Returns the updated + source dict. While backfill_runs_remaining > 0, downloads use + gallery-dl's full-walk config (skip: True + 30-min timeout) instead + of the catch-up default (skip: "exit:20" + 14.5-min timeout).""" + payload = await request.get_json(silent=True) or {} + runs = payload.get("runs", 3) + try: + runs = int(runs) + except (TypeError, ValueError): + return _bad("invalid_runs", detail="runs must be an integer") + async with get_session() as session: + try: + record = await SourceService(session).set_backfill_runs( + source_id, runs, + ) + except LookupError: + return _bad("not_found", status=404) + except ValueError as exc: + return _bad("invalid_runs", detail=str(exc)) + return jsonify(record.to_dict()) + + @sources_bp.route("//check", methods=["POST"]) async def check_source(source_id: int): """FC-3c: enqueue a download for this source. diff --git a/backend/app/models/source.py b/backend/app/models/source.py index 9c90e81..407227e 100644 --- a/backend/app/models/source.py +++ b/backend/app/models/source.py @@ -29,4 +29,13 @@ class Source(Base): check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True) consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + # alembic 0031: sticky deep-scan budget. When > 0, the next N download + # runs use gallery-dl's full-walk config (skip: True + 1800s timeout); + # when 0, runs use tick mode (skip: "exit:20" + 870s, exits early once + # 20 contiguous archived items are seen). Auto-decrements per run, with + # an auto-reset to 0 on clean exit + zero downloads (queue drained). + backfill_runs_remaining: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0", + ) + artist = relationship("Artist", back_populates="sources") diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 6e917c3..529da83 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -25,7 +25,14 @@ from sqlalchemy.orm import joinedload from ..models import Artist, DownloadEvent, Source from .credential_service import CredentialService -from .gallery_dl import GalleryDLService, SourceConfig +from .gallery_dl import ( + BACKFILL_SKIP_VALUE, + BACKFILL_TIMEOUT_SECONDS, + TICK_SKIP_VALUE, + ErrorType, + GalleryDLService, + SourceConfig, +) from .importer import Importer from .patreon_resolver import resolve_campaign_id from .scheduler_service import set_platform_cooldown @@ -84,6 +91,18 @@ class DownloadService: ctx = setup source_config = SourceConfig.from_dict(ctx["config_overrides"] or {}) + # alembic 0031 / plan #544: derive skip_value + timeout from the + # source's backfill_runs_remaining counter. When > 0, walk the full + # post history (skip: True + 1800s); when 0, exit gallery-dl after + # 20 contiguous archived items (skip: "exit:20" + the default + # 870s). Operator sets backfill via POST /api/sources/{id}/backfill. + backfill_remaining = ctx.get("backfill_runs_remaining", 0) or 0 + if backfill_remaining > 0: + skip_value: bool | str = BACKFILL_SKIP_VALUE + source_config.timeout = BACKFILL_TIMEOUT_SECONDS + else: + skip_value = TICK_SKIP_VALUE + effective_url = _effective_url( ctx["platform"], ctx["url"], ctx["config_overrides"] or {} ) @@ -95,6 +114,7 @@ class DownloadService: source_config=source_config, cookies_path=ctx["cookies_path"], auth_token=ctx["auth_token"], + skip_value=skip_value, ) resolved_campaign_id: str | None = None @@ -121,6 +141,7 @@ class DownloadService: source_config=source_config, cookies_path=ctx["cookies_path"], auth_token=ctx["auth_token"], + skip_value=skip_value, ) return await self._phase3_persist( @@ -185,6 +206,7 @@ class DownloadService: "config_overrides": dict(source.config_overrides or {}), "cookies_path": cookies_path, "auth_token": auth_token, + "backfill_runs_remaining": source.backfill_runs_remaining or 0, } async def _phase3_persist( @@ -259,12 +281,21 @@ class DownloadService: run_stats["quarantined_count"] = dl_result.files_quarantined stderr_summary = self.gdl._extract_errors_warnings(dl_result.stderr) - status = "ok" if (dl_result.success and import_summary["errors"] == 0) else "error" + # Plan #544: PARTIAL means the run downloaded ≥1 file but the + # subprocess didn't finish in budget (typically wall-clock timeout + # mid-walk). Real work happened; the next tick continues via + # gallery-dl's archive. NOT a failure for status purposes. + if dl_result.success and import_summary["errors"] == 0: + status = "ok" + elif dl_result.error_type == ErrorType.PARTIAL and import_summary["errors"] == 0: + status = "ok" + else: + status = "error" ev.status = status ev.finished_at = datetime.now(UTC) ev.files_count = import_summary["attached"] ev.bytes_downloaded = bytes_downloaded - ev.error = dl_result.error_message if not dl_result.success else None + ev.error = dl_result.error_message if status == "error" else None ev.metadata_ = { "run_stats": run_stats, "error_type": dl_result.error_type.value if dl_result.error_type else None, @@ -279,6 +310,19 @@ class DownloadService: source_id=ctx["source_id"], status=status, error_message=ev.error, error_type=dl_result.error_type.value if dl_result.error_type else None, ) + # Plan #544: backfill lifecycle — auto-complete when a clean + # backfill run drained the queue (gallery-dl exited 0 + zero files + # downloaded means there was nothing to fetch); otherwise decrement + # the counter. Next tick falls back to tick mode once it hits 0. + backfill_remaining = ctx.get("backfill_runs_remaining", 0) or 0 + if backfill_remaining > 0: + src = (await self.async_session.execute( + select(Source).where(Source.id == ctx["source_id"]) + )).scalar_one() + if dl_result.return_code == 0 and dl_result.files_downloaded == 0: + src.backfill_runs_remaining = 0 + else: + src.backfill_runs_remaining = max(0, backfill_remaining - 1) await self.async_session.commit() return event_id diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 3e19a69..67fc59b 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -39,9 +39,29 @@ class ErrorType(StrEnum): HTTP_ERROR = "http_error" UNSUPPORTED_URL = "unsupported_url" VALIDATION_FAILED = "validation_failed" + # Run made real progress (downloaded ≥1 file) but did not finish in the + # subprocess budget. Distinct from UNKNOWN_ERROR — the downstream status + # mapping classifies this as "ok" because the next tick continues. + PARTIAL = "partial" UNKNOWN_ERROR = "unknown_error" +# Tick mode (routine cron polls): skip ≤20 contiguous already-archived +# items, then exit gallery-dl. Established subscription with zero new +# content exits in ~30s of HEAD requests instead of walking to the bottom +# of the post history (which can be hours for prolific creators). 20 (not +# 5) is operator-set headroom against any edge case where paywalled or +# otherwise-non-downloadable items might interleave with archived ones — +# 20 contiguous HEADs is still negligible. +TICK_SKIP_VALUE = "exit:20" + +# Backfill mode (operator-triggered deep scan): walk the full history. +# Source.backfill_runs_remaining > 0 selects this mode; the longer +# timeout below absorbs creators with thousands of posts. +BACKFILL_SKIP_VALUE = True +BACKFILL_TIMEOUT_SECONDS = 1800 + + # 30 seconds shy of download_source's Celery soft_time_limit (900s, see # tasks/download.py:32). subprocess.run MUST raise TimeoutExpired before # Celery raises SoftTimeLimitExceeded — otherwise Celery wins the race, @@ -56,12 +76,19 @@ _DEFAULT_GDL_TIMEOUT_SECONDS = 870 @dataclass class SourceConfig: + """Per-source overrides loaded from Source.config_overrides JSON. + + Note: the gallery-dl `skip` value (tick vs backfill, see TICK_SKIP_VALUE / + BACKFILL_SKIP_VALUE) is NOT carried here — it derives from the + Source.backfill_runs_remaining column at the download_service layer + and is passed to _build_config_for_source as `skip_value`. Same for + the per-run subprocess timeout. + """ content_types: list[str] = field(default_factory=lambda: ["all"]) sleep: float | None = None sleep_request: float | None = None directory_pattern: str | None = None filename_pattern: str | None = None - skip_existing: bool = True save_metadata: bool = True timeout: int = _DEFAULT_GDL_TIMEOUT_SECONDS @@ -73,7 +100,6 @@ class SourceConfig: sleep_request=data.get("sleep_request"), directory_pattern=data.get("directory_pattern"), filename_pattern=data.get("filename_pattern"), - skip_existing=data.get("skip_existing", True), save_metadata=data.get("save_metadata", True), timeout=data.get("timeout", _DEFAULT_GDL_TIMEOUT_SECONDS), ) @@ -245,7 +271,18 @@ class GalleryDLService: platform: str, source_config: SourceConfig, artist_slug: str, + skip_value: bool | str = BACKFILL_SKIP_VALUE, ) -> dict: + """`skip_value` controls gallery-dl's archive-walk behavior: + - True (BACKFILL_SKIP_VALUE): walk full post history, skipping + archived items but continuing past them. Used in backfill mode. + - "exit:20" (TICK_SKIP_VALUE): exit gallery-dl after 20 + contiguous archived items. Used in tick (routine catch-up) + mode for fast no-op syncs on creators with deep history. + - False: don't skip — redownload everything (not used in FC). + The caller (download_service) chooses based on + Source.backfill_runs_remaining. + """ config = json.loads(json.dumps(self._get_default_config())) # deep copy destination = str(self.images_root / artist_slug / platform) @@ -255,7 +292,7 @@ class GalleryDLService: config["extractor"]["sleep"] = source_config.sleep if source_config.sleep_request is not None: config["extractor"]["sleep-request"] = source_config.sleep_request - config["extractor"]["skip"] = source_config.skip_existing + config["extractor"]["skip"] = skip_value if source_config.save_metadata: config["extractor"]["postprocessors"] = [ @@ -394,6 +431,22 @@ class GalleryDLService: f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}", ) + # Partial-success: the subprocess exited non-zero (typically because + # the wall-clock timeout fired mid-walk), but it had downloaded ≥1 + # file by then and no source-level error category fired. The work + # the run DID do is real; gallery-dl's archive will pick up where + # it left off on the next tick. Mapped to status="ok" downstream + # (download_service.py) so this doesn't flag the source as + # "needs attention." Operator-flagged 2026-06-01 after a Knuxy + # patreon run downloaded hundreds of files then ran red on timeout. + files_downloaded = self._count_downloaded_files(stdout) + if not has_actual_error and files_downloaded > 0: + return ( + ErrorType.PARTIAL, + f"Downloaded {files_downloaded} file{'s' if files_downloaded != 1 else ''}; " + "run did not complete in budget — next tick will continue", + ) + return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})" def _count_downloaded_files(self, stdout: str) -> int: @@ -545,6 +598,7 @@ class GalleryDLService: source_config: SourceConfig | None = None, cookies_path: str | None = None, auth_token: str | None = None, + skip_value: bool | str = BACKFILL_SKIP_VALUE, ) -> DownloadResult: start_time = time.time() started_at = datetime.now(UTC).isoformat() @@ -552,7 +606,9 @@ class GalleryDLService: if source_config is None: source_config = SourceConfig() - config = self._build_config_for_source(platform, source_config, artist_slug) + config = self._build_config_for_source( + platform, source_config, artist_slug, skip_value=skip_value, + ) if cookies_path: config["extractor"]["cookies"] = cookies_path diff --git a/backend/app/services/platforms/base.py b/backend/app/services/platforms/base.py index 8a626b7..737b529 100644 --- a/backend/app/services/platforms/base.py +++ b/backend/app/services/platforms/base.py @@ -96,10 +96,12 @@ def str_field(v) -> str | None: # Shared gallery-dl invocation defaults. Embedded in each platform's # default_config (with platform-specific overrides) so per-platform # choices stay explicit. +# Note: the gallery-dl `skip` value (tick "exit:20" vs backfill True) is +# NOT here — it's derived from Source.backfill_runs_remaining at download +# time. See plan #544 / gallery_dl.TICK_SKIP_VALUE,BACKFILL_SKIP_VALUE. GD_DEFAULTS = { "sleep": 3.0, "sleep_request": 1.5, - "skip_existing": True, "save_metadata": True, "timeout": 3600, } diff --git a/backend/app/services/refetch_service.py b/backend/app/services/refetch_service.py index 41832bc..f2e941e 100644 --- a/backend/app/services/refetch_service.py +++ b/backend/app/services/refetch_service.py @@ -86,7 +86,7 @@ def attempt_refetch( if src is None: return {"status": "no_source"} - # Remove the bad copy so gallery-dl (skip_existing) re-fetches it on + # Remove the bad copy so gallery-dl's archive-skip re-fetches it on # the source re-check instead of skipping the still-present corrupt # file. try: diff --git a/backend/app/services/source_service.py b/backend/app/services/source_service.py index cd0dd70..83f2460 100644 --- a/backend/app/services/source_service.py +++ b/backend/app/services/source_service.py @@ -62,6 +62,7 @@ class SourceRecord: check_interval_override: int | None consecutive_failures: int next_check_at: str | None + backfill_runs_remaining: int def to_dict(self) -> dict: return { @@ -78,6 +79,7 @@ class SourceRecord: "check_interval_override": self.check_interval_override, "consecutive_failures": self.consecutive_failures, "next_check_at": self.next_check_at, + "backfill_runs_remaining": self.backfill_runs_remaining, } @@ -140,6 +142,7 @@ class SourceService: check_interval_override=source.check_interval_override, consecutive_failures=source.consecutive_failures or 0, next_check_at=nxt.isoformat() if nxt else None, + backfill_runs_remaining=source.backfill_runs_remaining or 0, ) async def _row_to_record(self, source: Source) -> SourceRecord: @@ -264,6 +267,25 @@ class SourceService: await self.session.commit() return await self._row_to_record(source) + async def set_backfill_runs( + self, source_id: int, runs: int, + ) -> SourceRecord: + """Plan #544: arm a source for backfill mode. The next `runs` + download runs will use gallery-dl's full-walk config (skip: True + + 30-min timeout) instead of the catch-up default. Runs must be + 1..10 — bigger is rejected to keep the operator from accidentally + setting a runaway budget.""" + if not isinstance(runs, int) or runs < 1 or runs > 10: + raise ValueError("runs must be an integer in [1, 10]") + source = (await self.session.execute( + select(Source).where(Source.id == source_id) + )).scalar_one_or_none() + if source is None: + raise LookupError(f"source id={source_id} not found") + source.backfill_runs_remaining = runs + await self.session.commit() + return await self._row_to_record(source) + async def delete(self, source_id: int) -> None: source = (await self.session.execute( select(Source).where(Source.id == source_id) diff --git a/frontend/src/components/subscriptions/SourceRow.vue b/frontend/src/components/subscriptions/SourceRow.vue index c30a45c..011645f 100644 --- a/frontend/src/components/subscriptions/SourceRow.vue +++ b/frontend/src/components/subscriptions/SourceRow.vue @@ -31,6 +31,12 @@ v-if="(source.consecutive_failures || 0) > 0" size="x-small" color="error" variant="tonal" label >{{ source.consecutive_failures }} + + backfill ({{ source.backfill_runs_remaining }}×) + 0 @@ -42,6 +48,16 @@ mdi-play Check now + + mdi-magnify-scan + + Deep scan — walk full history for next few runs + + @@ -431,6 +432,36 @@ async function onCheck(source) { } } +// Plan #544: arm a source for backfill mode (gallery-dl walks the full +// post history) for the next N download runs. Default 3 — enough budget +// to finish a deep creator without re-prompting the operator across +// timeout boundaries. The chip on the row reflects the remaining count. +async function onBackfill(source) { + const raw = globalThis.window?.prompt( + `Deep scan "${source.artist_name} (${source.platform})" — walk full history for the next how many download runs? (1–10, default 3)`, + '3', + ) + if (raw == null) return + const runs = parseInt(raw, 10) + if (!Number.isFinite(runs) || runs < 1 || runs > 10) { + toast({ text: 'Deep scan: runs must be 1–10', type: 'error' }) + return + } + try { + await store.setBackfill(source.id, runs, source.artist_id) + toast({ + text: `Deep scan armed for ${runs} run${runs === 1 ? '' : 's'}`, + type: 'success', + }) + await store.loadAll() + } catch (e) { + toast({ + text: `Deep scan failed: ${e?.detail || e?.message || e}`, + type: 'error', + }) + } +} + async function checkAll(group) { let ok = 0 let conflict = 0 diff --git a/frontend/src/stores/sources.js b/frontend/src/stores/sources.js index a04c5a0..2e9406e 100644 --- a/frontend/src/stores/sources.js +++ b/frontend/src/stores/sources.js @@ -85,6 +85,15 @@ export const useSourcesStore = defineStore('sources', () => { } } + // Plan #544: arm a source for backfill mode. The next `runs` download + // runs (default 3) walk gallery-dl's full post history instead of + // exiting early at the first contiguous archived block. + async function setBackfill(id, runs = 3, artistIdHint = null) { + const body = await api.post(`/api/sources/${id}/backfill`, { body: { runs } }) + _invalidate(artistIdHint ?? body.artist_id) + return body + } + function sourcesByArtistGrouped() { // returns [{artist: {id,name,slug}, sources: [...]}, ...] const arr = byArtist.value.get(null) ?? [] @@ -111,6 +120,7 @@ export const useSourcesStore = defineStore('sources', () => { loadAll, loadForArtist, create, update, remove, checkNow, + setBackfill, findOrCreateArtist, autocompleteArtist, loadScheduleStatus, sourcesByArtistGrouped, diff --git a/tests/test_api_sources.py b/tests/test_api_sources.py index 28d1a11..c72c8b9 100644 --- a/tests/test_api_sources.py +++ b/tests/test_api_sources.py @@ -193,3 +193,61 @@ async def test_list_derives_next_check_at_when_last_checked_set( ) assert target["next_check_at"] is not None assert "T" in target["next_check_at"] # ISO 8601 + + +# --- Plan #544: POST /api/sources/{id}/backfill ---------------------------- + + +@pytest.mark.asyncio +async def test_backfill_endpoint_arms_source(client, artist, db): + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-backfill", enabled=True, + ) + db.add(src) + await db.commit() + sid = src.id + + resp = await client.post(f"/api/sources/{sid}/backfill", json={"runs": 5}) + assert resp.status_code == 200 + body = await resp.get_json() + assert body["backfill_runs_remaining"] == 5 + + # GET reflects the new state. + one = await client.get(f"/api/sources/{sid}") + assert (await one.get_json())["backfill_runs_remaining"] == 5 + + +@pytest.mark.asyncio +async def test_backfill_endpoint_defaults_to_three(client, artist, db): + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-backfill-default", enabled=True, + ) + db.add(src) + await db.commit() + resp = await client.post(f"/api/sources/{src.id}/backfill", json={}) + body = await resp.get_json() + assert body["backfill_runs_remaining"] == 3 + + +@pytest.mark.asyncio +async def test_backfill_endpoint_rejects_out_of_range(client, artist, db): + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-backfill-bad", enabled=True, + ) + db.add(src) + await db.commit() + bad = await client.post(f"/api/sources/{src.id}/backfill", json={"runs": 0}) + assert bad.status_code == 400 + too_big = await client.post( + f"/api/sources/{src.id}/backfill", json={"runs": 99} + ) + assert too_big.status_code == 400 + + +@pytest.mark.asyncio +async def test_backfill_endpoint_404_when_source_missing(client): + resp = await client.post("/api/sources/999999/backfill", json={"runs": 3}) + assert resp.status_code == 404 diff --git a/tests/test_download_service.py b/tests/test_download_service.py index 15ad337..ebced00 100644 --- a/tests/test_download_service.py +++ b/tests/test_download_service.py @@ -370,3 +370,179 @@ async def test_finalize_skipped_preserves_failures_clears_error(db): assert row.consecutive_failures == 4 assert row.last_error is None assert row.last_checked_at is not None + + +# --- Plan #544: backfill lifecycle + PARTIAL → status=ok ------------------- + + +@pytest.mark.asyncio +async def test_backfill_decrements_after_run( + db, db_sync, tmp_path, seed_artist_and_source, +): + """When backfill_runs_remaining > 0 going in, a non-clean / non-empty + run decrements by 1 — operator gets N runs to complete the deep scan + before tick mode resumes.""" + from backend.app.services.download_service import DownloadService + from backend.app.services.importer import Importer + + _artist, source = seed_artist_and_source + source.backfill_runs_remaining = 3 + await db.commit() + + images_root = tmp_path / "images" + f1 = images_root / "alice" / "patreon" / "post" / "a.jpg" + _make_jpg(f1, split="h") + + fake_gdl = _fake_gdl_with_result(_make_fake_dl_result( + success=True, written_paths=[str(f1)], files_downloaded=1, + stdout=f"{f1}\n", + )) + + sync_settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + importer = Importer( + session=db_sync, images_root=images_root, import_root=images_root, + thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings, + ) + cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64")) + svc = DownloadService( + async_session=db, sync_session=db_sync, + gdl=fake_gdl, importer=importer, cred_service=cred_service, + ) + await svc.download_source(source.id) + + remaining = (await db.execute( + select(Source.backfill_runs_remaining).where(Source.id == source.id) + )).scalar_one() + assert remaining == 2 + + +@pytest.mark.asyncio +async def test_backfill_auto_resets_on_clean_zero_files( + db, db_sync, tmp_path, seed_artist_and_source, +): + """A clean run (rc=0) that downloaded zero files means the backfill + queue drained — reset to 0 immediately instead of wasting the rest of + the N-run budget on no-op walks.""" + from backend.app.services.download_service import DownloadService + from backend.app.services.importer import Importer + + _artist, source = seed_artist_and_source + source.backfill_runs_remaining = 3 + await db.commit() + + fake_result = _make_fake_dl_result( + success=True, written_paths=[], files_downloaded=0, + ) + fake_gdl = _fake_gdl_with_result(fake_result) + + sync_settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + importer = Importer( + session=db_sync, images_root=tmp_path / "images", + import_root=tmp_path / "images", + thumbnailer=Thumbnailer(images_root=tmp_path / "images"), + settings=sync_settings, + ) + cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64")) + svc = DownloadService( + async_session=db, sync_session=db_sync, + gdl=fake_gdl, importer=importer, cred_service=cred_service, + ) + await svc.download_source(source.id) + + remaining = (await db.execute( + select(Source.backfill_runs_remaining).where(Source.id == source.id) + )).scalar_one() + assert remaining == 0 + + +@pytest.mark.asyncio +async def test_tick_mode_does_not_touch_backfill_counter( + db, db_sync, tmp_path, seed_artist_and_source, +): + """When backfill_runs_remaining is already 0, downloads don't go + negative or otherwise mutate the counter.""" + from backend.app.services.download_service import DownloadService + from backend.app.services.importer import Importer + + _artist, source = seed_artist_and_source + assert source.backfill_runs_remaining == 0 + + fake_gdl = _fake_gdl_with_result(_make_fake_dl_result( + success=True, written_paths=[], files_downloaded=0, + )) + + sync_settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + importer = Importer( + session=db_sync, images_root=tmp_path / "images", + import_root=tmp_path / "images", + thumbnailer=Thumbnailer(images_root=tmp_path / "images"), + settings=sync_settings, + ) + cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64")) + svc = DownloadService( + async_session=db, sync_session=db_sync, + gdl=fake_gdl, importer=importer, cred_service=cred_service, + ) + await svc.download_source(source.id) + + remaining = (await db.execute( + select(Source.backfill_runs_remaining).where(Source.id == source.id) + )).scalar_one() + assert remaining == 0 + + +@pytest.mark.asyncio +async def test_partial_error_type_maps_to_ok_status( + db, db_sync, tmp_path, seed_artist_and_source, +): + """A non-zero gallery-dl exit accompanied by PARTIAL error_type + (real files were downloaded before the run was cut short) is treated + as status=ok by the orchestrator — next tick continues.""" + from backend.app.services.download_service import DownloadService + from backend.app.services.gallery_dl import ErrorType + from backend.app.services.importer import Importer + + _artist, source = seed_artist_and_source + + images_root = tmp_path / "images" + f1 = images_root / "alice" / "patreon" / "post" / "a.jpg" + _make_jpg(f1, split="h") + + fake_result = _make_fake_dl_result( + success=False, written_paths=[str(f1)], files_downloaded=1, + error_type=ErrorType.PARTIAL, + error_message="Downloaded 1 file; run did not complete in budget", + stdout=f"{f1}\n", + ) + fake_gdl = _fake_gdl_with_result(fake_result) + + sync_settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + importer = Importer( + session=db_sync, images_root=images_root, import_root=images_root, + thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings, + ) + cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64")) + svc = DownloadService( + async_session=db, sync_session=db_sync, + gdl=fake_gdl, importer=importer, cred_service=cred_service, + ) + event_id = await svc.download_source(source.id) + + ev = (await db.execute( + select(DownloadEvent).where(DownloadEvent.id == event_id) + )).scalar_one() + assert ev.status == "ok" + assert ev.error is None + # And the source's failure counter isn't bumped — PARTIAL isn't a failure. + src_after = (await db.execute( + select(Source.consecutive_failures).where(Source.id == source.id) + )).scalar_one() + assert src_after == 0 diff --git a/tests/test_gallery_dl_service.py b/tests/test_gallery_dl_service.py index af490d1..ef196b5 100644 --- a/tests/test_gallery_dl_service.py +++ b/tests/test_gallery_dl_service.py @@ -249,3 +249,69 @@ def test_truncate_log_caps_large_text(gdl): def test_truncate_log_passes_short_text(gdl): text = "small\n" assert gdl._truncate_log(text) == text + + +# --- Plan #544: tick/backfill skip emission + PARTIAL classifier ------------ + + +def test_build_config_emits_tick_skip_value(gdl): + """Tick mode emits gallery-dl's exit:20 to short-circuit catch-up + scans once 20 contiguous archived items are seen.""" + from backend.app.services.gallery_dl import TICK_SKIP_VALUE + cfg = gdl._build_config_for_source( + platform="patreon", + source_config=SourceConfig(), + artist_slug="alice", + skip_value=TICK_SKIP_VALUE, + ) + assert cfg["extractor"]["skip"] == "exit:20" + + +def test_build_config_emits_backfill_skip_value(gdl): + """Backfill mode keeps gallery-dl's default skip=True so it walks the + full post history.""" + from backend.app.services.gallery_dl import BACKFILL_SKIP_VALUE + cfg = gdl._build_config_for_source( + platform="patreon", + source_config=SourceConfig(), + artist_slug="alice", + skip_value=BACKFILL_SKIP_VALUE, + ) + assert cfg["extractor"]["skip"] is True + + +def test_categorize_partial_when_files_downloaded_then_nonzero_exit(gdl): + """Non-zero exit + ≥1 file downloaded + no source-level error → + PARTIAL. Downstream maps to status=ok because the next tick continues.""" + # stdout has a single downloaded file (line starting with `/`); no + # error-class indicators in stderr; non-zero return code. + etype, msg = gdl._categorize_error( + return_code=1, + stdout="/images/alice/patreon/post1/img.jpg\n", + stderr="", + ) + assert etype == ErrorType.PARTIAL + assert "Downloaded 1 file" in msg + + +def test_categorize_unknown_when_no_files_and_no_pattern(gdl): + """PARTIAL only kicks in when files were actually written. A non-zero + exit with no downloads and no recognized pattern still falls through + to UNKNOWN_ERROR — distinct from the partial-success case.""" + etype, _ = gdl._categorize_error( + return_code=2, + stdout="", + stderr="some unrecognized noise\n", + ) + assert etype == ErrorType.UNKNOWN_ERROR + + +def test_categorize_tier_limited_wins_over_partial(gdl): + """A run that hit tier-limited warnings AND downloaded some files + classifies as TIER_LIMITED (the more specific category), not PARTIAL.""" + stdout = "/images/alice/patreon/post1/img.jpg\n" + stderr = "[patreon][warning] Not allowed to view post 123\n" + etype, _ = gdl._categorize_error( + return_code=1, stdout=stdout, stderr=stderr, + ) + assert etype == ErrorType.TIER_LIMITED diff --git a/tests/test_source_service.py b/tests/test_source_service.py index 14e6e57..baeb096 100644 --- a/tests/test_source_service.py +++ b/tests/test_source_service.py @@ -169,3 +169,47 @@ async def test_list_hides_sidecar_synthetic_anchors(db): assert {s.url for s in everything} >= { "https://patreon.com/alice", "sidecar:patreon:alice", } + + +# --- Plan #544: backfill counter ------------------------------------------- + + +@pytest.mark.asyncio +async def test_set_backfill_runs_arms_source(db): + """The service method sets backfill_runs_remaining and returns the + updated record so the API can echo it back.""" + artist = await _artist(db, "Alice") + svc = SourceService(db) + rec = await svc.create( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice", + ) + assert rec.backfill_runs_remaining == 0 + updated = await svc.set_backfill_runs(rec.id, 5) + assert updated.backfill_runs_remaining == 5 + + db_value = (await db.execute( + select(Source.backfill_runs_remaining).where(Source.id == rec.id) + )).scalar_one() + assert db_value == 5 + + +@pytest.mark.asyncio +async def test_set_backfill_runs_rejects_out_of_range(db): + artist = await _artist(db, "Alice") + svc = SourceService(db) + rec = await svc.create( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice", + ) + with pytest.raises(ValueError): + await svc.set_backfill_runs(rec.id, 0) + with pytest.raises(ValueError): + await svc.set_backfill_runs(rec.id, 11) + + +@pytest.mark.asyncio +async def test_set_backfill_runs_raises_when_source_missing(db): + svc = SourceService(db) + with pytest.raises(LookupError): + await svc.set_backfill_runs(99999, 3)