diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index d33e317..56c5cd6 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -122,23 +122,30 @@ async def delete_source(source_id: int): @sources_bp.route("//backfill", methods=["POST"]) async def set_backfill(source_id: int): - """Plan #693: start or stop a run-until-done backfill. Body: - `{"action": "start" | "stop"}` (default "start"). 'start' walks the full - post history in time-boxed chunks until it reaches the bottom (then the - source shows 'complete'); 'stop' cancels back to tick mode. Returns the - updated source dict (incl. backfill_state / backfill_chunks).""" + """Plan #693/#697: start/stop a run-until-done backfill, or start a recovery. + Body: `{"action": "start" | "stop" | "recover"}` (default "start"). 'start' + walks the full post history in time-boxed chunks until it reaches the bottom + (then the source shows 'complete'); 'recover' is the same walk but bypasses + the Patreon seen-ledger to re-fetch dropped-and-deleted near-dups under the + current pHash threshold; 'stop' cancels either back to tick mode. Returns the + updated source dict (incl. backfill_state / backfill_chunks / + backfill_bypass_seen).""" payload = await request.get_json(silent=True) or {} action = payload.get("action", "start") - if action not in ("start", "stop"): - return _bad("invalid_action", detail="action must be 'start' or 'stop'") + if action not in ("start", "stop", "recover"): + return _bad( + "invalid_action", + detail="action must be 'start', 'stop', or 'recover'", + ) async with get_session() as session: try: svc = SourceService(session) - record = ( - await svc.start_backfill(source_id) - if action == "start" - else await svc.stop_backfill(source_id) - ) + if action == "start": + record = await svc.start_backfill(source_id) + elif action == "recover": + record = await svc.start_recovery(source_id) + else: + record = await svc.stop_backfill(source_id) except LookupError: return _bad("not_found", status=404) return jsonify(record.to_dict()) diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index c84d657..7a86917 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -44,11 +44,14 @@ from .scheduler_service import set_platform_cooldown log = logging.getLogger(__name__) +# Vanity → campaign-id resolution is still needed by the native Patreon +# ingester (phase 2 resolves the campaign id before the walk). gallery-dl's +# reactive campaign-id retry + the `id:` effective-URL rewrite were removed at +# the #697 cutover (Patreon no longer flows through gallery-dl). _PATREON_VANITY_RE = re.compile( r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)", re.IGNORECASE, ) -_CAMPAIGN_ID_FAILURE_PATTERN = "failed to extract campaign id" def _extract_patreon_vanity(url: str) -> str | None: @@ -56,16 +59,6 @@ def _extract_patreon_vanity(url: str) -> str | None: return m.group(1) if m else None -def _looks_like_campaign_id_failure(stdout: str, stderr: str) -> bool: - return _CAMPAIGN_ID_FAILURE_PATTERN in f"{stdout}\n{stderr}".lower() - - -def _effective_url(platform: str, source_url: str, overrides: dict) -> str: - if platform == "patreon" and overrides.get("patreon_campaign_id"): - return f"https://www.patreon.com/id:{overrides['patreon_campaign_id']}" - return source_url - - class DownloadService: """Async orchestrator. The Celery task runs `asyncio.run(svc.download_source(N))`. @@ -160,11 +153,8 @@ class DownloadService: ctx, source_config, mode, ) else: - effective_url = _effective_url( - ctx["platform"], ctx["url"], ctx["config_overrides"] or {} - ) dl_result = await self.gdl.download( - url=effective_url, + url=ctx["url"], artist_slug=ctx["artist_slug"], platform=ctx["platform"], source_config=source_config, diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 9d3dc92..bab1d63 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -107,12 +107,14 @@ class SourceConfig: `resume_cursor` is RUNTIME state, not persisted operator config: the cursor-paged backfill checkpoint (see parse_last_cursor + the - download_service backfill lifecycle). When set on a patreon backfill - run, gallery-dl resumes its newest→oldest walk from that pagination - cursor instead of restarting from the top. It is threaded by - download_service from config_overrides["_backfill_cursor"]; SourceConfig - deliberately does NOT read it in from_dict (config_overrides also holds - operator config, and resume_cursor must only apply in backfill mode). + download_service backfill lifecycle). It is consumed only by the native + Patreon ingester (the sole cursor-paged platform; gallery-dl's Patreon path + was removed at the #697 cutover): on a backfill/recovery run the ingester + resumes its newest→oldest walk from that pagination cursor instead of + restarting from the top. download_service threads it from + config_overrides["_backfill_cursor"]; SourceConfig deliberately does NOT read + it in from_dict (config_overrides also holds operator config, and + resume_cursor must only apply in backfill/recovery mode). """ content_types: list[str] = field(default_factory=lambda: ["all"]) sleep: float | None = None @@ -170,13 +172,14 @@ def _summarize_validation_failures(failures: list[dict]) -> str: return f"{n} files quarantined ({top_count}× {top_reason}, mixed)" -# gallery-dl logs its pagination cursor (when extractor.*.cursor is truthy) -# as `Cursor: ` — once per fetched page, at debug verbosity. The LAST -# such line is the furthest-progressed page, i.e. the resume point. We scan -# both streams (the debug log lands on stderr, but be defensive) and take the -# final match; passing it back as extractor.patreon.cursor resumes the walk. -# Survives a timed-out run too: the TimeoutExpired path returns the partial -# stdout/stderr, so an interrupted backfill still yields its last cursor. +# The native Patreon ingester emits its pagination cursor as `Cursor: ` +# — one line per fetched page — into the DownloadResult stdout (mirroring the +# convention gallery-dl used before the #697 cutover, so the backfill lifecycle +# stayed unchanged). The LAST such line is the furthest-progressed page, i.e. +# the resume point. We scan both streams (be defensive) and take the final +# match; download_service checkpoints it as the next chunk's resume_cursor. +# Survives a time-boxed chunk too: the ingester emits the cursor for a page when +# it STARTS it, so an interrupted walk still yields its last cursor. _CURSOR_RE = re.compile(r"Cursor:\s*(\S+)") @@ -218,16 +221,10 @@ class GalleryDLService: "permission denied", "tier required", "pledge required", ] - # Per-platform defaults. Lifted from GS — same six platforms FC supports. + # Per-platform defaults for the gallery-dl-backed platforms. Patreon was + # removed at the plan-#697 cutover — it now uses the native ingester + # (services/patreon_ingester.py), not gallery-dl. PLATFORM_DEFAULTS = { - "patreon": { - "content_types": ["images", "image_large", "attachments", "postfile", "content"], - "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], - "filename": "{num:>02}_{filename}.{extension}", - "videos": True, - "embeds": True, - "cursor": True, - }, "subscribestar": { "content_types": ["all"], "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], @@ -312,25 +309,11 @@ class GalleryDLService: # transfer cap — large GIFs that keep streaming are unaffected. "retries": 2, "timeout": 60.0, - # Forward Patreon as Referer/Origin to yt-dlp when it - # fetches video manifests. Operator-flagged 2026-06-01 - # (DaferQ patreon): video posts hosted on Mux carry a JWT - # playback restriction that checks Referer/Origin on every - # request — not just the token signature. gallery-dl's - # HEAD probe to stream.mux.com returns 200 (the token is - # valid), but yt-dlp's actual GET-with-Range to fetch the - # m3u8 manifest 403s because yt-dlp sends its own default - # Referer, which Mux's policy rejects. Forcing the right - # headers fixes the headers-only case; Mux IP-range - # restrictions are unfixable from here. - "ytdl": { - "raw-options": { - "http_headers": { - "Referer": "https://www.patreon.com/", - "Origin": "https://www.patreon.com", - }, - }, - }, + # NOTE: the Patreon/Mux yt-dlp Referer/Origin forwarding lived + # here until the plan-#697 cutover. It was Patreon-specific (and + # would have wrongly tagged the other platforms' yt-dlp fetches); + # Patreon video is now handled by the native ingester's + # downloader (patreon_downloader._VIDEO_HEADERS), so it's gone. }, "output": {"progress": True}, } @@ -382,23 +365,7 @@ class GalleryDLService: platform_section = config["extractor"].setdefault(platform, {}) - if platform == "patreon": - if "all" in source_config.content_types: - platform_section["files"] = [ - "images", "image_large", "attachments", "postfile", "content", - ] - else: - platform_section["files"] = source_config.content_types - # Cursor-paged backfill: resume the newest→oldest walk from the - # checkpoint saved by the previous run instead of restarting from - # the top. Without this, a large catalog (Anduo's weekly - # image-dense Reports) never reaches its frontier inside the - # subprocess budget — the HEAD walk alone times out, 0 files - # written, no progress (event #40411). The PLATFORM_DEFAULTS leave - # `cursor: True` (log-only) for the first/fresh run. - if source_config.resume_cursor: - platform_section["cursor"] = source_config.resume_cursor - elif platform == "hentaifoundry": + if platform == "hentaifoundry": if "pictures" in source_config.content_types or "all" in source_config.content_types: platform_section["include"] = "all" diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index bd920ce..7103ec8 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -50,10 +50,11 @@ _TITLE_MAX = 40 _TIMEOUT_SECONDS = 120.0 _CHUNK = 1 << 16 -# Same Referer/Origin gallery-dl forwards to yt-dlp for Mux-hosted Patreon -# video (gallery_dl.py _get_default_config -> downloader.ytdl.raw-options). -# Mux's JWT playback policy checks Referer/Origin on every request, so yt-dlp -# must send Patreon's, not its own default. +# Referer/Origin yt-dlp must send for Mux-hosted Patreon video. Mux's JWT +# playback policy checks Referer/Origin on every request, so yt-dlp must send +# Patreon's, not its own default. (gallery-dl forwarded the same headers before +# the #697 cutover removed its Patreon path; this is now the only place they +# live.) _VIDEO_HEADERS = { "Referer": "https://www.patreon.com/", "Origin": "https://www.patreon.com", diff --git a/backend/app/services/source_service.py b/backend/app/services/source_service.py index 0192a9c..67b2cf2 100644 --- a/backend/app/services/source_service.py +++ b/backend/app/services/source_service.py @@ -67,6 +67,9 @@ class SourceRecord: # plan #693: derived from config_overrides for the UI badge. backfill_state: str | None # "running" | "complete" | "stalled" | None (idle) backfill_chunks: int + # plan #697: a running deep-walk that bypasses the Patreon seen-ledger + # (recovery) vs. a normal backfill. Lets the badge label it "Recovering". + backfill_bypass_seen: bool def to_dict(self) -> dict: return { @@ -87,6 +90,7 @@ class SourceRecord: "backfill_runs_remaining": self.backfill_runs_remaining, "backfill_state": self.backfill_state, "backfill_chunks": self.backfill_chunks, + "backfill_bypass_seen": self.backfill_bypass_seen, } @@ -162,6 +166,7 @@ class SourceService: backfill_runs_remaining=source.backfill_runs_remaining or 0, backfill_state=co.get("_backfill_state"), backfill_chunks=int(co.get("_backfill_chunks", 0)), + backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")), ) async def _row_to_record(self, source: Source) -> SourceRecord: @@ -319,6 +324,34 @@ class SourceService: await self.session.commit() return await self._row_to_record(source) + async def start_recovery(self, source_id: int) -> SourceRecord: + """Plan #697: arm a RECOVERY walk — a backfill that bypasses the Patreon + seen-ledger so deliberately-dropped-and-deleted near-dups get re-fetched + and re-evaluated under the CURRENT pHash threshold (tier-2 disk still + spares files we kept). Reuses the entire #693 backfill state machine + (time-boxed chunks, cursor checkpoint, complete/stall lifecycle) plus the + `_backfill_bypass_seen` flag that flips download mode to recovery. Clears + any prior cursor/chunk/stall state so it walks fresh from the top. The + flag is cleared on completion (download_service) and on stop. + + Recovery is Patreon-only (the seen-ledger is Patreon's); for other + platforms the flag is inert (download_service ignores it) and the walk + runs as a plain backfill. The UI gates the action to Patreon sources.""" + source = (await self.session.execute( + select(Source).where(Source.id == source_id) + )).scalar_one_or_none() + if source is None: + raise LookupError(f"source id={source_id} not found") + co = dict(source.config_overrides or {}) + co["_backfill_state"] = "running" + co["_backfill_bypass_seen"] = True + for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks"): + co.pop(k, None) + source.config_overrides = co + source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS + await self.session.commit() + return await self._row_to_record(source) + async def stop_backfill(self, source_id: int) -> SourceRecord: """Plan #693: cancel an in-progress backfill — back to idle/tick mode. Clears the running state + cursor/chunk/stall bookkeeping.""" diff --git a/frontend/src/components/subscriptions/SourceCard.vue b/frontend/src/components/subscriptions/SourceCard.vue index 7b8f9bc..4312d8f 100644 --- a/frontend/src/components/subscriptions/SourceCard.vue +++ b/frontend/src/components/subscriptions/SourceCard.vue @@ -27,7 +27,8 @@ Backfilling{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }} + >{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling' + }}{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }} {{ source.backfill_state === 'running' ? 'mdi-stop' : 'mdi-magnify-scan' }} - {{ source.backfill_state === 'running' ? 'Stop backfill' : 'Backfill full history' }} + {{ source.backfill_state === 'running' + ? (source.backfill_bypass_seen ? 'Stop recovery' : 'Stop backfill') + : 'Backfill full history' }} + + + + mdi-backup-restore + + Recover — re-fetch dropped near-dups & re-evaluate under the current threshold @@ -83,7 +96,7 @@ const props = defineProps({ checking: { type: Boolean, default: false }, warningThreshold: { type: Number, default: 5 }, }) -const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill']) +const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover']) function onToggleEnabled(value) { emit('toggle', { source: props.source, enabled: value }) diff --git a/frontend/src/components/subscriptions/SourceRow.vue b/frontend/src/components/subscriptions/SourceRow.vue index 9fe412e..d74caaf 100644 --- a/frontend/src/components/subscriptions/SourceRow.vue +++ b/frontend/src/components/subscriptions/SourceRow.vue @@ -34,7 +34,8 @@ Backfilling{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }} + >{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling' + }}{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }} {{ source.backfill_state === 'running' ? 'mdi-stop' : 'mdi-magnify-scan' }} {{ source.backfill_state === 'running' - ? 'Stop backfill' + ? (source.backfill_bypass_seen ? 'Stop recovery' : 'Stop backfill') : 'Backfill — walk the full post history until complete' }} + + mdi-backup-restore + + Recover — re-fetch dropped near-dups & re-evaluate under the current threshold + + @@ -251,6 +252,7 @@ @toggle="toggleSourceEnabled" @check="onCheck" @backfill="onBackfill" + @recover="onRecover" />
No sources yet. Tap + to add one. @@ -554,6 +556,22 @@ async function onBackfill(source) { } } +// Plan #697: arm a recovery walk (Patreon-only) — re-fetches dropped-and-deleted +// near-dups and re-evaluates them under the current pHash threshold. Reuses the +// backfill lifecycle/badge; stop via the same Stop control (onBackfill). +async function onRecover(source) { + try { + await store.recoverSource(source.id, source.artist_id) + toast({ text: `Recovery started for ${source.artist_name}`, type: 'success' }) + await store.loadAll() + } catch (e) { + toast({ + text: `Recovery start failed: ${e?.detail || e?.message || e}`, + type: 'error', + }) + } +} + async function checkAll(group) { let ok = 0 let conflict = 0 diff --git a/frontend/src/stores/sources.js b/frontend/src/stores/sources.js index 5eb90fe..e7916bc 100644 --- a/frontend/src/stores/sources.js +++ b/frontend/src/stores/sources.js @@ -98,6 +98,14 @@ export const useSourcesStore = defineStore('sources', () => { _invalidate(artistIdHint ?? body.artist_id) return body } + // Plan #697: arm a recovery walk — a backfill that bypasses the Patreon + // seen-ledger to re-fetch dropped-and-deleted near-dups and re-evaluate them + // under the current pHash threshold. Stop with stopBackfill (shared lifecycle). + async function recoverSource(id, artistIdHint = null) { + const body = await api.post(`/api/sources/${id}/backfill`, { body: { action: 'recover' } }) + _invalidate(artistIdHint ?? body.artist_id) + return body + } function sourcesByArtistGrouped() { // returns [{artist: {id,name,slug}, sources: [...]}, ...] @@ -127,6 +135,7 @@ export const useSourcesStore = defineStore('sources', () => { checkNow, startBackfill, stopBackfill, + recoverSource, findOrCreateArtist, autocompleteArtist, loadScheduleStatus, sourcesByArtistGrouped, diff --git a/tests/test_api_sources.py b/tests/test_api_sources.py index 3a13d97..438b305 100644 --- a/tests/test_api_sources.py +++ b/tests/test_api_sources.py @@ -250,3 +250,28 @@ async def test_backfill_endpoint_rejects_bad_action(client, artist, db): async def test_backfill_endpoint_404_when_source_missing(client): resp = await client.post("/api/sources/999999/backfill", json={"action": "start"}) assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_backfill_endpoint_recover_arms_bypass(client, artist, db): + """Plan #697: action='recover' arms a backfill that bypasses the Patreon + seen-ledger; the response exposes backfill_bypass_seen=True so the UI badge + can label it 'Recovering'. Stop clears it.""" + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-recover", enabled=True, + ) + db.add(src) + await db.commit() + sid = src.id + + resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "recover"}) + assert resp.status_code == 200 + body = await resp.get_json() + assert body["backfill_state"] == "running" + assert body["backfill_bypass_seen"] is True + + stopped = await client.post(f"/api/sources/{sid}/backfill", json={"action": "stop"}) + stopped_body = await stopped.get_json() + assert stopped_body["backfill_state"] is None + assert stopped_body["backfill_bypass_seen"] is False diff --git a/tests/test_gallery_dl_service.py b/tests/test_gallery_dl_service.py index 5eadee9..be06108 100644 --- a/tests/test_gallery_dl_service.py +++ b/tests/test_gallery_dl_service.py @@ -260,7 +260,7 @@ def test_build_config_emits_tick_skip_value(gdl): scans once 20 contiguous archived items are seen.""" from backend.app.services.gallery_dl import TICK_SKIP_VALUE cfg = gdl._build_config_for_source( - platform="patreon", + platform="subscribestar", source_config=SourceConfig(), artist_slug="alice", skip_value=TICK_SKIP_VALUE, @@ -273,7 +273,7 @@ def test_build_config_emits_backfill_skip_value(gdl): full post history.""" from backend.app.services.gallery_dl import BACKFILL_SKIP_VALUE cfg = gdl._build_config_for_source( - platform="patreon", + platform="subscribestar", source_config=SourceConfig(), artist_slug="alice", skip_value=BACKFILL_SKIP_VALUE, @@ -318,23 +318,17 @@ def test_categorize_tier_limited_wins_over_partial(gdl): assert etype == ErrorType.TIER_LIMITED -def test_default_config_forwards_patreon_referer_to_ytdl(gdl): - """Operator-flagged 2026-06-01: Mux video playback restrictions reject - yt-dlp's manifest fetch when Referer/Origin don't match Patreon. The - fix is a static `downloader.ytdl.raw-options.http_headers` block, so - it's enough to assert the keys land in the default config.""" - cfg = gdl._get_default_config() - headers = cfg["downloader"]["ytdl"]["raw-options"]["http_headers"] - assert headers["Referer"] == "https://www.patreon.com/" - assert headers["Origin"] == "https://www.patreon.com" - - -# --- Cursor-paged backfill (plan #689) ------------------------------------- +# --- Cursor parsing (plan #689 / native ingester #697) --------------------- +# parse_last_cursor is retained post-cutover: the native Patreon ingester emits +# `Cursor: ` per page into DownloadResult.stdout and download_service +# checkpoints the last one. (The gallery-dl Patreon `files`/`cursor` config and +# the Mux yt-dlp Referer block were removed at the #697 cutover, along with the +# tests that pinned them.) def test_parse_last_cursor_returns_last_match_across_streams(): - # gallery-dl logs `Cursor: ` per page (debug → stderr); the last - # is the furthest-progressed resume point. + # One `Cursor: ` line per page; the last is the furthest-progressed + # resume point. stderr = ( "[patreon][debug] Cursor: 03:AAAA:bbb\n" "[urllib3] GET ...\n" @@ -347,18 +341,3 @@ def test_parse_last_cursor_scans_stdout_too_and_none_when_absent(): assert parse_last_cursor("Cursor: 99:ZZZ", "") == "99:ZZZ" assert parse_last_cursor("no marker here", "still nothing") is None assert parse_last_cursor("", "") is None - - -def test_build_config_patreon_resume_cursor_overrides_default(gdl): - # No resume cursor → PLATFORM_DEFAULTS leaves the log-only `cursor: True`. - fresh = gdl._build_config_for_source( - "patreon", SourceConfig(), "alice", skip_value=True, - ) - assert fresh["extractor"]["patreon"]["cursor"] is True - - # Resume cursor set → gallery-dl restarts the walk from that page. - resumed = gdl._build_config_for_source( - "patreon", SourceConfig(resume_cursor="03:CCCC:ddd"), "alice", - skip_value=True, - ) - assert resumed["extractor"]["patreon"]["cursor"] == "03:CCCC:ddd" diff --git a/tests/test_source_service.py b/tests/test_source_service.py index 7e53850..ce75349 100644 --- a/tests/test_source_service.py +++ b/tests/test_source_service.py @@ -225,6 +225,38 @@ async def test_stop_backfill_returns_to_idle(db): assert "_backfill_state" not in (co or {}) +@pytest.mark.asyncio +async def test_start_recovery_arms_bypass_flag(db): + """Plan #697: start_recovery arms the backfill state machine PLUS the + _backfill_bypass_seen flag (recovery), surfaced on the record so the UI badge + can label it 'Recovering'. Clears any prior checkpoint state.""" + from backend.app.services.source_service import BACKFILL_MAX_CHUNKS + + artist = await _artist(db, "Alice") + svc = SourceService(db) + rec = await svc.create( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice", + ) + updated = await svc.start_recovery(rec.id) + assert updated.backfill_state == "running" + assert updated.backfill_bypass_seen is True + assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS + + co = (await db.execute( + select(Source.config_overrides).where(Source.id == rec.id) + )).scalar_one() + assert co.get("_backfill_bypass_seen") is True + + # Stop clears the bypass flag too (shared lifecycle). + stopped = await svc.stop_backfill(rec.id) + assert stopped.backfill_bypass_seen is False + co2 = (await db.execute( + select(Source.config_overrides).where(Source.id == rec.id) + )).scalar_one() + assert "_backfill_bypass_seen" not in (co2 or {}) + + @pytest.mark.asyncio async def test_start_backfill_raises_when_source_missing(db): svc = SourceService(db)