From 1c2dc7659a41fe70c9c37d27ea5d26222805c49a Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 19:16:20 -0400 Subject: [PATCH 01/26] =?UTF-8?q?feat(patreon):=20native=20JSON-API=20clie?= =?UTF-8?q?nt=20=E2=80=94=20ingester=20build=20step=201=20(plan=20#697)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PatreonClient: cookie-auth requests session, /api/posts cursor pagination, JSON:API included flattening, per-post media extraction (images/image_large/ attachments/postfile/content) with filehash dedup, loud drift detection. Zero per-file HEADs — every media URL+file_name comes from the API. Not yet wired into download_service (later step). Pure-parsing unit tests + fixture. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/patreon_client.py | 477 ++++++++++++++++++++++++ tests/fixtures/patreon_posts_page1.json | 121 ++++++ tests/test_patreon_client.py | 190 ++++++++++ 3 files changed, 788 insertions(+) create mode 100644 backend/app/services/patreon_client.py create mode 100644 tests/fixtures/patreon_posts_page1.json create mode 100644 tests/test_patreon_client.py diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py new file mode 100644 index 0000000..28bfab4 --- /dev/null +++ b/backend/app/services/patreon_client.py @@ -0,0 +1,477 @@ +"""Native Patreon JSON:API client (build step 1 of the native ingester). + +Clean-room reimplementation of the Patreon `/api/posts` read path. This is a +plain, synchronous client over `requests` — the orchestrator already wraps +sync importer calls, so nothing here needs to be async (mirrors +patreon_resolver.py, which wraps its sync lookup in run_in_executor at the +call site). + +Scope (build step 1): fetch + page + parse only. This module is NOT wired into +download_service yet — that is a later step. The public surface here exists so +the later step can drive it: + - PatreonClient(cookies_path).iter_posts(campaign_id) + → (post, included_index, page_cursor) + - extract_media(post, included_index) → list[MediaItem] + - parse_cursor_from_url(url) → cursor + +Drift detection is loud on purpose: Patreon ships JSON:API and the shapes we +depend on (top-level `data`, media resources carrying `file_name`/`url`) are +the contract. If a response comes back as an HTML login page or a media +resource is missing the fields we resolve against, we raise PatreonDriftError +rather than silently yielding empty media — so the later import step surfaces +"Patreon changed something" instead of "creator has no posts". + +FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. +""" + +from __future__ import annotations + +import http.cookiejar +import logging +import os +import re +from collections.abc import Iterator +from dataclasses import dataclass +from html import unescape +from pathlib import Path +from urllib.parse import parse_qs, urlsplit + +import requests + +log = logging.getLogger(__name__) + +_POSTS_URL = "https://www.patreon.com/api/posts" +_USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" +) +_TIMEOUT_SECONDS = 30.0 + +# JSON:API request contract (observed from real traffic — see module plan). +_INCLUDE = ( + "campaign,access_rules,attachments,attachments_media,audio,images,media," + "native_video_insights,user,user_defined_tags,ti_checks" +) +_FIELDS_POST = ( + "content,post_file,image,post_type,published_at,title,url,patreon_url," + "current_user_can_view" +) +_FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name" +_FIELDS_CAMPAIGN = "name,url" + +# A CDN download URL embeds a 32-char hex (MD5) path segment; that segment is +# Patreon's stable per-file identity and is what we dedup + ledger against. +# Same role gallery-dl's _filehash plays. Match the FIRST 32-hex run anywhere +# in the URL (path or query); real Patreon CDN URLs carry exactly one. +_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})") + +# Inline post `content` is HTML; images are emitted as . +# Pull every src; downstream dedup collapses any that duplicate a gallery item +# by filehash. Tolerant of attribute ordering and single/double quotes. +_CONTENT_IMG_RE = re.compile(r"]*?\bsrc=[\"']([^\"']+)[\"']", re.IGNORECASE) + +# A bounded, sane extension parse (see importer._safe_ext for the FC gotcha: +# URL-encoded basenames make Path.suffix return base64-ish junk). We only ever +# use this for a fallback filename, never a network call. +_MAX_EXT_LEN = 16 + + +class PatreonAPIError(Exception): + """Base for native Patreon client failures.""" + + +class PatreonDriftError(PatreonAPIError): + """A response did not match the JSON:API shape we depend on. + + Raised for: a non-JSON / HTML-login response where JSON was expected, a + missing top-level `data` list, or a media resource lacking the + `file_name`/`url` fields we resolve against. Fail loud so the later import + step flags API drift instead of silently importing nothing. + """ + + +@dataclass +class MediaItem: + """One resolved downloadable item belonging to a post. + + Fields: + url — the CDN/download URL to fetch. + filename — media `file_name` when present; otherwise the URL basename + (NEVER a network call). Bounded/sane extension. + kind — one of: "images", "image_large", "attachments", "postfile", + "content". Mirrors gallery-dl's `files` content-type names so + the later step can honor the same per-source content_types. + filehash — the 32-char hex (MD5) segment from the CDN URL, or None if + the URL carries no such segment. Used for in-post dedup and + the cross-run seen-ledger. + post_id — the owning post's id (so a flattened media list stays + traceable to its post). + """ + + url: str + filename: str + kind: str + filehash: str | None + post_id: str + + +def _load_session(cookies_path: str | Path | None) -> requests.Session: + session = requests.Session() + session.headers.update( + { + "User-Agent": _USER_AGENT, + "Accept": "application/vnd.api+json", + } + ) + if cookies_path and os.path.isfile(str(cookies_path)): + try: + jar = http.cookiejar.MozillaCookieJar(str(cookies_path)) + jar.load(ignore_discard=True, ignore_expires=True) + session.cookies = jar # type: ignore[assignment] + except (OSError, http.cookiejar.LoadError) as exc: + log.warning("Could not load Patreon cookies from %s: %s", cookies_path, exc) + return session + + +def _filehash(url: str) -> str | None: + if not url: + return None + match = _FILEHASH_RE.search(url) + return match.group(1).lower() if match else None + + +def _safe_ext(name: str) -> str: + suffix = Path(name).suffix.lower() + if not suffix or len(suffix) > _MAX_EXT_LEN: + return "" + if not all(c.isalnum() for c in suffix[1:]): + return "" + return suffix + + +def _basename_from_url(url: str) -> str: + """Derive a sane filename from a URL when the media has no file_name. + + Strips query/fragment, takes the path basename, and drops a junk + extension (the importer._safe_ext gotcha) so we never write base64 noise + as a name. Falls back to the filehash, then to "file". + """ + path = urlsplit(url).path + base = os.path.basename(path) + if base: + ext = _safe_ext(base) + stem = base[: -len(Path(base).suffix)] if Path(base).suffix else base + # Keep the stem bounded; URL-encoded stems can be enormous. + stem = stem[:120] or "file" + return f"{stem}{ext}" + fh = _filehash(url) + return fh or "file" + + +def parse_cursor_from_url(url: str | None) -> str | None: + """Extract the `page[cursor]` query param from a links.next URL.""" + if not url: + return None + query = urlsplit(url).query + values = parse_qs(query).get("page[cursor]") + if values and values[0]: + return values[0] + return None + + +class PatreonClient: + """Synchronous Patreon JSON:API read client. + + Construct with a path to a Netscape cookies.txt (the same file + CredentialService.get_cookies_path materializes). Cookies are loaded into a + requests.Session; no secure-context APIs are used. + """ + + def __init__(self, cookies_path: str | Path | None): + self.cookies_path = str(cookies_path) if cookies_path else None + self._session = _load_session(cookies_path) + + # -- request ----------------------------------------------------------- + + def _params(self, campaign_id: str, cursor: str | None) -> dict[str, str]: + params = { + "include": _INCLUDE, + "fields[post]": _FIELDS_POST, + "fields[media]": _FIELDS_MEDIA, + "fields[campaign]": _FIELDS_CAMPAIGN, + "filter[campaign_id]": campaign_id, + "filter[contains_exclusive_posts]": "true", + "filter[is_draft]": "false", + "sort": "-published_at", + "json-api-version": "1.0", + } + if cursor: + params["page[cursor]"] = cursor + return params + + def _fetch(self, campaign_id: str, cursor: str | None) -> dict: + try: + resp = self._session.get( + _POSTS_URL, + params=self._params(campaign_id, cursor), + timeout=_TIMEOUT_SECONDS, + ) + except requests.RequestException as exc: + raise PatreonAPIError( + f"Patreon posts request failed (campaign_id={campaign_id}): {exc}" + ) from exc + + if resp.status_code != 200: + raise PatreonAPIError( + f"Patreon posts API returned HTTP {resp.status_code} " + f"(campaign_id={campaign_id})" + ) + try: + payload = resp.json() + except ValueError as exc: + # A non-JSON body here is almost always the HTML login/challenge + # page served when cookies are missing/expired — that is drift + # from the JSON:API contract, not a transient network error. + raise PatreonDriftError( + "Patreon posts API returned a non-JSON response (likely an " + f"HTML login/challenge page; campaign_id={campaign_id}): {exc}" + ) from exc + return payload + + # -- parsing ----------------------------------------------------------- + + @staticmethod + def _transform(response: dict) -> dict: + """Flatten the JSON:API `included` array for relationship resolution. + + Returns a dict keyed by `(type, id)` → that resource's `attributes` + (so a post's relationships can be resolved in O(1)). Missing/oddly + shaped `included` entries are skipped rather than fatal — drift + detection for the top-level shape lives in _validate_response. + """ + index: dict[tuple[str, str], dict] = {} + for inc in response.get("included") or []: + if not isinstance(inc, dict): + continue + rtype = inc.get("type") + rid = inc.get("id") + if rtype is None or rid is None: + continue + index[(str(rtype), str(rid))] = inc.get("attributes") or {} + return index + + @staticmethod + def _validate_response(response: dict) -> None: + if not isinstance(response, dict): + raise PatreonDriftError("Patreon response was not a JSON object") + if "data" not in response: + raise PatreonDriftError("Patreon response missing top-level 'data' key") + data = response.get("data") + if not isinstance(data, list): + raise PatreonDriftError("Patreon response 'data' was not a list") + + def _related_ids(self, post: dict, rel_name: str) -> list[str]: + rels = post.get("relationships") or {} + rel = rels.get(rel_name) or {} + data = rel.get("data") + if isinstance(data, dict): # to-one relationship + data = [data] + if not isinstance(data, list): + return [] + ids: list[str] = [] + for ref in data: + if isinstance(ref, dict) and ref.get("id") is not None: + ids.append(str(ref["id"])) + return ids + + @staticmethod + def _media_url(attrs: dict) -> str | None: + """Pick the best fetchable URL for a media resource. + + Prefer the full-size `download_url`; fall back to the largest + `image_urls` size. gallery-dl prefers download_url too, only dipping + into image_urls when a smaller configured size is requested — FC + always wants the original, so download_url first. + """ + download_url = attrs.get("download_url") + if isinstance(download_url, str) and download_url: + return download_url + image_urls = attrs.get("image_urls") + if isinstance(image_urls, dict): + for key in ("original", "full", "large", "default"): + candidate = image_urls.get(key) + if isinstance(candidate, str) and candidate: + return candidate + # Otherwise take any non-empty string value. + for candidate in image_urls.values(): + if isinstance(candidate, str) and candidate: + return candidate + return None + + def _media_item( + self, attrs: dict, kind: str, post_id: str, *, require_file_name: bool + ) -> MediaItem: + url = self._media_url(attrs) + if not url: + raise PatreonDriftError( + f"Patreon media (post {post_id}, kind={kind}) had no resolvable URL " + f"(no download_url / image_urls)" + ) + file_name = attrs.get("file_name") + if require_file_name and not (isinstance(file_name, str) and file_name): + raise PatreonDriftError( + f"Patreon media resource (post {post_id}, kind={kind}) missing " + f"'file_name'" + ) + filename = file_name if isinstance(file_name, str) and file_name else _basename_from_url(url) + return MediaItem( + url=url, + filename=filename, + kind=kind, + filehash=_filehash(url), + post_id=post_id, + ) + + def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]: + """Resolve all downloadable media for one post. + + Walks the kinds in the same order gallery-dl does — images, + image_large (post cover), attachments, postfile, content (inline + ) — and dedups within the post by filehash (first wins). The + image_large cover commonly duplicates a gallery image; deduping by + filehash collapses them to the gallery item (encountered first). + """ + post_id = str(post.get("id") or "") + attrs = post.get("attributes") or {} + items: list[MediaItem] = [] + + def _resolve_rel(rel_name: str, kind: str) -> None: + for mid in self._related_ids(post, rel_name): + media_attrs = included_index.get(("media", mid)) + if media_attrs is None: + # Referenced but not in `included`: a media id with no + # resource is drift (we asked for include=media). + raise PatreonDriftError( + f"Patreon post {post_id} references media {mid} " + f"({rel_name}) not present in 'included'" + ) + items.append( + self._media_item( + media_attrs, kind, post_id, require_file_name=True + ) + ) + + # 1. gallery images + _resolve_rel("images", "images") + + # 2. image_large — the post-level cover (`image.large_url`). Not a + # media relationship; it lives on the post attributes. + image = attrs.get("image") + if isinstance(image, dict): + large_url = image.get("large_url") or image.get("url") + if isinstance(large_url, str) and large_url: + items.append( + MediaItem( + url=large_url, + filename=_basename_from_url(large_url), + kind="image_large", + filehash=_filehash(large_url), + post_id=post_id, + ) + ) + + # 3. attachments + _resolve_rel("attachments_media", "attachments") + + # 4. postfile — the post's primary attached file (`post_file`). + post_file = attrs.get("post_file") + if isinstance(post_file, dict): + pf_url = post_file.get("url") or post_file.get("download_url") + if isinstance(pf_url, str) and pf_url: + pf_name = post_file.get("name") + filename = ( + pf_name + if isinstance(pf_name, str) and pf_name + else _basename_from_url(pf_url) + ) + items.append( + MediaItem( + url=pf_url, + filename=filename, + kind="postfile", + filehash=_filehash(pf_url), + post_id=post_id, + ) + ) + + # 5. content — inline in the post HTML body. + content = attrs.get("content") + if isinstance(content, str) and content: + for raw_src in _CONTENT_IMG_RE.findall(content): + src = unescape(raw_src) + if not src: + continue + items.append( + MediaItem( + url=src, + filename=_basename_from_url(src), + kind="content", + filehash=_filehash(src), + post_id=post_id, + ) + ) + + return _dedup_by_filehash(items) + + # -- iteration --------------------------------------------------------- + + def iter_posts( + self, campaign_id: str, cursor: str | None = None + ) -> Iterator[tuple[dict, dict, str | None]]: + """Yield (post, included_index, page_cursor) for every post in the feed. + + Pages newest→oldest via `links.next`, validating each response for + drift before yielding. The triple gives a caller everything it needs to + resolve and checkpoint without re-fetching: + - post — the raw post resource. + - included_index — the page's flattened `included` (the same object + for every post on a page), to pass straight to + extract_media(post, included_index). + - page_cursor — the cursor that FETCHED this post's page (None for + the first page). The caller checkpoints THIS value, + matching the existing backfill cursor logic where + the saved cursor re-fetches the page being + processed (so a chunk cut mid-page resumes the page, + not the one after it). + """ + current_cursor = cursor + while True: + response = self._fetch(campaign_id, current_cursor) + self._validate_response(response) + page_cursor = current_cursor + included_index = self._transform(response) + for post in response.get("data") or []: + if isinstance(post, dict): + yield post, included_index, page_cursor + next_url = (response.get("links") or {}).get("next") + next_cursor = parse_cursor_from_url(next_url) + if not next_cursor: + return + current_cursor = next_cursor + + +def _dedup_by_filehash(items: list[MediaItem]) -> list[MediaItem]: + """Drop later items sharing a filehash with an earlier one (first wins). + + Items with no filehash (None) are never deduped against each other — we + can't prove they're the same file, so keep them all. + """ + seen: set[str] = set() + out: list[MediaItem] = [] + for item in items: + if item.filehash is not None: + if item.filehash in seen: + continue + seen.add(item.filehash) + out.append(item) + return out diff --git a/tests/fixtures/patreon_posts_page1.json b/tests/fixtures/patreon_posts_page1.json new file mode 100644 index 0000000..6515c34 --- /dev/null +++ b/tests/fixtures/patreon_posts_page1.json @@ -0,0 +1,121 @@ +{ + "data": [ + { + "id": "1001", + "type": "post", + "attributes": { + "title": "Image gallery post", + "published_at": "2026-05-01T12:00:00.000+00:00", + "post_type": "image_file", + "content": "

just a gallery

", + "url": "https://www.patreon.com/posts/1001", + "image": { + "large_url": "https://c10.patreonusercontent.com/4/large/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg", + "url": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg" + } + }, + "relationships": { + "images": { + "data": [ + {"id": "9001", "type": "media"}, + {"id": "9002", "type": "media"} + ] + } + } + }, + { + "id": "1002", + "type": "post", + "attributes": { + "title": "Attachment post", + "published_at": "2026-04-15T09:30:00.000+00:00", + "post_type": "text_only", + "content": "

here is a zip

", + "url": "https://www.patreon.com/posts/1002", + "post_file": { + "name": "bonus-pack.zip", + "url": "https://www.patreon.com/file?h=cccccccccccccccccccccccccccccccc&i=1002" + } + }, + "relationships": { + "attachments_media": { + "data": [ + {"id": "9003", "type": "media"} + ] + } + } + }, + { + "id": "1003", + "type": "post", + "attributes": { + "title": "Inline content post", + "published_at": "2026-04-01T08:00:00.000+00:00", + "post_type": "text_only", + "content": "

look

\"inline\"
", + "url": "https://www.patreon.com/posts/1003" + }, + "relationships": {} + }, + { + "id": "1004", + "type": "post", + "attributes": { + "title": "Video post", + "published_at": "2026-03-20T18:45:00.000+00:00", + "post_type": "video_external_file", + "content": "

watch

", + "url": "https://www.patreon.com/posts/1004", + "post_file": { + "name": "clip.m3u8", + "url": "https://stream.mux.com/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee.m3u8?token=jwt" + } + }, + "relationships": {} + } + ], + "included": [ + { + "id": "9001", + "type": "media", + "attributes": { + "file_name": "gallery-one.jpg", + "download_url": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg", + "image_urls": { + "original": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg", + "default": "https://c10.patreonusercontent.com/4/default/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg" + } + } + }, + { + "id": "9002", + "type": "media", + "attributes": { + "file_name": "gallery-two.jpg", + "download_url": "https://c10.patreonusercontent.com/4/orig/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/gallery-two.jpg", + "image_urls": { + "original": "https://c10.patreonusercontent.com/4/orig/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/gallery-two.jpg" + } + } + }, + { + "id": "9003", + "type": "media", + "attributes": { + "file_name": "bonus-pack.zip", + "download_url": "https://www.patreon.com/file/download?h=cccccccccccccccccccccccccccccccc" + } + }, + { + "id": "5555", + "type": "campaign", + "attributes": { + "name": "Example Creator", + "url": "https://www.patreon.com/example" + } + } + ], + "links": { + "next": "https://www.patreon.com/api/posts?filter%5Bcampaign_id%5D=5555&page%5Bcursor%5D=NEXTCURSOR123&sort=-published_at" + } +} diff --git a/tests/test_patreon_client.py b/tests/test_patreon_client.py new file mode 100644 index 0000000..3ef504a --- /dev/null +++ b/tests/test_patreon_client.py @@ -0,0 +1,190 @@ +"""PatreonClient tests — parsing only, no network. + +The fixture is loaded from disk and fed directly to the pure parsing methods +(_transform / extract_media / _validate_response / parse_cursor_from_url); no +live requests call is exercised here. +""" + +import json +from pathlib import Path + +import pytest + +from backend.app.services.patreon_client import ( + MediaItem, + PatreonClient, + PatreonDriftError, + parse_cursor_from_url, +) + +_FIXTURE = Path(__file__).parent / "fixtures" / "patreon_posts_page1.json" + + +@pytest.fixture +def response(): + return json.loads(_FIXTURE.read_text()) + + +@pytest.fixture +def client(): + # No cookies file → empty session, but we never make a request in tests. + return PatreonClient(cookies_path=None) + + +def _post_by_id(response, post_id): + for post in response["data"]: + if post["id"] == post_id: + return post + raise AssertionError(f"no post {post_id} in fixture") + + +def test_transform_indexes_included(client, response): + index = client._transform(response) + assert index[("media", "9001")]["file_name"] == "gallery-one.jpg" + assert index[("media", "9003")]["file_name"] == "bonus-pack.zip" + assert index[("campaign", "5555")]["name"] == "Example Creator" + # Unknown keys are absent, not errors. + assert ("media", "0000") not in index + + +def test_extract_gallery_dedups_image_large(client, response): + index = client._transform(response) + post = _post_by_id(response, "1001") + items = client.extract_media(post, index) + + # Two gallery images; the image_large cover shares filehash aaaa... with + # gallery image 9001 → collapses to one, leaving exactly 2 items. + assert len(items) == 2 + assert all(isinstance(m, MediaItem) for m in items) + + kinds = [m.kind for m in items] + assert kinds == ["images", "images"] # image_large deduped away + + filenames = {m.filename for m in items} + assert filenames == {"gallery-one.jpg", "gallery-two.jpg"} + + hashes = {m.filehash for m in items} + assert hashes == {"a" * 32, "b" * 32} + assert all(m.post_id == "1001" for m in items) + + +def test_extract_attachment_and_postfile(client, response): + index = client._transform(response) + post = _post_by_id(response, "1002") + items = client.extract_media(post, index) + + by_kind = {m.kind: m for m in items} + # attachments_media → "attachments", plus the post_file → "postfile". + assert set(by_kind) == {"attachments", "postfile"} + assert by_kind["attachments"].filename == "bonus-pack.zip" + assert by_kind["attachments"].filehash == "c" * 32 + assert by_kind["postfile"].filename == "bonus-pack.zip" + # postfile URL also carries the same hash, but they're different kinds in + # different relationships; both are kept because dedup is by filehash and + # the attachment is encountered first... assert the post_file collapsed. + # (attachments resolved before postfile, same hash → postfile dropped.) + # So only ONE survives with that hash: + surviving_c = [m for m in items if m.filehash == "c" * 32] + assert len(surviving_c) == 1 + + +def test_extract_inline_content_img(client, response): + index = client._transform(response) + post = _post_by_id(response, "1003") + items = client.extract_media(post, index) + + assert len(items) == 1 + item = items[0] + assert item.kind == "content" + assert item.filehash == "d" * 32 + # & in the src must be unescaped to & . + assert "&" not in item.url + assert "token=x&v=2" in item.url + + +def test_extract_video_postfile(client, response): + index = client._transform(response) + post = _post_by_id(response, "1004") + items = client.extract_media(post, index) + + assert len(items) == 1 + item = items[0] + assert item.kind == "postfile" + assert item.url.startswith("https://stream.mux.com/") + assert item.filehash == "e" * 32 + + +def test_parse_cursor_from_url_extracts_cursor(response): + next_url = response["links"]["next"] + assert parse_cursor_from_url(next_url) == "NEXTCURSOR123" + + +def test_parse_cursor_from_url_none_when_absent(): + assert parse_cursor_from_url("https://www.patreon.com/api/posts?sort=-x") is None + assert parse_cursor_from_url(None) is None + + +def test_iter_posts_yields_triples_and_paginates(client, response, monkeypatch): + # Page 1 = the fixture (4 posts, links.next → NEXTCURSOR123); page 2 = one + # post, no links.next → stop. Monkeypatch _fetch so no network is touched. + page2 = { + "data": [{"id": "1005", "type": "post", "attributes": {"title": "older"}}], + "included": [], + "links": {}, + } + + def fake_fetch(campaign_id, cursor): + return page2 if cursor == "NEXTCURSOR123" else response + + monkeypatch.setattr(client, "_fetch", fake_fetch) + + seen = list(client.iter_posts("5555")) + # 4 posts on page 1 + 1 on page 2. + assert [p["id"] for p, _idx, _cur in seen] == ["1001", "1002", "1003", "1004", "1005"] + # page_cursor is the cursor that FETCHED the post's page. + cursors = {p["id"]: cur for p, _idx, cur in seen} + assert cursors["1001"] is None and cursors["1005"] == "NEXTCURSOR123" + # included_index is the page's flattened resources, ready for extract_media. + page1_index = next(idx for p, idx, _cur in seen if p["id"] == "1001") + assert page1_index[("media", "9001")]["file_name"] == "gallery-one.jpg" + + +def test_missing_data_raises_drift(client): + with pytest.raises(PatreonDriftError): + client._validate_response({"included": []}) + + +def test_non_list_data_raises_drift(client): + with pytest.raises(PatreonDriftError): + client._validate_response({"data": {"id": "1"}}) + + +def test_media_missing_file_name_raises_drift(client): + # A media resource referenced by a gallery relationship but lacking + # file_name is drift from the contract. + post = { + "id": "2000", + "type": "post", + "attributes": {"title": "broken"}, + "relationships": {"images": {"data": [{"id": "7777", "type": "media"}]}}, + } + index = { + ("media", "7777"): { + "download_url": "https://c10.patreonusercontent.com/4/orig/" + + "f" * 32 + + "/x.jpg" + } + } + with pytest.raises(PatreonDriftError): + client.extract_media(post, index) + + +def test_media_referenced_but_absent_raises_drift(client): + post = { + "id": "2001", + "type": "post", + "attributes": {"title": "dangling"}, + "relationships": {"images": {"data": [{"id": "8888", "type": "media"}]}}, + } + with pytest.raises(PatreonDriftError): + client.extract_media(post, {}) -- 2.52.0 From 1bdaa04aa2eb970653f594898c03aefc30c60cbc Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 19:17:56 -0400 Subject: [PATCH 02/26] test(patreon): fix self-contradictory attachment/postfile dedup assertion The fixture gives the attachment and post_file the same filehash, so they correctly collapse to one item; the test asserted both survival and collapse. Rewrite to verify the cross-kind dedup (postfile kind covered by video test). Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_patreon_client.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/tests/test_patreon_client.py b/tests/test_patreon_client.py index 3ef504a..02bd328 100644 --- a/tests/test_patreon_client.py +++ b/tests/test_patreon_client.py @@ -68,24 +68,20 @@ def test_extract_gallery_dedups_image_large(client, response): assert all(m.post_id == "1001" for m in items) -def test_extract_attachment_and_postfile(client, response): +def test_extract_attachment_postfile_same_file_collapses(client, response): index = client._transform(response) post = _post_by_id(response, "1002") items = client.extract_media(post, index) - by_kind = {m.kind: m for m in items} - # attachments_media → "attachments", plus the post_file → "postfile". - assert set(by_kind) == {"attachments", "postfile"} - assert by_kind["attachments"].filename == "bonus-pack.zip" - assert by_kind["attachments"].filehash == "c" * 32 - assert by_kind["postfile"].filename == "bonus-pack.zip" - # postfile URL also carries the same hash, but they're different kinds in - # different relationships; both are kept because dedup is by filehash and - # the attachment is encountered first... assert the post_file collapsed. - # (attachments resolved before postfile, same hash → postfile dropped.) - # So only ONE survives with that hash: - surviving_c = [m for m in items if m.filehash == "c" * 32] - assert len(surviving_c) == 1 + # The attachment (attachments_media 9003) and the post_file are the SAME + # zip — both carry filehash c…. attachments resolve before postfile, so the + # postfile collapses into the attachment by filehash → exactly one survives. + # (postfile-as-a-kind is exercised separately by the video post test.) + assert len(items) == 1 + item = items[0] + assert item.kind == "attachments" + assert item.filename == "bonus-pack.zip" + assert item.filehash == "c" * 32 def test_extract_inline_content_img(client, response): -- 2.52.0 From 6222928746bd7eb40af430cdb35039cc6fdfea1e Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 19:26:57 -0400 Subject: [PATCH 03/26] =?UTF-8?q?feat(patreon):=20seen-ledger=20table=20+?= =?UTF-8?q?=20model=20=E2=80=94=20ingester=20build=20step=202a=20(plan=20#?= =?UTF-8?q?697)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit patreon_seen_media(source_id, filehash, post_id, seen_at), UNIQUE(source_id, filehash) — our own queryable replacement for gallery-dl's archive.sqlite3. Routine walks skip seen media; recovery mode bypasses the ledger. filehash is a 32-hex CDN MD5 or a video:: sentinel (String(128)). alembic 0037 (← 0036). Integration test covers dedup + savepoint recovery. Co-Authored-By: Claude Opus 4.8 (1M context) --- alembic/versions/0037_patreon_seen_media.py | 53 +++++++++++++++ backend/app/models/__init__.py | 2 + backend/app/models/patreon_seen_media.py | 38 +++++++++++ tests/test_patreon_seen_media.py | 73 +++++++++++++++++++++ 4 files changed, 166 insertions(+) create mode 100644 alembic/versions/0037_patreon_seen_media.py create mode 100644 backend/app/models/patreon_seen_media.py create mode 100644 tests/test_patreon_seen_media.py diff --git a/alembic/versions/0037_patreon_seen_media.py b/alembic/versions/0037_patreon_seen_media.py new file mode 100644 index 0000000..255484e --- /dev/null +++ b/alembic/versions/0037_patreon_seen_media.py @@ -0,0 +1,53 @@ +"""patreon_seen_media: per-source ledger of already-ingested Patreon media + +Revision ID: 0037 +Revises: 0036 +Create Date: 2026-06-05 + +Native Patreon ingester (build step 2a). Replaces gallery-dl's +archive.sqlite3 with our own queryable table. The downloader upserts one +row per (source, media) so routine walks skip media we've already +processed; a future "recovery" mode bypasses the ledger to re-walk. + +`filehash` is a 32-hex Patreon CDN MD5, OR a video sentinel of the form +``video::`` — hence String(128). The unique +constraint on (source_id, filehash) is the dedup upsert key. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0037" +down_revision: Union[str, None] = "0036" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "patreon_seen_media", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "source_id", + sa.Integer, + sa.ForeignKey("source.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("filehash", sa.String(128), nullable=False), + sa.Column("post_id", sa.String(64), nullable=True), + sa.Column( + "seen_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.UniqueConstraint( + "source_id", "filehash", name="uq_patreon_seen_media_source_id" + ), + ) + + +def downgrade() -> None: + op.drop_table("patreon_seen_media") diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 137311c..1445788 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -14,6 +14,7 @@ from .import_settings import ImportSettings from .import_task import ImportTask from .library_audit_run import LibraryAuditRun from .ml_settings import MLSettings +from .patreon_seen_media import PatreonSeenMedia from .post import Post from .post_attachment import PostAttachment from .series_page import SeriesPage @@ -33,6 +34,7 @@ __all__ = [ "BackupRun", "Source", "Credential", + "PatreonSeenMedia", "Post", "PostAttachment", "SeriesPage", diff --git a/backend/app/models/patreon_seen_media.py b/backend/app/models/patreon_seen_media.py new file mode 100644 index 0000000..9ba92f2 --- /dev/null +++ b/backend/app/models/patreon_seen_media.py @@ -0,0 +1,38 @@ +"""PatreonSeenMedia — per-source ledger of Patreon media already +downloaded+processed. + +Replaces gallery-dl's archive.sqlite3 with our own queryable table so +routine walks can skip media we've already ingested (and a future +"recovery" mode can deliberately bypass the ledger to re-walk). + +`filehash` is normally a Patreon CDN MD5 (32 hex chars), but videos — +which have no stable content hash at discovery time — use a sentinel of +the form ``video::``, hence String(128) rather than 32. +""" + +from datetime import datetime + +from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import DateTime + +from .base import Base + + +class PatreonSeenMedia(Base): + __tablename__ = "patreon_seen_media" + __table_args__ = ( + # Dedup key the downloader upserts against: one ledger row per + # (source, media). A second sighting of the same media is a no-op. + UniqueConstraint("source_id", "filehash", name="uq_patreon_seen_media_source_id"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source_id: Mapped[int] = mapped_column( + ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + ) + filehash: Mapped[str] = mapped_column(String(128), nullable=False) + post_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/tests/test_patreon_seen_media.py b/tests/test_patreon_seen_media.py new file mode 100644 index 0000000..e80e9bf --- /dev/null +++ b/tests/test_patreon_seen_media.py @@ -0,0 +1,73 @@ +import pytest +from sqlalchemy import func, select +from sqlalchemy.exc import IntegrityError + +from backend.app.models import Artist, PatreonSeenMedia, Source + +pytestmark = pytest.mark.integration + + +async def _source(db): + artist = Artist(name="Seen Ledger Artist", slug="seen-ledger-artist") + db.add(artist) + await db.flush() + source = Source( + artist_id=artist.id, + platform="patreon", + url="https://www.patreon.com/seenledger", + ) + db.add(source) + await db.flush() + return source.id + + +@pytest.mark.asyncio +async def test_patreon_seen_media_dedup(db): + source_id = await _source(db) + db.add( + PatreonSeenMedia( + source_id=source_id, + filehash="0123456789abcdef0123456789abcdef", + post_id="123456", + ) + ) + await db.flush() + + seen_at = await db.scalar( + select(PatreonSeenMedia.seen_at).where( + PatreonSeenMedia.source_id == source_id + ) + ) + assert seen_at is not None + + # A second sighting of the same (source_id, filehash) is rejected by the + # unique constraint. Wrap in a savepoint so the IntegrityError doesn't + # poison the outer transaction. + with pytest.raises(IntegrityError): + async with db.begin_nested(): + db.add( + PatreonSeenMedia( + source_id=source_id, + filehash="0123456789abcdef0123456789abcdef", + post_id="654321", + ) + ) + await db.flush() + + # Same filehash but a different post_id sentinel still inserts: the + # dedup axis is (source_id, filehash), not post_id. + db.add( + PatreonSeenMedia( + source_id=source_id, + filehash="video:123456:7890", + post_id="123456", + ) + ) + await db.flush() + + count = await db.scalar( + select(func.count(PatreonSeenMedia.id)).where( + PatreonSeenMedia.source_id == source_id + ) + ) + assert count == 2 -- 2.52.0 From 2ec7d86a3b44d204d7490476acba2dd8e7a8cea0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 19:39:12 -0400 Subject: [PATCH 04/26] =?UTF-8?q?feat(patreon):=20native=20media=20downloa?= =?UTF-8?q?der=20=E2=80=94=20ingester=20build=20step=202b=20(plan=20#697)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PatreonDownloader.download_post: writes resolved MediaItems to gallery-dl's exact on-disk layout (/patreon/__/_) + a sidecar the importer's find_sidecar/parse_sidecar consume unchanged. Two-tier skip (injected seen predicate, then disk). Streamed GET (.part→rename) + file_validator quarantine; Mux/m3u8 video shells out to yt-dlp with Patreon Referer/Origin. Pure (no DB) — ledger + orchestration land in step 3. Unit tests stub the session + yt-dlp seams (no network/subprocess). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/patreon_downloader.py | 384 +++++++++++++++++++++ tests/test_patreon_downloader.py | 252 ++++++++++++++ 2 files changed, 636 insertions(+) create mode 100644 backend/app/services/patreon_downloader.py create mode 100644 tests/test_patreon_downloader.py diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py new file mode 100644 index 0000000..bd920ce --- /dev/null +++ b/backend/app/services/patreon_downloader.py @@ -0,0 +1,384 @@ +"""Native Patreon media downloader (build step 2b of the native ingester). + +Given a Patreon post and its already-resolved `MediaItem`s (from +patreon_client.extract_media), download the media to the EXACT on-disk layout +gallery-dl produces, write a sidecar JSON the existing importer consumes, and +report per-media outcomes. + +This module is PURE: no DB. The cross-run seen-ledger and the iter_posts +orchestration are a LATER step. The tier-1 (seen) skip is an INJECTED predicate +(`is_seen`), so this module needs no DB and is unit-testable without network. + +On-disk layout (matches gallery-dl): + //patreon//_ + where = "__" (date prefix omitted when the + post's published_at is unparseable), is the 1-based index of the item in + the post zero-padded to 2, and is MediaItem.filename. The sidecar + is written next to the media as _.json (media_path.with_suffix). + +Video: a MediaItem whose URL is a Mux/HLS stream (host stream.mux.com or path +endswith .m3u8) is fetched with yt-dlp (subprocess), passing the same +Referer/Origin headers gallery-dl forwards for Mux playback (see gallery_dl.py +_get_default_config). yt-dlp may remux to a container of its own choosing, so we +accept the actual output extension and record the real path. + +FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. +""" + +from __future__ import annotations + +import contextlib +import json +import logging +import os +import shutil +import subprocess +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from urllib.parse import urlsplit + +import requests + +from .file_validator import is_validatable, validate_file +from .patreon_client import _load_session + +log = logging.getLogger(__name__) + +_TITLE_MAX = 40 +_TIMEOUT_SECONDS = 120.0 +_CHUNK = 1 << 16 + +# Same Referer/Origin gallery-dl forwards to yt-dlp for Mux-hosted Patreon +# video (gallery_dl.py _get_default_config -> downloader.ytdl.raw-options). +# Mux's JWT playback policy checks Referer/Origin on every request, so yt-dlp +# must send Patreon's, not its own default. +_VIDEO_HEADERS = { + "Referer": "https://www.patreon.com/", + "Origin": "https://www.patreon.com", +} + +# Characters Windows/gallery-dl path-restrict forbids, plus path separators. +_FORBIDDEN = set('<>:"/\\|?*') + + +def _sanitize(name: str) -> str: + """Make `name` safe for a single filesystem path segment. + + Replaces path separators, the Windows-forbidden set <>:"/\\|?* and control + characters with `_`, then strips trailing dots/spaces (gallery-dl + path-restrict behavior). Never returns empty (falls back to "_"). + """ + out = [] + for ch in name: + if ch in _FORBIDDEN or ord(ch) < 32: + out.append("_") + else: + out.append(ch) + cleaned = "".join(out).rstrip(". ") + return cleaned or "_" + + +def _is_video_url(url: str) -> bool: + parts = urlsplit(url) + if parts.hostname and parts.hostname.lower() == "stream.mux.com": + return True + return parts.path.lower().endswith(".m3u8") + + +def _post_dir_name(post: dict) -> str: + """Build the post directory name matching gallery-dl's layout.""" + post_id = str(post.get("id") or "") + attrs = post.get("attributes") or {} + title = attrs.get("title") + title = title if isinstance(title, str) else "" + title40 = title[:_TITLE_MAX] + + published = attrs.get("published_at") + date_prefix = None + if isinstance(published, str) and published: + s = published.strip() + if s.endswith("Z"): + s = s[:-1] + "+00:00" + try: + dt = datetime.fromisoformat(s) + except ValueError: + dt = None + if dt is not None: + date_prefix = f"{dt:%Y-%m-%d}" + + if date_prefix: + raw = f"{date_prefix}_{post_id}_{title40}" + else: + raw = f"{post_id}_{title40}" + return _sanitize(raw) + + +@dataclass +class MediaOutcome: + """Per-media result of a download_post pass. + + status is one of: "downloaded", "skipped_seen", "skipped_disk", "error". + `path` is the final on-disk path for "downloaded" (the actual yt-dlp output + for video), or the path that already existed for "skipped_disk"; None for + "skipped_seen" and (usually) "error". `error` carries the failure reason for + "error", else None. + """ + + media: object # MediaItem (avoid importing the name for a bare annotation) + status: str + path: Path | None + error: str | None + + +class PatreonDownloader: + """Download resolved Patreon media to gallery-dl's on-disk layout. + + PURE: no DB. The HTTP session and the yt-dlp invocation are injectable seams + so tests run without network or a real subprocess: + - pass `session=` to stub `session.get`, or monkeypatch `_fetch_to_file`. + - monkeypatch `_run_ytdlp` to avoid spawning yt-dlp. + """ + + def __init__( + self, + images_root: Path, + cookies_path: str | None = None, + *, + validate: bool = True, + session: requests.Session | None = None, + ): + self.images_root = Path(images_root) + self.cookies_path = str(cookies_path) if cookies_path else None + self._validate = validate + # Build a cookie-loaded session the same way patreon_client does, so the + # CDN GETs carry the creator's auth. + self.session = session if session is not None else _load_session(cookies_path) + + # -- public ------------------------------------------------------------ + + def download_post( + self, + post: dict, + media_items: list, + artist_slug: str, + *, + is_seen: Callable[[object], bool] = lambda m: False, + ) -> list[MediaOutcome]: + """Download every media item of one post; return per-item outcomes. + + Builds the post directory, iterates media (1-based NN), applies the + two-tier skip (injected is_seen, then disk), downloads (plain GET or + yt-dlp for video), writes the sidecar for each freshly-downloaded item, + validates, and returns outcomes. Resilient: one media's failure yields + an "error" outcome for that item; the rest proceed. + """ + post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post) + outcomes: list[MediaOutcome] = [] + + for i, media in enumerate(media_items, start=1): + try: + outcomes.append( + self._download_one(post, media, post_dir, artist_slug, i, is_seen) + ) + except Exception as exc: # resilient: isolate one item's failure + log.warning( + "Patreon media failed (post %s, item %d): %s", + post.get("id"), i, exc, + ) + outcomes.append( + MediaOutcome(media=media, status="error", path=None, error=str(exc)) + ) + return outcomes + + # -- per-item ---------------------------------------------------------- + + def _download_one( + self, + post: dict, + media, + post_dir: Path, + artist_slug: str, + index: int, + is_seen: Callable[[object], bool], + ) -> MediaOutcome: + # tier-1: seen ledger (injected; no DB here). + if is_seen(media): + return MediaOutcome(media=media, status="skipped_seen", path=None, error=None) + + nn = f"{index:02d}" + final_name = _sanitize(f"{nn}_{media.filename}") + media_path = post_dir / final_name + + # tier-2: already on disk. + if media_path.exists(): + return MediaOutcome( + media=media, status="skipped_disk", path=media_path, error=None + ) + # Video may land at a different extension; honor a pre-existing remux. + if _is_video_url(media.url): + existing = self._existing_video_output(media_path) + if existing is not None: + return MediaOutcome( + media=media, status="skipped_disk", path=existing, error=None + ) + + post_dir.mkdir(parents=True, exist_ok=True) + + if _is_video_url(media.url): + out_path = self._run_ytdlp(media.url, media_path, _VIDEO_HEADERS) + if out_path is None or not Path(out_path).exists(): + return MediaOutcome( + media=media, + status="error", + path=None, + error="yt-dlp produced no output", + ) + out_path = Path(out_path) + else: + out_path = self._fetch_get(media.url, media_path) + invalid = self._validate_path(out_path, artist_slug) + if invalid is not None: + return MediaOutcome( + media=media, status="error", path=None, error=invalid + ) + + self._write_sidecar(post, out_path) + return MediaOutcome(media=media, status="downloaded", path=out_path, error=None) + + # -- download seams ---------------------------------------------------- + + def _fetch_get(self, url: str, dest: Path) -> Path: + """Stream `url` to a .part file then atomic-rename to `dest`. + + Thin wrapper over `_fetch_to_file` so tests can stub either the whole + GET path (`_fetch_to_file`) or just `session.get`. + """ + part = dest.with_name(dest.name + ".part") + try: + self._fetch_to_file(url, part) + except Exception: + with contextlib.suppress(OSError): + part.unlink() + raise + os.replace(part, dest) + return dest + + def _fetch_to_file(self, url: str, dest: Path) -> None: + """Stream a non-video URL to `dest` via the (stubbable) session.""" + resp = self.session.get(url, stream=True, timeout=_TIMEOUT_SECONDS) + resp.raise_for_status() + with open(dest, "wb") as fh: + for chunk in resp.iter_content(chunk_size=_CHUNK): + if chunk: + fh.write(chunk) + + def _run_ytdlp(self, url: str, dest: Path, headers: dict) -> Path | None: + """Invoke yt-dlp to fetch a Mux/HLS stream to (around) `dest`. + + Returns the actual output path (yt-dlp may remux to a different + container, so we resolve the real file afterward). Overridden/ + monkeypatched in tests to avoid spawning a real subprocess. + + The output template uses `dest` without its extension; yt-dlp appends + the chosen container extension. We pass Referer/Origin (Mux JWT policy) + and the cookies file. + """ + dest = Path(dest) + out_template = str(dest.with_suffix("")) + ".%(ext)s" + cmd = ["yt-dlp", "--no-progress", "-o", out_template] + for key, value in headers.items(): + cmd += ["--add-header", f"{key}:{value}"] + if self.cookies_path and os.path.isfile(self.cookies_path): + cmd += ["--cookies", self.cookies_path] + cmd.append(url) + try: + subprocess.run( + cmd, + check=True, + capture_output=True, + text=True, + timeout=_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.SubprocessError) as exc: + log.warning("yt-dlp failed for %s: %s", url, exc) + return None + return self._existing_video_output(dest) + + def _existing_video_output(self, dest: Path) -> Path | None: + """Find a yt-dlp output for `dest` regardless of chosen extension. + + Returns `dest` itself if present, else any sibling sharing the same + stem (the remuxed container). None if nothing matched. + """ + dest = Path(dest) + if dest.exists(): + return dest + stem = dest.with_suffix("").name + parent = dest.parent + if not parent.is_dir(): + return None + for cand in sorted(parent.iterdir()): + if cand.is_file() and cand.stem == stem and cand.name != dest.name: + return cand + return None + + # -- validation -------------------------------------------------------- + + def _validate_path(self, path: Path, artist_slug: str) -> str | None: + """Validate a freshly-written file; quarantine + return reason if bad. + + Mirrors gallery_dl._validate_and_quarantine: fail-open for unknown + formats, move the corrupt file to _quarantine//patreon, and return + the failure reason string (None when ok / not validatable / disabled). + """ + if not self._validate or not is_validatable(path): + return None + try: + result = validate_file(path) + except Exception as exc: + log.warning("Validator raised on %s: %s", path, exc) + return None + if result.ok: + return None + quarantine_root = self.images_root / "_quarantine" / artist_slug / "patreon" + try: + quarantine_root.mkdir(parents=True, exist_ok=True) + dest = quarantine_root / path.name + counter = 1 + while dest.exists(): + dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}" + counter += 1 + shutil.move(str(path), str(dest)) + except OSError as exc: + log.error("Failed to quarantine %s: %s. File left in place.", path, exc) + return result.reason or "validation failed" + + # -- sidecar ----------------------------------------------------------- + + def _write_sidecar(self, post: dict, media_path: Path) -> Path: + """Write the importer-consumed sidecar next to `media_path`. + + Patreon uses base-default sidecar keys (parse_sidecar maps + category->platform, id->external_post_id, title->post_title, + content->description, published_at->post_date, url->post_url). Patreon + registers no derive_post_url, so `url` is trusted as the permalink — we + pass the post's attributes.url. + """ + attrs = post.get("attributes") or {} + title = attrs.get("title") + content = attrs.get("content") + published = attrs.get("published_at") + url = attrs.get("url") + data = { + "category": "patreon", + "id": str(post.get("id") or ""), + "title": title if isinstance(title, str) else "", + "content": content if isinstance(content, str) else "", + "published_at": published if isinstance(published, str) else None, + "url": url if isinstance(url, str) else None, + } + sidecar_path = media_path.with_suffix(".json") + sidecar_path.write_text(json.dumps(data, indent=2)) + return sidecar_path diff --git a/tests/test_patreon_downloader.py b/tests/test_patreon_downloader.py new file mode 100644 index 0000000..6b79975 --- /dev/null +++ b/tests/test_patreon_downloader.py @@ -0,0 +1,252 @@ +"""Unit tests for PatreonDownloader (build step 2b). + +No network, no real subprocess. The HTTP layer is stubbed via the `session` +seam (a fake session whose `.get` returns a fake streaming response) and yt-dlp +via monkeypatching `_run_ytdlp`. PURE module → no DB, so unmarked (fast lane). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from backend.app.services.patreon_client import MediaItem +from backend.app.services.patreon_downloader import ( + MediaOutcome, + PatreonDownloader, + _sanitize, +) +from backend.app.utils.sidecar import find_sidecar + +# Minimal valid PNG so the file_validator passes on the happy path. +_PNG_HEAD = b"\x89PNG\r\n\x1a\n" +_PNG_TAIL = b"\x00\x00\x00\x00IEND\xaeB`\x82" +_PNG_BYTES = _PNG_HEAD + (b"\x00" * 16) + _PNG_TAIL + + +class _FakeResponse: + def __init__(self, payload: bytes): + self._payload = payload + + def raise_for_status(self): + return None + + def iter_content(self, chunk_size=65536): + for i in range(0, len(self._payload), chunk_size): + yield self._payload[i : i + chunk_size] + + +class _FakeSession: + """Records GETs and returns canned bytes (per-URL, default _PNG_BYTES).""" + + def __init__(self, payloads: dict[str, bytes] | None = None): + self.payloads = payloads or {} + self.calls: list[str] = [] + + def get(self, url, stream=False, timeout=None): + self.calls.append(url) + return _FakeResponse(self.payloads.get(url, _PNG_BYTES)) + + +def _post(post_id="1001", title="My Post Title", published="2026-05-01T12:00:00Z"): + return { + "id": post_id, + "attributes": { + "title": title, + "content": "

body

", + "published_at": published, + "url": f"https://www.patreon.com/posts/{post_id}", + }, + } + + +def _img(name, post_id="1001", url=None): + return MediaItem( + url=url or f"https://cdn.patreon.com/{name}", + filename=name, + kind="images", + filehash=None, + post_id=post_id, + ) + + +def _downloader(tmp_path: Path, session=None, validate=True): + return PatreonDownloader( + images_root=tmp_path, + cookies_path=None, + validate=validate, + session=session or _FakeSession(), + ) + + +def test_path_layout_two_images(tmp_path): + dl = _downloader(tmp_path) + items = [_img("media1.png"), _img("media2.png")] + outcomes = dl.download_post(_post(), items, "artist-x") + + post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title" + p1 = post_dir / "01_media1.png" + p2 = post_dir / "02_media2.png" + assert p1.is_file() + assert p2.is_file() + assert [o.status for o in outcomes] == ["downloaded", "downloaded"] + assert outcomes[0].path == p1 + assert p1.read_bytes() == _PNG_BYTES + + +def test_dir_sanitized(tmp_path): + dl = _downloader(tmp_path) + post = _post(title='bad/name:with*chars?') + dl.download_post(post, [_img("a.png")], "artist-x") + patreon_dir = tmp_path / "artist-x" / "patreon" + children = list(patreon_dir.iterdir()) + assert len(children) == 1 + name = children[0].name + assert "/" not in name + assert not any(c in name for c in '<>:"\\|?*') + + +def test_no_date_prefix_when_unparseable(tmp_path): + dl = _downloader(tmp_path) + post = _post(published="not-a-date") + dl.download_post(post, [_img("a.png")], "artist-x") + patreon_dir = tmp_path / "artist-x" / "patreon" + children = [c.name for c in patreon_dir.iterdir()] + assert children == ["1001_My Post Title"] + + +def test_sidecar_written_and_findable(tmp_path): + dl = _downloader(tmp_path) + outcomes = dl.download_post(_post(), [_img("media1.png")], "artist-x") + media_path = outcomes[0].path + + sidecar = find_sidecar(media_path) + assert sidecar is not None + assert sidecar.name == "01_media1.json" + + import json + + data = json.loads(sidecar.read_text()) + assert data["category"] == "patreon" + assert data["id"] == "1001" + assert data["title"] == "My Post Title" + assert data["url"] == "https://www.patreon.com/posts/1001" + + +def test_skip_seen(tmp_path): + session = _FakeSession() + dl = _downloader(tmp_path, session=session) + outcomes = dl.download_post( + _post(), [_img("media1.png")], "artist-x", is_seen=lambda m: True + ) + assert outcomes[0].status == "skipped_seen" + assert outcomes[0].path is None + assert session.calls == [] + # No file written anywhere. + assert not (tmp_path / "artist-x").exists() + + +def test_skip_disk(tmp_path): + dl = _downloader(tmp_path) + post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title" + post_dir.mkdir(parents=True) + existing = post_dir / "01_media1.png" + existing.write_bytes(b"already here") + + outcomes = dl.download_post(_post(), [_img("media1.png")], "artist-x") + assert outcomes[0].status == "skipped_disk" + assert outcomes[0].path == existing + assert existing.read_bytes() == b"already here" # untouched + + +def test_video_routes_to_ytdlp(tmp_path, monkeypatch): + session = _FakeSession() + dl = _downloader(tmp_path, session=session) + captured = {} + + def fake_ytdlp(url, dest, headers): + captured["url"] = url + captured["headers"] = headers + out = Path(dest).with_suffix(".mp4") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_bytes(b"video-bytes") + return out + + monkeypatch.setattr(dl, "_run_ytdlp", fake_ytdlp) + + video = MediaItem( + url="https://stream.mux.com/abc123.m3u8", + filename="clip.mp4", + kind="postfile", + filehash=None, + post_id="1001", + ) + outcomes = dl.download_post(_post(), [video], "artist-x") + + assert captured["url"] == "https://stream.mux.com/abc123.m3u8" + assert captured["headers"]["Referer"] == "https://www.patreon.com/" + assert captured["headers"]["Origin"] == "https://www.patreon.com" + assert session.calls == [] # NOT the plain-GET path + assert outcomes[0].status == "downloaded" + assert outcomes[0].path.read_bytes() == b"video-bytes" + # Sidecar written next to the actual (remuxed) output. + assert find_sidecar(outcomes[0].path) is not None + + +def test_one_failure_isolated(tmp_path): + class _BoomSession(_FakeSession): + def get(self, url, stream=False, timeout=None): + self.calls.append(url) + if "bad" in url: + raise RuntimeError("network boom") + return _FakeResponse(_PNG_BYTES) + + dl = _downloader(tmp_path, session=_BoomSession()) + items = [ + _img("good.png", url="https://cdn.patreon.com/good.png"), + _img("bad.png", url="https://cdn.patreon.com/bad.png"), + ] + outcomes = dl.download_post(_post(), items, "artist-x") + assert outcomes[0].status == "downloaded" + assert outcomes[1].status == "error" + assert "boom" in outcomes[1].error + + +def test_validation_quarantines_corrupt(tmp_path): + # A .png whose bytes are not a valid PNG → validator fails → error + + # quarantine move; with validate=False it would pass. + bad = b"not a real png" + session = _FakeSession({"https://cdn.patreon.com/corrupt.png": bad}) + dl = _downloader(tmp_path, session=session, validate=True) + item = _img("corrupt.png", url="https://cdn.patreon.com/corrupt.png") + outcomes = dl.download_post(_post(), [item], "artist-x") + assert outcomes[0].status == "error" + post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title" + assert not (post_dir / "01_corrupt.png").exists() + quarantine = tmp_path / "_quarantine" / "artist-x" / "patreon" + assert quarantine.is_dir() + assert any(quarantine.iterdir()) + + +def test_validation_off_keeps_bytes(tmp_path): + bad = b"not a real png" + session = _FakeSession({"https://cdn.patreon.com/x.png": bad}) + dl = _downloader(tmp_path, session=session, validate=False) + item = _img("x.png", url="https://cdn.patreon.com/x.png") + outcomes = dl.download_post(_post(), [item], "artist-x") + assert outcomes[0].status == "downloaded" + assert outcomes[0].path.read_bytes() == bad + + +def test_sanitize_helper(): + assert _sanitize('a/b') == "a_b" + assert _sanitize('x<>:"|?*y') == "x_______y" + assert _sanitize("trail. ") == "trail" + assert _sanitize("") == "_" + + +def test_media_outcome_shape(): + o = MediaOutcome(media=None, status="downloaded", path=Path("/tmp/x"), error=None) + assert o.status == "downloaded" + assert o.path == Path("/tmp/x") -- 2.52.0 From 96c30eba1338d206b7cb74287bf2f68ce03506d2 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 21:38:42 -0400 Subject: [PATCH 05/26] =?UTF-8?q?feat(patreon):=20phase-2=20ingester=20int?= =?UTF-8?q?egration=20=E2=80=94=20build=20step=203=20(plan=20#697)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Branch download_service phase 2 by platform: Patreon now routes to the native PatreonIngester (zero per-file HEADs, native cursor/resume, loud drift detection) instead of gallery-dl; the other 5 platforms are unchanged. The ingester returns a DownloadResult-shaped object so phase 1 (DB setup) and phase 3 (import → pHash → thumbs → ML) are untouched. Three modes wired from config_overrides state: - tick: skip seen (tier-1 ledger + tier-2 disk), early-out after N contiguous already-have-it items. - backfill: full-history time-boxed chunk, cursor checkpoint via gallery-dl-style "Cursor: " lines in stdout (reuses the #693 lifecycle + parse_last_cursor verbatim). - recovery: backfill that BYPASSES the tier-1 seen-ledger so dropped-and-deleted near-dups get re-fetched and re-evaluated under the current pHash threshold. Rides the #693 state machine via a _backfill_bypass_seen flag, cleared on completion / stop. The seen-ledger uses short-lived sync sessions (injected sessionmaker), never held across the walk (avoids the connection-reaping trap). Campaign id resolves from override, an id: URL, or a vanity lookup; unresolvable = loud NOT_FOUND, never a silent empty success. Tests: new test_patreon_ingester.py (modes, ledger skip/idempotency, budget→PARTIAL, recovery bypass, tier-2 disk, drift). The patreon-oriented download_service tests now drive the ingester branch via a stub; the gallery-dl campaign-retry test is replaced by resolution/caching coverage. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/download_service.py | 151 +++++++--- backend/app/services/patreon_ingester.py | 365 +++++++++++++++++++++++ backend/app/services/source_service.py | 2 +- backend/app/tasks/download.py | 5 + tests/test_download_service.py | 246 ++++++++++----- tests/test_patreon_ingester.py | 324 ++++++++++++++++++++ 6 files changed, 984 insertions(+), 109 deletions(-) create mode 100644 backend/app/services/patreon_ingester.py create mode 100644 tests/test_patreon_ingester.py diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 8804483..c84d657 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -29,12 +29,14 @@ from .gallery_dl import ( BACKFILL_CHUNK_SECONDS, BACKFILL_SKIP_VALUE, TICK_SKIP_VALUE, + DownloadResult, ErrorType, GalleryDLService, SourceConfig, parse_last_cursor, ) from .importer import Importer +from .patreon_ingester import PatreonIngester from .patreon_resolver import resolve_campaign_id from .platforms import auth_type_for from .scheduler_service import set_platform_cooldown @@ -78,12 +80,18 @@ class DownloadService: gdl: GalleryDLService, importer: Importer, cred_service: CredentialService, + sync_session_factory=None, ): self.async_session = async_session self.sync_session = sync_session self.gdl = gdl self.importer = importer self.cred_service = cred_service + # Sync sessionmaker the native Patreon ingester opens SHORT-LIVED + # sessions from for its seen-ledger reads/writes (never one held across + # the multi-minute walk — see PatreonIngester). Only the patreon branch + # of phase 2 uses it; gallery-dl sources leave it None. + self.sync_session_factory = sync_session_factory async def download_source(self, source_id: int) -> int: """Returns DownloadEvent.id. Idempotent: in-flight events are returned as-is.""" @@ -118,6 +126,14 @@ class DownloadService: # Operator drives this via POST /api/sources/{id}/backfill {action}. overrides = ctx["config_overrides"] or {} in_backfill = overrides.get("_backfill_state") == "running" + # Recovery (plan #697) reuses the entire #693 backfill state machine — + # cursor checkpoint, time-boxed chunks, complete/stall lifecycle — and + # differs only in bypassing the tier-1 seen-ledger so dropped-and-deleted + # near-dups get re-fetched and re-evaluated under the CURRENT pHash + # threshold (tier-2 disk still spares files we kept). The + # `_backfill_bypass_seen` flag rides alongside the running backfill state; + # download mode is "recovery" when both are set. + bypass_seen = bool(overrides.get("_backfill_bypass_seen")) if in_backfill: skip_value: bool | str = BACKFILL_SKIP_VALUE source_config.timeout = BACKFILL_CHUNK_SECONDS @@ -127,46 +143,35 @@ class DownloadService: else: skip_value = TICK_SKIP_VALUE - effective_url = _effective_url( - ctx["platform"], ctx["url"], ctx["config_overrides"] or {} - ) - - dl_result = await self.gdl.download( - url=effective_url, - artist_slug=ctx["artist_slug"], - platform=ctx["platform"], - source_config=source_config, - cookies_path=ctx["cookies_path"], - auth_token=ctx["auth_token"], - skip_value=skip_value, - ) - resolved_campaign_id: str | None = None - if ( - ctx["platform"] == "patreon" - and not dl_result.success - and "patreon_campaign_id" not in (ctx["config_overrides"] or {}) - and _looks_like_campaign_id_failure(dl_result.stdout, dl_result.stderr) - ): - vanity = _extract_patreon_vanity(ctx["url"]) - if vanity: - log.info( - "Attempting campaign-ID resolution for %s (%s)", - ctx["artist_slug"], vanity, - ) - resolved_campaign_id = await resolve_campaign_id( - vanity, ctx["cookies_path"] - ) - if resolved_campaign_id: - dl_result = await self.gdl.download( - url=f"https://www.patreon.com/id:{resolved_campaign_id}", - artist_slug=ctx["artist_slug"], - platform=ctx["platform"], - source_config=source_config, - cookies_path=ctx["cookies_path"], - auth_token=ctx["auth_token"], - skip_value=skip_value, - ) + if ctx["platform"] == "patreon": + # Native ingester (plan #697) fully replaces gallery-dl for Patreon + # in phase 2 — zero per-file HEADs, native cursor/resume, loud drift + # detection. Returns a DownloadResult-shaped object so phase 3 is + # untouched. The gallery-dl Patreon path + its campaign-id retry are + # removed at the step-5 cutover. + if in_backfill and bypass_seen: + mode = "recovery" + elif in_backfill: + mode = "backfill" + else: + mode = "tick" + dl_result, resolved_campaign_id = await self._run_patreon_ingester( + ctx, source_config, mode, + ) + else: + effective_url = _effective_url( + ctx["platform"], ctx["url"], ctx["config_overrides"] or {} + ) + dl_result = await self.gdl.download( + url=effective_url, + artist_slug=ctx["artist_slug"], + platform=ctx["platform"], + source_config=source_config, + cookies_path=ctx["cookies_path"], + auth_token=ctx["auth_token"], + skip_value=skip_value, + ) # A backfill chunk that hit its time-box but made forward progress is # NORMAL, not a failure — reclassify TIMEOUT → PARTIAL so it reads as @@ -190,6 +195,71 @@ class DownloadService: ctx["event_id"], ctx, dl_result, resolved_campaign_id, ) + async def _run_patreon_ingester( + self, ctx: dict, source_config, mode: str, + ) -> tuple[DownloadResult, str | None]: + """Phase-2 Patreon branch: resolve the campaign id, then run the native + ingester in a worker thread (it is sync requests/subprocess). + + Returns (DownloadResult, resolved_campaign_id). `resolved_campaign_id` is + non-None only when we had to look it up from the vanity URL this run, so + phase 3 caches it on the source the same way the old gallery-dl retry did. + A campaign id we cannot resolve is a loud failure (NOT_FOUND) — never a + silent empty success. + """ + overrides = ctx["config_overrides"] or {} + campaign_id = overrides.get("patreon_campaign_id") + resolved_campaign_id: str | None = None + if not campaign_id: + # A `.../id:` URL already carries the campaign id — no lookup + # needed (and the vanity regex deliberately excludes the id: form). + id_match = re.search(r"/id:(\d+)", ctx["url"]) + if id_match: + campaign_id = id_match.group(1) + else: + vanity = _extract_patreon_vanity(ctx["url"]) + if vanity: + campaign_id = await resolve_campaign_id(vanity, ctx["cookies_path"]) + resolved_campaign_id = campaign_id + + if not campaign_id: + return ( + DownloadResult( + success=False, + url=ctx["url"], + artist_slug=ctx["artist_slug"], + platform="patreon", + error_type=ErrorType.NOT_FOUND, + error_message=( + "Could not resolve Patreon campaign id from the source " + "URL (vanity lookup failed — cookies expired or creator " + "moved?)" + ), + ), + None, + ) + + ingester = PatreonIngester( + images_root=self.gdl.images_root, + cookies_path=ctx["cookies_path"], + session_factory=self.sync_session_factory, + validate=self.gdl._validate_files, + ) + loop = asyncio.get_running_loop() + dl_result = await loop.run_in_executor( + None, + lambda: ingester.run( + source_id=ctx["source_id"], + campaign_id=campaign_id, + artist_slug=ctx["artist_slug"], + url=ctx["url"], + mode=mode, + resume_cursor=source_config.resume_cursor, + time_budget_seconds=source_config.timeout, + ), + ) + return dl_result, resolved_campaign_id + async def _phase1_setup(self, source_id: int) -> dict[str, Any]: source = (await self.async_session.execute( select(Source).options(joinedload(Source.artist)).where(Source.id == source_id) @@ -468,6 +538,9 @@ class DownloadService: new_overrides["_backfill_state"] = "complete" new_overrides.pop("_backfill_cursor", None) new_overrides.pop("_backfill_cursor_stalls", None) + # plan #697: a recovery walk shares this lifecycle; clear its bypass + # flag on completion so the next routine tick honors the seen-ledger. + new_overrides.pop("_backfill_bypass_seen", None) src.config_overrides = new_overrides src.backfill_runs_remaining = 0 return diff --git a/backend/app/services/patreon_ingester.py b/backend/app/services/patreon_ingester.py new file mode 100644 index 0000000..6b26f8c --- /dev/null +++ b/backend/app/services/patreon_ingester.py @@ -0,0 +1,365 @@ +"""Native Patreon ingester — phase-2 orchestrator (build step 3). + +Ties build steps 1 (`patreon_client`) and 2 (`patreon_downloader` + +`patreon_seen_media`) together into a single sync walk that REPLACES the +gallery-dl subprocess for Patreon. `download_service.download_source` calls +`PatreonIngester.run(...)` from phase 2 (in a thread, via run_in_executor, since +everything here is sync `requests`/`subprocess`) and gets back a +`DownloadResult` — the exact shape gallery-dl returns — so phase 1 (DB setup) +and phase 3 (import → pHash dedup → thumbnails → ML) are untouched and cannot +tell the difference. + +Three modes (selected by `download_service` from `config_overrides` state): + - tick — newest→oldest, skip seen (tier-1 ledger + tier-2 disk), early-out + after N contiguous already-have-it items (the cheap native + equivalent of gallery-dl's `exit:20`, now free of per-file HEADs). + - backfill — full-history walk in a time-boxed chunk, resuming from the + pagination cursor checkpoint; reaches the bottom → "complete". + - recovery — like backfill but BYPASSES the tier-1 seen-ledger, so + deliberately-dropped-and-deleted near-dups get re-fetched and + re-evaluated under the current pHash threshold (tier-2 disk skip + still spares files we kept). Triggered by the same #693 backfill + state machine plus the `_backfill_bypass_seen` flag, so the whole + cursor/chunk/complete/stall lifecycle is reused verbatim. + +Cursor contract: the ingester emits gallery-dl-style ``Cursor: `` lines +into the returned `stdout`, one per page it walks. That is exactly what +`download_service`'s existing backfill lifecycle reads via +`parse_last_cursor(stdout, stderr)` — so checkpointing, the TIMEOUT→PARTIAL +reclassification, and completion detection all keep working unchanged. + +The seen-ledger lives in Postgres (`patreon_seen_media`). The ingester opens a +SHORT-LIVED sync session per page batch (via an injected sessionmaker) — never +held across a network fetch — so the multi-minute walk can't strand a checked-out +connection for the server to reap ([[db-connection-held-across-subprocess]]). + +FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. +""" + +from __future__ import annotations + +import logging +import time +from collections.abc import Callable +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.dialects.postgresql import insert as pg_insert + +from ..models import PatreonSeenMedia +from .gallery_dl import DownloadResult, ErrorType +from .patreon_client import ( + MediaItem, + PatreonAPIError, + PatreonClient, + PatreonDriftError, +) +from .patreon_downloader import PatreonDownloader + +log = logging.getLogger(__name__) + +# gallery-dl's `exit:20` default ported over: stop a tick after this many +# CONTIGUOUS already-have-it media (seen-ledger or on-disk). Native walks have +# zero per-file HEADs, so the only cost of a higher number is a few extra cheap +# ledger lookups — 20 is operator-set headroom against paywalled/undownloadable +# items interleaving with archived ones. +_TICK_SEEN_THRESHOLD = 20 + +# Ledger keys are stored in patreon_seen_media.filehash VARCHAR(128); bound any +# synthesized key so a pathologically long file_name can't overflow the column. +_LEDGER_KEY_MAX = 128 + + +def _ledger_key(media: MediaItem) -> str: + """Stable per-media identity for the cross-run seen-ledger. + + A Patreon CDN URL carries a 32-char MD5 (`media.filehash`) — that is the + natural key. Some media have none: Mux/HLS video (`stream.mux.com`, no + content hash at discovery) and the odd inline-content `` pointing at a + hashless URL. The plan calls the video case the ``video::`` + sentinel; `MediaItem` carries no media_id, so the post-scoped filename is the + stable proxy. Bounded to the column width. + """ + if media.filehash: + return media.filehash + return f"{media.post_id}:{media.filename}"[:_LEDGER_KEY_MAX] + + +class PatreonIngester: + """Walk a Patreon campaign's posts, download unseen media, return a + `DownloadResult`. + + Construct with the per-source `cookies_path` and a sync sessionmaker for the + seen-ledger. `client` / `downloader` are injectable seams so unit tests run + without network, subprocess, or a real CDN. + """ + + def __init__( + self, + images_root: Path, + cookies_path: str | None, + session_factory: Callable[[], object], + *, + validate: bool = True, + client: PatreonClient | None = None, + downloader: PatreonDownloader | None = None, + ): + self.images_root = Path(images_root) + self.cookies_path = str(cookies_path) if cookies_path else None + self.session_factory = session_factory + self.client = client if client is not None else PatreonClient(cookies_path) + self.downloader = ( + downloader + if downloader is not None + else PatreonDownloader( + self.images_root, cookies_path, validate=validate + ) + ) + + # -- public ------------------------------------------------------------ + + def run( + self, + *, + source_id: int, + campaign_id: str, + artist_slug: str, + url: str, + mode: str, + resume_cursor: str | None = None, + time_budget_seconds: float = 870.0, + seen_threshold: int = _TICK_SEEN_THRESHOLD, + ) -> DownloadResult: + """Walk + download for one source, returning a gallery-dl-shaped result. + + `mode` is "tick" | "backfill" | "recovery". Recovery bypasses the tier-1 + seen-ledger (tier-2 disk still skips kept files). The walk stops on: + - budget exhaustion (time_budget_seconds) → TIMEOUT / PARTIAL + - tick early-out (seen_threshold contiguous seen) → success + - reaching the bottom of the feed → success (rc 0) + A client-level failure (drift / auth / network) fails the whole run loud. + """ + bypass_seen = mode == "recovery" + start = time.monotonic() + log_lines: list[str] = [] + written: list[str] = [] + downloaded = 0 + errors = 0 + consecutive_seen = 0 + emitted_cursor: str | None = None + reached_bottom = False + budget_hit = False + early_out = False + + def _result( + *, success: bool, return_code: int, + error_type: ErrorType | None, error_message: str | None, + ) -> DownloadResult: + return DownloadResult( + success=success, + url=url, + artist_slug=artist_slug, + platform="patreon", + files_downloaded=downloaded, + files_quarantined=0, + quarantined_paths=[], + written_paths=written, + stdout="\n".join(log_lines), + stderr="", + return_code=return_code, + error_type=error_type, + error_message=error_message, + duration_seconds=time.monotonic() - start, + ) + + try: + for post, included, page_cursor in self.client.iter_posts( + campaign_id, cursor=resume_cursor + ): + # Checkpoint: emit the cursor that FETCHED this page once, the + # moment we START it — so a chunk cut mid-page resumes the page, + # not the one after it (matches the gallery-dl cursor semantics + # download_service's lifecycle already depends on). + if page_cursor and page_cursor != emitted_cursor: + log_lines.append(f"Cursor: {page_cursor}") + emitted_cursor = page_cursor + + # Time-box check at the post boundary (coarse, like a gallery-dl + # chunk). Backfill/recovery resume from emitted_cursor next chunk. + if time.monotonic() - start >= time_budget_seconds: + budget_hit = True + break + + media = self.client.extract_media(post, included) + if not media: + continue + + keys = [_ledger_key(m) for m in media] + seen = ( + set() + if bypass_seen + else self._seen_keys(source_id, keys) + ) + + def _is_seen(m: MediaItem, _seen=seen) -> bool: + return _ledger_key(m) in _seen + + outcomes = self.downloader.download_post( + post, media, artist_slug, is_seen=_is_seen + ) + + to_mark: list[tuple[str, str]] = [] + for media_item, outcome in zip(media, outcomes, strict=False): + key = _ledger_key(media_item) + if outcome.status == "downloaded": + downloaded += 1 + if outcome.path is not None: + written.append(str(outcome.path)) + to_mark.append((key, media_item.post_id)) + consecutive_seen = 0 + elif outcome.status == "skipped_disk": + # Already on disk (a prior run). Reconcile the ledger so a + # later tick skips it at tier-1 without a disk stat, but + # do NOT re-feed it to phase 3 — attach_in_place would see + # the duplicate sha256 and unlink the on-disk copy. + to_mark.append((key, media_item.post_id)) + consecutive_seen += 1 + elif outcome.status == "skipped_seen": + consecutive_seen += 1 + elif outcome.status == "error": + errors += 1 + # An error neither advances nor resets the run-of-seen. + + if mode == "tick" and consecutive_seen >= seen_threshold: + early_out = True + break + + # Mark seen AFTER the network fetch, on its own short session. + if to_mark: + self._mark_seen(source_id, to_mark) + + if early_out: + break + else: + reached_bottom = True + except (PatreonDriftError, PatreonAPIError) as exc: + return self._failure_result(exc, _result) + + if errors: + log_lines.append(f"{errors} media item(s) failed") + log_lines.append( + f"Patreon ingest ({mode}): {downloaded} downloaded, " + f"{errors} error(s)" + + (", reached end" if reached_bottom else "") + + (", time-boxed" if budget_hit else "") + ) + + if budget_hit: + # A chunk that hit its time-box but made forward progress is a + # NORMAL chunk boundary, not a failure (PARTIAL → status "ok"); the + # next chunk resumes from the emitted cursor. No progress → TIMEOUT, + # which feeds download_service's backfill stall-guard. rc<0 mirrors + # subprocess TimeoutExpired so completion detection stays false. + made_progress = downloaded > 0 or emitted_cursor != resume_cursor + if made_progress: + return _result( + success=False, return_code=-1, + error_type=ErrorType.PARTIAL, + error_message=( + f"Patreon backfill chunk: {downloaded} file(s) — continuing" + ), + ) + return _result( + success=False, return_code=-1, + error_type=ErrorType.TIMEOUT, + error_message="Patreon chunk timed out with no progress", + ) + + # Normal success: reached the bottom, or a tick that early-outed. rc 0 + + # error_type None is REQUIRED for a backfill/recovery walk that reached + # the bottom to be marked COMPLETE by + # download_service._apply_backfill_lifecycle — so we return None even + # when downloaded == 0 (a re-confirming walk that found nothing new still + # completed). NO_NEW_CONTENT would read as "not finished" there and trip + # the stall guard. success=True maps to status "ok" regardless. A tick + # that early-outed also returns here; ticks never set backfill state so + # the lifecycle is a no-op for them. + return _result( + success=True, return_code=0, + error_type=None, error_message=None, + ) + + # -- failure mapping --------------------------------------------------- + + def _failure_result(self, exc: Exception, _result) -> DownloadResult: + """Map a client-level exception to a loud, typed failed DownloadResult. + + Step 3 keeps this deliberately coarse — auth-ish drift vs. everything + else — so the integration is correct end-to-end. Step 4 (drift detection + + error categorization) refines the typed-error mapping and adds the + contract test. Either way we NEVER return a silent zero-download + "success": the whole point of the native ingester is to fail red when + Patreon's API shape or our auth changes. + """ + message = str(exc) + lowered = message.lower() + if isinstance(exc, PatreonDriftError): + if "login" in lowered or "challenge" in lowered or "auth" in lowered: + error_type = ErrorType.AUTH_ERROR + else: + error_type = ErrorType.UNKNOWN_ERROR + message = f"Patreon API changed — ingester needs update: {message}" + else: + error_type = ErrorType.NETWORK_ERROR + log.warning("Patreon ingest failed: %s", message) + return _result( + success=False, return_code=1, + error_type=error_type, error_message=message, + ) + + # -- seen-ledger (short-lived sessions) -------------------------------- + + def _seen_keys(self, source_id: int, keys: list[str]) -> set[str]: + """Which of `keys` are already in the ledger for this source. + + One short SELECT on its own session — opened and closed without any + network in between (the GETs happen after, in download_post). + """ + if not keys: + return set() + with self.session_factory() as session: + rows = session.execute( + select(PatreonSeenMedia.filehash).where( + PatreonSeenMedia.source_id == source_id, + PatreonSeenMedia.filehash.in_(keys), + ) + ).scalars().all() + return set(rows) + + def _mark_seen(self, source_id: int, items: list[tuple[str, str]]) -> None: + """Idempotent upsert of (filehash, post_id) ledger rows for a page. + + ON CONFLICT DO NOTHING against the (source_id, filehash) UNIQUE so a + re-sighting — or a concurrent walk — is a harmless no-op + ([[scalar_one_or_none-duplicates]]: never check-then-insert without the + DB constraint backing it). De-dup the batch locally first so a single + page can't present the same key twice to one INSERT. + """ + seen_local: set[str] = set() + values = [] + for key, post_id in items: + if key in seen_local: + continue + seen_local.add(key) + values.append( + {"source_id": source_id, "filehash": key, "post_id": post_id} + ) + if not values: + return + with self.session_factory() as session: + stmt = pg_insert(PatreonSeenMedia).values(values) + stmt = stmt.on_conflict_do_nothing( + constraint="uq_patreon_seen_media_source_id" + ) + session.execute(stmt) + session.commit() diff --git a/backend/app/services/source_service.py b/backend/app/services/source_service.py index 7ae8f9c..0192a9c 100644 --- a/backend/app/services/source_service.py +++ b/backend/app/services/source_service.py @@ -329,7 +329,7 @@ class SourceService: raise LookupError(f"source id={source_id} not found") co = dict(source.config_overrides or {}) for k in ("_backfill_state", "_backfill_cursor", "_backfill_cursor_stalls", - "_backfill_chunks"): + "_backfill_chunks", "_backfill_bypass_seen"): co.pop(k, None) source.config_overrides = co source.backfill_runs_remaining = 0 diff --git a/backend/app/tasks/download.py b/backend/app/tasks/download.py index d72e78b..42c075d 100644 --- a/backend/app/tasks/download.py +++ b/backend/app/tasks/download.py @@ -143,6 +143,11 @@ def download_source(self, source_id: int) -> int: gdl=gdl, importer=importer, cred_service=cred_service, + # The native Patreon ingester opens its own short-lived + # sync sessions for the seen-ledger (never held across + # the walk). Same factory the importer's sync session + # comes from — a different DB connection per checkout. + sync_session_factory=SyncFactory, ) return await svc.download_source(source_id) finally: diff --git a/tests/test_download_service.py b/tests/test_download_service.py index ee5766f..5e44df9 100644 --- a/tests/test_download_service.py +++ b/tests/test_download_service.py @@ -94,6 +94,23 @@ def _fake_gdl_with_result(result): return fake +def _stub_patreon_ingester(svc, result, resolved_campaign_id=None): + """Route the Patreon phase-2 branch (plan #697 native ingester) to a canned + DownloadResult so these tests exercise download_service's + phase-2-result→phase-3 handling (status mapping, backfill lifecycle, health) + without the real ingester's network/DB. The ingester itself is covered by + test_patreon_ingester.py. Returns a list capturing (ctx, source_config, mode) + per call so a test can assert mode / resume_cursor threading.""" + calls = [] + + async def fake(ctx, source_config, mode): + calls.append({"ctx": ctx, "source_config": source_config, "mode": mode}) + return result, resolved_campaign_id + + svc._run_patreon_ingester = fake + return calls + + @pytest.mark.asyncio async def test_download_source_attaches_written_files( db, db_sync, tmp_path, seed_artist_and_source, @@ -112,12 +129,13 @@ async def test_download_source_attaches_written_files( _make_jpg(f1, split="h") _make_jpg(f2, split="v") - fake_gdl = _fake_gdl_with_result(_make_fake_dl_result( + result = _make_fake_dl_result( success=True, written_paths=[str(f1), str(f2)], files_downloaded=2, stdout=f"{f1}\n{f2}\n", - )) + ) + fake_gdl = _fake_gdl_with_result(result) sync_settings = db_sync.execute( select(ImportSettings).where(ImportSettings.id == 1) @@ -137,6 +155,7 @@ async def test_download_source_attaches_written_files( async_session=db, sync_session=db_sync, gdl=fake_gdl, importer=importer, cred_service=cred_service, ) + _stub_patreon_ingester(svc, result) event_id = await svc.download_source(source.id) ev = (await db.execute( @@ -214,40 +233,19 @@ async def test_in_flight_idempotency_returns_existing_event( @pytest.mark.asyncio -async def test_patreon_retry_caches_campaign_id( - db, db_sync, tmp_path, seed_artist_and_source, monkeypatch, +async def test_patreon_resolved_campaign_id_is_cached( + db, db_sync, tmp_path, seed_artist_and_source, ): + """plan #697: the native ingester resolves a Patreon vanity → campaign id + on the fly (replacing gallery-dl's reactive campaign-id retry). When phase 2 + reports a freshly-resolved id, phase 3 caches it on the source so later runs + skip the lookup. Stub the ingester to report a resolved id; assert it lands + in config_overrides.""" from backend.app.services.download_service import DownloadService from backend.app.services.importer import Importer _artist, source = seed_artist_and_source - failed = _make_fake_dl_result( - success=False, written_paths=[], stdout="", - stderr="[patreon][error] Failed to extract campaign ID for alice\n", - ) - succeeded = _make_fake_dl_result(success=True, written_paths=[]) - - download_calls = [] - - async def fake_download(url, **k): - download_calls.append(url) - return failed if len(download_calls) == 1 else succeeded - - fake_gdl = MagicMock() - fake_gdl.download = fake_download - fake_gdl._compute_run_stats = lambda *a, **k: { - "exit_code": 0, "downloaded_count": 0, "skipped_count": 0, - "per_item_failures": 0, "warning_count": 0, "tier_gated_count": 0, - } - fake_gdl._extract_errors_warnings = lambda *a, **k: "" - fake_gdl._truncate_log = lambda x, **k: x - - monkeypatch.setattr( - "backend.app.services.download_service.resolve_campaign_id", - AsyncMock(return_value="99"), - ) - sync_settings = db_sync.execute( select(ImportSettings).where(ImportSettings.id == 1) ).scalar_one() @@ -261,19 +259,102 @@ async def test_patreon_retry_caches_campaign_id( svc = DownloadService( async_session=db, sync_session=db_sync, - gdl=fake_gdl, importer=importer, cred_service=cred_service, + gdl=_fake_gdl_with_result(_make_fake_dl_result(success=True)), + importer=importer, cred_service=cred_service, + ) + _stub_patreon_ingester( + svc, _make_fake_dl_result(success=True, written_paths=[]), + resolved_campaign_id="99", ) await svc.download_source(source.id) - assert len(download_calls) == 2 - assert "id:99" in download_calls[1] - overrides = db_sync.execute( select(Source.config_overrides).where(Source.id == source.id) ).scalar_one() assert overrides.get("patreon_campaign_id") == "99" +@pytest.mark.asyncio +async def test_run_patreon_ingester_resolves_vanity_and_runs( + db, db_sync, tmp_path, seed_artist_and_source, monkeypatch, +): + """_run_patreon_ingester: with no cached campaign id, resolve the vanity and + pass the resolved id straight to the ingester; report it back so phase 3 can + cache it.""" + from backend.app.services import download_service as dl_mod + from backend.app.services.download_service import DownloadService + from backend.app.services.gallery_dl import SourceConfig + + _artist, source = seed_artist_and_source + + monkeypatch.setattr( + dl_mod, "resolve_campaign_id", AsyncMock(return_value="4242"), + ) + run_kwargs = {} + + class _FakeIngester: + def __init__(self, **kw): + pass + + def run(self, **kw): + run_kwargs.update(kw) + return _make_fake_dl_result(success=True, written_paths=[]) + + monkeypatch.setattr(dl_mod, "PatreonIngester", _FakeIngester) + + svc = DownloadService( + async_session=db, sync_session=db_sync, + gdl=_fake_gdl_with_result(_make_fake_dl_result(success=True)), + importer=MagicMock(), cred_service=MagicMock(), + sync_session_factory=MagicMock(), + ) + ctx = { + "source_id": source.id, "url": "https://patreon.com/alice", + "artist_slug": "alice", "cookies_path": None, + "config_overrides": {}, + } + result, resolved = await svc._run_patreon_ingester( + ctx, SourceConfig.from_dict({}), "tick", + ) + assert result.success is True + assert resolved == "4242" + assert run_kwargs["campaign_id"] == "4242" + assert run_kwargs["mode"] == "tick" + + +@pytest.mark.asyncio +async def test_run_patreon_ingester_unresolvable_fails_loud( + db, db_sync, tmp_path, seed_artist_and_source, monkeypatch, +): + """A campaign id we can't resolve is a loud NOT_FOUND failure, never a + silent empty success.""" + from backend.app.services import download_service as dl_mod + from backend.app.services.download_service import DownloadService + from backend.app.services.gallery_dl import ErrorType, SourceConfig + + _artist, source = seed_artist_and_source + + monkeypatch.setattr( + dl_mod, "resolve_campaign_id", AsyncMock(return_value=None), + ) + svc = DownloadService( + async_session=db, sync_session=db_sync, + gdl=MagicMock(), importer=MagicMock(), cred_service=MagicMock(), + sync_session_factory=MagicMock(), + ) + ctx = { + "source_id": source.id, "url": "https://patreon.com/alice", + "artist_slug": "alice", "cookies_path": None, + "config_overrides": {}, + } + result, resolved = await svc._run_patreon_ingester( + ctx, SourceConfig.from_dict({}), "tick", + ) + assert result.success is False + assert result.error_type == ErrorType.NOT_FOUND + assert resolved is None + + # --- FC-3d: finalize hook updates Source health columns ------------------- @@ -376,8 +457,12 @@ async def test_finalize_skipped_preserves_failures_clears_error(db): def _backfill_svc(db, db_sync, tmp_path, result): - """DownloadService wired with a fake gdl returning `result` + a real - importer (so an empty written_paths just attaches nothing).""" + """DownloadService wired with a real importer (so empty written_paths just + attaches nothing) whose Patreon phase-2 branch is stubbed to return `result`. + + The seeded source is Patreon, so phase 2 routes to the native ingester + (plan #697); the stub returns the canned result + captures the per-call + (ctx, source_config, mode). Returns (svc, ingester_calls).""" from backend.app.services.download_service import DownloadService from backend.app.services.importer import Importer @@ -395,7 +480,8 @@ def _backfill_svc(db, db_sync, tmp_path, result): async_session=db, sync_session=db_sync, gdl=fake_gdl, importer=importer, cred_service=cred_service, ) - return svc, fake_gdl + calls = _stub_patreon_ingester(svc, result) + return svc, calls @pytest.mark.asyncio @@ -430,28 +516,25 @@ async def test_backfill_chunk_progress_advances_cursor( async def test_backfill_state_running_selects_backfill_mode_and_resumes( db, db_sync, tmp_path, seed_artist_and_source, ): - """state=='running' drives backfill mode (skip:True + chunk budget) and - threads the stored cursor to gallery-dl as the resume point.""" - from backend.app.services.gallery_dl import ( - BACKFILL_CHUNK_SECONDS, - BACKFILL_SKIP_VALUE, - ) + """state=='running' drives backfill mode (chunk budget) and threads the + stored cursor to the native ingester as the resume point.""" + from backend.app.services.gallery_dl import BACKFILL_CHUNK_SECONDS _artist, source = seed_artist_and_source source.config_overrides = {"_backfill_state": "running", "_backfill_cursor": "03:RESUME:here"} source.backfill_runs_remaining = 5 await db.commit() - svc, fake_gdl = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result( + svc, calls = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result( success=False, written_paths=[], stderr="[patreon][debug] Cursor: 03:RESUME2:next\n", )) await svc.download_source(source.id) - kwargs = fake_gdl.download.call_args.kwargs - assert kwargs["skip_value"] is BACKFILL_SKIP_VALUE - assert kwargs["source_config"].resume_cursor == "03:RESUME:here" - assert kwargs["source_config"].timeout == BACKFILL_CHUNK_SECONDS + assert len(calls) == 1 + assert calls[0]["mode"] == "backfill" + assert calls[0]["source_config"].resume_cursor == "03:RESUME:here" + assert calls[0]["source_config"].timeout == BACKFILL_CHUNK_SECONDS co = (await db.execute( select(Source.config_overrides).where(Source.id == source.id) @@ -605,6 +688,32 @@ async def test_tick_mode_when_not_running_leaves_state_untouched( assert (co or {}).get("_backfill_state") is None +@pytest.mark.asyncio +async def test_recovery_mode_selected_and_flag_cleared_on_complete( + db, db_sync, tmp_path, seed_artist_and_source, +): + """plan #697: `_backfill_bypass_seen` alongside a running backfill selects + recovery mode (ingester bypasses the seen-ledger). A clean rc=0 walk + completes the shared lifecycle AND clears the bypass flag so the next tick + honors the ledger again.""" + _artist, source = seed_artist_and_source + source.config_overrides = {"_backfill_state": "running", "_backfill_bypass_seen": True} + source.backfill_runs_remaining = 5 + await db.commit() + + svc, calls = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result( + success=True, written_paths=[], files_downloaded=0, + )) + await svc.download_source(source.id) + + assert calls[0]["mode"] == "recovery" + co = (await db.execute( + select(Source.config_overrides).where(Source.id == source.id) + )).scalar_one() + assert co.get("_backfill_state") == "complete" + assert "_backfill_bypass_seen" not in co + + @pytest.mark.asyncio async def test_partial_error_type_maps_to_ok_status( db, db_sync, tmp_path, seed_artist_and_source, @@ -642,6 +751,7 @@ async def test_partial_error_type_maps_to_ok_status( async_session=db, sync_session=db_sync, gdl=fake_gdl, importer=importer, cred_service=cred_service, ) + _stub_patreon_ingester(svc, fake_result) event_id = await svc.download_source(source.id) ev = (await db.execute( @@ -677,10 +787,11 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image( _make_jpg(f1, split="h") _make_jpg(f2, split="v") - fake_gdl = _fake_gdl_with_result(_make_fake_dl_result( + result = _make_fake_dl_result( success=True, written_paths=[str(f1), str(f2)], files_downloaded=2, stdout=f"{f1}\n{f2}\n", - )) + ) + fake_gdl = _fake_gdl_with_result(result) sync_settings = db_sync.execute( select(ImportSettings).where(ImportSettings.id == 1) @@ -713,6 +824,7 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image( async_session=db, sync_session=db_sync, gdl=fake_gdl, importer=importer, cred_service=cred_service, ) + _stub_patreon_ingester(svc, result) await svc.download_source(source.id) # Two files attached → two thumbnail enqueues + two ML enqueues, IDs @@ -727,11 +839,11 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image( async def test_releases_db_connections_before_subprocess( db, db_sync, tmp_path, seed_artist_and_source, monkeypatch, ): - """Phase 1's DB connections must be released BEFORE the (up to ~19.5-min - in backfill) gallery-dl subprocess, so they don't idle-die and strand - phase 3 with ConnectionDoesNotExistError (Anduo #40014). Spy on the - async session close and assert it happened before gdl.download runs; - phase 3 must still finalize the event.""" + """Phase 1's DB connections must be released BEFORE the (multi-minute) + phase-2 fetch, so they don't idle-die and strand phase 3 with + ConnectionDoesNotExistError (Anduo #40014). Spy on the async session close + and assert it happened before the phase-2 work (here the native Patreon + ingester) runs; phase 3 must still finalize the event.""" from backend.app.services.download_service import DownloadService from backend.app.services.importer import Importer @@ -747,19 +859,9 @@ async def test_releases_db_connections_before_subprocess( monkeypatch.setattr(db, "close", spy_close) seen = {} - - async def fake_download(url, **k): - seen["closed_before_download"] = closed["async"] - return _make_fake_dl_result(success=True, written_paths=[], stdout="") - - fake_gdl = MagicMock() - fake_gdl.download = fake_download - fake_gdl._compute_run_stats = lambda *a, **k: { - "exit_code": 0, "downloaded_count": 0, "skipped_count": 0, - "per_item_failures": 0, "warning_count": 0, "tier_gated_count": 0, - } - fake_gdl._extract_errors_warnings = lambda *a, **k: "" - fake_gdl._truncate_log = lambda x, **k: x + fake_gdl = _fake_gdl_with_result( + _make_fake_dl_result(success=True, written_paths=[], stdout="") + ) sync_settings = db_sync.execute( select(ImportSettings).where(ImportSettings.id == 1) @@ -776,9 +878,15 @@ async def test_releases_db_connections_before_subprocess( async_session=db, sync_session=db_sync, gdl=fake_gdl, importer=importer, cred_service=cred_service, ) + + async def fake_ingest(ctx, source_config, mode): + seen["closed_before_phase2"] = closed["async"] + return _make_fake_dl_result(success=True, written_paths=[], stdout=""), None + + svc._run_patreon_ingester = fake_ingest event_id = await svc.download_source(source.id) - assert seen["closed_before_download"] is True + assert seen["closed_before_phase2"] is True ev = (await db.execute( select(DownloadEvent).where(DownloadEvent.id == event_id) )).scalar_one() diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py new file mode 100644 index 0000000..17ac2cf --- /dev/null +++ b/tests/test_patreon_ingester.py @@ -0,0 +1,324 @@ +"""PatreonIngester tests (native ingester build step 3). + +The HTTP client and the media downloader are stubbed (injected seams), so these +exercise the ingester's own logic — mode behavior, the two-tier skip, cursor +emission, the budget cutoff, and the Postgres seen-ledger — without network or a +real CDN. The ledger is real (a sync sessionmaker bound to the test engine), so +the tier-1 skip and the idempotent mark-seen run against actual rows. +""" + +import pytest +from sqlalchemy import func, select +from sqlalchemy.orm import sessionmaker + +from backend.app.models import Artist, PatreonSeenMedia, Source +from backend.app.services.gallery_dl import ErrorType +from backend.app.services.patreon_client import MediaItem, PatreonDriftError +from backend.app.services.patreon_downloader import MediaOutcome +from backend.app.services.patreon_ingester import PatreonIngester, _ledger_key + +pytestmark = pytest.mark.integration + + +# --- fakes ---------------------------------------------------------------- + + +def _media(post_id, n, *, filehash=None, kind="images"): + # Deterministic 32-hex-ish key, unique per (post_id, n). + fh = filehash if filehash is not None else f"{post_id}{n}".encode().hex().ljust(32, "0")[:32] + return MediaItem( + url=f"https://cdn.patreon.com/{fh}/{n}.jpg", + filename=f"{n}.jpg", + kind=kind, + filehash=fh, + post_id=post_id, + ) + + +class _FakeClient: + """Stub PatreonClient. `pages` is a list of (page_cursor, [posts]); each post + is (post_id, [MediaItem]). `raise_on_first` lets a test trip drift.""" + + def __init__(self, pages, raise_exc=None): + self._pages = pages + self._raise_exc = raise_exc + self.consumed_posts = 0 + + def iter_posts(self, campaign_id, cursor=None): + if self._raise_exc is not None: + raise self._raise_exc + for page_cursor, posts in self._pages: + for post_id, media in posts: + self.consumed_posts += 1 + yield {"id": post_id, "_media": media}, {}, page_cursor + + def extract_media(self, post, included_index): + return post["_media"] + + +class _FakeDownloader: + """Stub PatreonDownloader. Honors the injected is_seen (tier-1); media in + `on_disk` report skipped_disk (tier-2); everything else downloads.""" + + def __init__(self, tmp_path, on_disk=None): + self.tmp_path = tmp_path + self.on_disk = set(on_disk or ()) + self.download_calls = 0 + + def download_post(self, post, media_items, artist_slug, *, is_seen): + outcomes = [] + for m in media_items: + if is_seen(m): + outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None)) + elif _ledger_key(m) in self.on_disk: + p = self.tmp_path / f"{m.post_id}_{m.filename}" + outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None)) + else: + self.download_calls += 1 + p = self.tmp_path / f"{m.post_id}_{m.filename}" + p.write_bytes(b"x") + outcomes.append(MediaOutcome(media=m, status="downloaded", path=p, error=None)) + return outcomes + + +@pytest.fixture +async def source_id(db): + artist = Artist(name="Ingest", slug="ingest") + db.add(artist) + await db.flush() + source = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/ingest", enabled=True, config_overrides={}, + ) + db.add(source) + await db.commit() + return source.id + + +def _ingester(sync_engine, tmp_path, client, downloader): + factory = sessionmaker(sync_engine, expire_on_commit=False) + return PatreonIngester( + images_root=tmp_path, cookies_path=None, session_factory=factory, + client=client, downloader=downloader, + ) + + +def _count_ledger(sync_engine, source_id): + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + return s.execute( + select(func.count(PatreonSeenMedia.id)).where( + PatreonSeenMedia.source_id == source_id + ) + ).scalar_one() + + +# --- tick ----------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_tick_downloads_unseen_and_marks_seen(source_id, sync_engine, tmp_path): + m1, m2 = _media("p1", 1), _media("p1", 2) + client = _FakeClient([(None, [("p1", [m1, m2])])]) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + + assert result.success is True + assert result.return_code == 0 + assert result.files_downloaded == 2 + assert len(result.written_paths) == 2 + assert _count_ledger(sync_engine, source_id) == 2 + + +@pytest.mark.asyncio +async def test_tick_skips_seen_via_ledger(source_id, sync_engine, tmp_path): + m1 = _media("p1", 1) + # Pre-seed the ledger with m1's key. + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1")) + s.commit() + + client = _FakeClient([(None, [("p1", [m1])])]) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + assert result.files_downloaded == 0 + assert downloader.download_calls == 0 + + +@pytest.mark.asyncio +async def test_tick_early_out_after_threshold(source_id, sync_engine, tmp_path): + # Three posts, one already-seen media each; threshold 2 → stop before the 3rd. + seen_media = [_media(f"p{i}", 1) for i in range(1, 4)] + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + for m in seen_media: + s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m), post_id=m.post_id)) + s.commit() + + pages = [(None, [(m.post_id, [m]) for m in seen_media])] + client = _FakeClient(pages) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", seen_threshold=2, + ) + assert result.success is True + # Stopped after the 2nd contiguous seen item — never consumed the 3rd post. + assert client.consumed_posts == 2 + + +# --- backfill ------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_backfill_reaches_bottom_signals_complete(source_id, sync_engine, tmp_path): + pages = [ + (None, [("p1", [_media("p1", 1)])]), + ("CUR2", [("p2", [_media("p2", 1)])]), + ] + client = _FakeClient(pages) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="backfill", + ) + # rc 0 + error_type None is what _apply_backfill_lifecycle reads as "complete". + assert result.return_code == 0 + assert result.error_type is None + assert result.files_downloaded == 2 + # The page-2 cursor was emitted for checkpointing. + assert "Cursor: CUR2" in result.stdout + + +@pytest.mark.asyncio +async def test_backfill_budget_cut_returns_partial_with_progress( + source_id, sync_engine, tmp_path, monkeypatch, +): + # Deterministic clock: start=0, post1 check=10 (ok), post2 check=200 (>budget). + import backend.app.services.patreon_ingester as mod + ticks = iter([0.0, 10.0, 200.0, 250.0]) + last = [0.0] + + def fake_monotonic(): + try: + last[0] = next(ticks) + except StopIteration: + pass + return last[0] + + monkeypatch.setattr(mod.time, "monotonic", fake_monotonic) + + pages = [ + ("CUR1", [("p1", [_media("p1", 1)])]), + ("CUR2", [("p2", [_media("p2", 1)])]), + ] + client = _FakeClient(pages) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="backfill", + resume_cursor=None, time_budget_seconds=100.0, + ) + assert result.error_type == ErrorType.PARTIAL + assert result.return_code == -1 + assert result.files_downloaded == 1 # post1 downloaded before the cut + assert downloader.download_calls == 1 # post2's body never ran + assert "Cursor: CUR1" in result.stdout + + +# --- recovery ------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_recovery_bypasses_seen_ledger(source_id, sync_engine, tmp_path): + m1 = _media("p1", 1) + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1")) + s.commit() + + client = _FakeClient([(None, [("p1", [m1])])]) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="recovery", + ) + # Ledger says seen, but recovery bypasses tier-1 → re-downloaded for a fresh + # pHash evaluation under the current threshold. + assert result.files_downloaded == 1 + assert downloader.download_calls == 1 + + +@pytest.mark.asyncio +async def test_recovery_tier2_disk_still_skips(source_id, sync_engine, tmp_path): + m1 = _media("p1", 1) + client = _FakeClient([(None, [("p1", [m1])])]) + # File still on disk (a kept image) → tier-2 spares it even under recovery. + downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)}) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="recovery", + ) + assert result.files_downloaded == 0 + assert downloader.download_calls == 0 + # Disk-skip still reconciles the ledger so a later tick skips at tier-1. + assert _count_ledger(sync_engine, source_id) == 1 + + +# --- ledger + drift ------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_mark_seen_is_idempotent(source_id, sync_engine, tmp_path): + m1 = _media("p1", 1) + client = _FakeClient([(None, [("p1", [m1])])]) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + # Same media downloaded twice across two runs — the second marks seen again. + ing._mark_seen(source_id, [(_ledger_key(m1), "p1")]) + ing._mark_seen(source_id, [(_ledger_key(m1), "p1")]) + assert _count_ledger(sync_engine, source_id) == 1 + + +@pytest.mark.asyncio +async def test_drift_fails_loud(source_id, sync_engine, tmp_path): + client = _FakeClient([], raise_exc=PatreonDriftError("missing top-level 'data' key")) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + assert result.success is False + assert "Patreon API changed" in (result.error_message or "") + + +def test_ledger_key_video_has_no_filehash(): + video = MediaItem( + url="https://stream.mux.com/abc.m3u8", filename="clip.mp4", + kind="postfile", filehash=None, post_id="p9", + ) + assert _ledger_key(video) == "p9:clip.mp4" -- 2.52.0 From 682beafbc586f8b6e5229dd29dc3983bdaa72bf5 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 21:56:58 -0400 Subject: [PATCH 06/26] =?UTF-8?q?feat(patreon):=20drift=20detection=20+=20?= =?UTF-8?q?error=20categorization=20=E2=80=94=20build=20step=204=20(plan?= =?UTF-8?q?=20#697)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Typed, loud failure mapping for the native Patreon ingester so a changed API shape or expired auth never silently zero-downloads as "success". - New ErrorType.API_DRIFT (free varchar error_type col → no migration): distinct from auth so the operator knows the fix is updating the ingester, not rotating cookies. - patreon_client: PatreonAPIError carries status_code; new PatreonAuthError for 401/403 + HTML-login/non-JSON bodies (reclassified from drift — expired-session is auth, actionable as "rotate cookies"). - patreon_ingester._failure_result maps: PatreonAuthError→AUTH_ERROR, PatreonDriftError→API_DRIFT ("Patreon API changed — ingester needs update"), HTTP 429→RATE_LIMITED, 404→NOT_FOUND, other HTTP→HTTP_ERROR, transport→NETWORK_ERROR. (429 thus drives the platform cooldown.) - FailingSourcesCard: api_drift chip (red) + hint. Contract test (new test_patreon_contract.py): the recorded /api/posts fixture must parse end-to-end (no drift, 5 media across 4 posts) AND the request params must still carry every field the parser depends on (file_name, image_urls/download_url, images/attachments_media/media includes, content/post_file/image post fields) — a trim of either trips a red build. Plus client HTTP-status classification tests and ingester error-type mapping tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/gallery_dl.py | 6 ++ backend/app/services/patreon_client.py | 52 +++++++++++--- backend/app/services/patreon_ingester.py | 46 +++++++----- .../subscriptions/FailingSourcesCard.vue | 2 + tests/test_patreon_client.py | 54 ++++++++++++++ tests/test_patreon_contract.py | 72 +++++++++++++++++++ tests/test_patreon_ingester.py | 64 ++++++++++++++--- 7 files changed, 261 insertions(+), 35 deletions(-) create mode 100644 tests/test_patreon_contract.py diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 9d59f3c..9d3dc92 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -40,6 +40,12 @@ class ErrorType(StrEnum): HTTP_ERROR = "http_error" UNSUPPORTED_URL = "unsupported_url" VALIDATION_FAILED = "validation_failed" + # Native Patreon ingester only (plan #697): a response parsed as JSON but + # didn't match the JSON:API shape the ingester depends on (missing `data`, + # media lacking `file_name`/`url`, etc.). Distinct from AUTH_ERROR — the fix + # is updating the ingester's field-set/parser, not rotating credentials. The + # contract test guards against silently shipping this. + API_DRIFT = "api_drift" # Run made real progress (downloaded ≥1 file) but did not finish in the # subprocess budget. Distinct from UNKNOWN_ERROR — the downstream status # mapping classifies this as "ok" because the next tick continues. diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index 28bfab4..a3685a9 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -77,16 +77,34 @@ _MAX_EXT_LEN = 16 class PatreonAPIError(Exception): - """Base for native Patreon client failures.""" + """Base for native Patreon client failures. + + `status_code` carries the HTTP status when the failure was an HTTP response + (None for transport-level / parse failures), so the ingester can map it to a + DownloadResult.error_type (429 → rate_limited, 404 → not_found, …). + """ + + def __init__(self, message: str, *, status_code: int | None = None): + super().__init__(message) + self.status_code = status_code + + +class PatreonAuthError(PatreonAPIError): + """Authentication / authorization failure — missing or expired session + cookies, an insufficient pledge tier, or an HTML login/challenge page served + where JSON was expected. DISTINCT from drift: the fix is rotating the + credential, not updating the ingester. Maps to error_type 'auth_error'. + """ class PatreonDriftError(PatreonAPIError): - """A response did not match the JSON:API shape we depend on. + """A JSON response did not match the JSON:API shape we depend on. - Raised for: a non-JSON / HTML-login response where JSON was expected, a - missing top-level `data` list, or a media resource lacking the - `file_name`/`url` fields we resolve against. Fail loud so the later import - step flags API drift instead of silently importing nothing. + Raised for: a missing top-level `data` list, `data` not a list, or a media + resource lacking the `file_name`/`url` fields we resolve against. Fail loud + so the import step flags API drift (ingester needs update) instead of + silently importing nothing. An HTML-login / non-JSON body is auth, not + drift — that raises PatreonAuthError. """ @@ -221,20 +239,32 @@ class PatreonClient: f"Patreon posts request failed (campaign_id={campaign_id}): {exc}" ) from exc + if resp.status_code in (401, 403): + # Auth rejected — expired/missing cookies or an insufficient tier. + # Actionable as "rotate credentials", so it's auth, not drift/http. + raise PatreonAuthError( + f"Patreon posts API returned HTTP {resp.status_code} — auth " + f"rejected (cookies expired or tier insufficient; " + f"campaign_id={campaign_id})", + status_code=resp.status_code, + ) if resp.status_code != 200: raise PatreonAPIError( f"Patreon posts API returned HTTP {resp.status_code} " - f"(campaign_id={campaign_id})" + f"(campaign_id={campaign_id})", + status_code=resp.status_code, ) try: payload = resp.json() except ValueError as exc: # A non-JSON body here is almost always the HTML login/challenge - # page served when cookies are missing/expired — that is drift - # from the JSON:API contract, not a transient network error. - raise PatreonDriftError( + # page served when cookies are missing/expired — that is an AUTH + # failure (rotate cookies), not API drift (update the ingester) and + # not a transient network error. + raise PatreonAuthError( "Patreon posts API returned a non-JSON response (likely an " - f"HTML login/challenge page; campaign_id={campaign_id}): {exc}" + f"HTML login/challenge page — session expired; " + f"campaign_id={campaign_id}): {exc}" ) from exc return payload diff --git a/backend/app/services/patreon_ingester.py b/backend/app/services/patreon_ingester.py index 6b26f8c..5409e81 100644 --- a/backend/app/services/patreon_ingester.py +++ b/backend/app/services/patreon_ingester.py @@ -51,6 +51,7 @@ from .gallery_dl import DownloadResult, ErrorType from .patreon_client import ( MediaItem, PatreonAPIError, + PatreonAuthError, PatreonClient, PatreonDriftError, ) @@ -242,7 +243,9 @@ class PatreonIngester: break else: reached_bottom = True - except (PatreonDriftError, PatreonAPIError) as exc: + except PatreonAPIError as exc: + # Base of PatreonAuthError + PatreonDriftError — catches every + # client-level failure; _failure_result maps it to a typed error. return self._failure_result(exc, _result) if errors: @@ -294,24 +297,35 @@ class PatreonIngester: def _failure_result(self, exc: Exception, _result) -> DownloadResult: """Map a client-level exception to a loud, typed failed DownloadResult. - Step 3 keeps this deliberately coarse — auth-ish drift vs. everything - else — so the integration is correct end-to-end. Step 4 (drift detection - + error categorization) refines the typed-error mapping and adds the - contract test. Either way we NEVER return a silent zero-download - "success": the whole point of the native ingester is to fail red when - Patreon's API shape or our auth changes. + We NEVER return a silent zero-download "success" — the whole point of the + native ingester is to fail RED when Patreon's API shape or our auth + changes. The typed mapping lets FailingSourcesCard render the right chip + and tells the operator what to do: + - PatreonAuthError → AUTH_ERROR (rotate cookies) + - PatreonDriftError → API_DRIFT (ingester field-set/parser needs update) + - HTTP 429 / 404 → RATE_LIMITED / NOT_FOUND + - other HTTP status → HTTP_ERROR; transport failure → NETWORK_ERROR + + PatreonAuthError and PatreonDriftError both subclass PatreonAPIError, so + they must be matched before the generic HTTP/transport fallthrough. """ message = str(exc) - lowered = message.lower() - if isinstance(exc, PatreonDriftError): - if "login" in lowered or "challenge" in lowered or "auth" in lowered: - error_type = ErrorType.AUTH_ERROR - else: - error_type = ErrorType.UNKNOWN_ERROR + if isinstance(exc, PatreonAuthError): + error_type = ErrorType.AUTH_ERROR + elif isinstance(exc, PatreonDriftError): + error_type = ErrorType.API_DRIFT message = f"Patreon API changed — ingester needs update: {message}" - else: - error_type = ErrorType.NETWORK_ERROR - log.warning("Patreon ingest failed: %s", message) + else: # generic PatreonAPIError: HTTP non-2xx (status_code set) or transport + status = getattr(exc, "status_code", None) + if status == 429: + error_type = ErrorType.RATE_LIMITED + elif status == 404: + error_type = ErrorType.NOT_FOUND + elif status is not None: + error_type = ErrorType.HTTP_ERROR + else: + error_type = ErrorType.NETWORK_ERROR + log.warning("Patreon ingest failed (%s): %s", error_type.value, message) return _result( success=False, return_code=1, error_type=error_type, error_message=message, diff --git a/frontend/src/components/subscriptions/FailingSourcesCard.vue b/frontend/src/components/subscriptions/FailingSourcesCard.vue index 024a4df..f456904 100644 --- a/frontend/src/components/subscriptions/FailingSourcesCard.vue +++ b/frontend/src/components/subscriptions/FailingSourcesCard.vue @@ -93,6 +93,7 @@ const ERROR_TYPE_COLOR = { unsupported_url: 'error', http_error: 'error', unknown_error: 'error', + api_drift: 'error', partial: 'info', tier_limited: 'info', no_new_content: 'info', @@ -108,6 +109,7 @@ const ERROR_TYPE_HINT = { http_error: 'Generic HTTP error — see Logs.', unsupported_url: 'gallery-dl does not support this URL pattern.', unknown_error: 'Could not classify — see Logs.', + api_drift: 'Patreon changed its API shape — the native ingester needs a code update.', } function errorTypeColor(t) { return ERROR_TYPE_COLOR[t] || 'error' } function errorTypeHint(t) { return ERROR_TYPE_HINT[t] || '' } diff --git a/tests/test_patreon_client.py b/tests/test_patreon_client.py index 02bd328..268c3bb 100644 --- a/tests/test_patreon_client.py +++ b/tests/test_patreon_client.py @@ -12,6 +12,8 @@ import pytest from backend.app.services.patreon_client import ( MediaItem, + PatreonAPIError, + PatreonAuthError, PatreonClient, PatreonDriftError, parse_cursor_from_url, @@ -184,3 +186,55 @@ def test_media_referenced_but_absent_raises_drift(client): } with pytest.raises(PatreonDriftError): client.extract_media(post, {}) + + +# -- HTTP status classification (plan #697 step 4) ------------------------- + + +class _FakeResp: + def __init__(self, status_code, *, json_data=None, raise_json=False): + self.status_code = status_code + self._json_data = json_data + self._raise_json = raise_json + + def json(self): + if self._raise_json: + raise ValueError("Expecting value: line 1 column 1") + return self._json_data + + +def _client_returning(monkeypatch, resp): + client = PatreonClient(cookies_path=None) + monkeypatch.setattr(client._session, "get", lambda *a, **k: resp) + return client + + +@pytest.mark.parametrize("status", [401, 403]) +def test_fetch_auth_status_raises_auth_error(monkeypatch, status): + client = _client_returning(monkeypatch, _FakeResp(status)) + with pytest.raises(PatreonAuthError) as ei: + client._fetch("5555", None) + assert ei.value.status_code == status + + +def test_fetch_non_json_body_is_auth_not_drift(monkeypatch): + # An HTML login/challenge page (non-JSON) means the session expired — auth, + # actionable as "rotate cookies", NOT API drift. + client = _client_returning(monkeypatch, _FakeResp(200, raise_json=True)) + with pytest.raises(PatreonAuthError): + client._fetch("5555", None) + + +@pytest.mark.parametrize("status", [404, 429, 500]) +def test_fetch_other_status_raises_api_error_with_code(monkeypatch, status): + client = _client_returning(monkeypatch, _FakeResp(status)) + with pytest.raises(PatreonAPIError) as ei: + client._fetch("5555", None) + assert ei.value.status_code == status + # Not auth — the ingester maps these by status, not to auth_error. + assert not isinstance(ei.value, PatreonAuthError) + + +def test_fetch_ok_returns_payload(monkeypatch): + client = _client_returning(monkeypatch, _FakeResp(200, json_data={"data": []})) + assert client._fetch("5555", None) == {"data": []} diff --git a/tests/test_patreon_contract.py b/tests/test_patreon_contract.py new file mode 100644 index 0000000..1f6a520 --- /dev/null +++ b/tests/test_patreon_contract.py @@ -0,0 +1,72 @@ +"""Patreon API contract test (native ingester build step 4). + +The drift tripwire the plan calls for: if the field-set we SEND or the parser we +resolve responses against drifts from what real Patreon returns, this build goes +RED — instead of the ingester silently importing nothing in production (the exact +failure mode the native ingester exists to escape). + +Two halves: + - the recorded `/api/posts` fixture must still parse end-to-end (no drift), and + - the request params must still carry every field the parser depends on (the + fixture already has the data, so trimming the request would NOT trip the + parse half — this half guards the request shape directly). + +Pure: no network, no DB. +""" + +import json +from pathlib import Path + +from backend.app.services.patreon_client import MediaItem, PatreonClient + +_FIXTURE = Path(__file__).parent / "fixtures" / "patreon_posts_page1.json" + +# Pinned media total across the 4 fixture posts: 1001→2 (gallery, image_large +# cover deduped), 1002→1 (attachment+postfile collapse), 1003→1 (inline content +# img), 1004→1 (video postfile). A parser change that drops or doubles media +# trips this. +_EXPECTED_MEDIA_TOTAL = 5 +_KNOWN_KINDS = {"images", "image_large", "attachments", "postfile", "content"} + + +def test_recorded_fixture_parses_end_to_end(): + response = json.loads(_FIXTURE.read_text()) + client = PatreonClient(cookies_path=None) + + client._validate_response(response) # top-level shape must hold + index = client._transform(response) + + total = 0 + for post in response["data"]: + items = client.extract_media(post, index) # must not raise drift + for m in items: + assert isinstance(m, MediaItem) + assert m.url, f"empty url in post {post['id']}" + assert m.filename, f"empty filename in post {post['id']}" + assert m.kind in _KNOWN_KINDS, f"unknown kind {m.kind!r}" + assert m.post_id == post["id"] + total += len(items) + + assert total == _EXPECTED_MEDIA_TOTAL + + +def test_request_field_set_carries_parser_dependencies(): + """The params we SEND must keep requesting the fields the parser resolves + against. Trimming `fields[media]=file_name`, an `include` relationship, or a + `fields[post]` key would make real responses parse to nothing — caught here, + not by the fixture parse above.""" + params = PatreonClient(cookies_path=None)._params("5555", None) + + media_fields = params["fields[media]"] + assert "file_name" in media_fields + assert "image_urls" in media_fields or "download_url" in media_fields + + includes = params["include"] + for rel in ("images", "attachments_media", "media"): + assert rel in includes, f"missing include relationship {rel}" + + post_fields = params["fields[post]"] + for field in ("content", "post_file", "image"): + assert field in post_fields, f"missing post field {field}" + + assert params["filter[campaign_id]"] == "5555" diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index 17ac2cf..ef356d7 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -13,7 +13,12 @@ from sqlalchemy.orm import sessionmaker from backend.app.models import Artist, PatreonSeenMedia, Source from backend.app.services.gallery_dl import ErrorType -from backend.app.services.patreon_client import MediaItem, PatreonDriftError +from backend.app.services.patreon_client import ( + MediaItem, + PatreonAPIError, + PatreonAuthError, + PatreonDriftError, +) from backend.app.services.patreon_downloader import MediaOutcome from backend.app.services.patreon_ingester import PatreonIngester, _ledger_key @@ -302,20 +307,63 @@ async def test_mark_seen_is_idempotent(source_id, sync_engine, tmp_path): assert _count_ledger(sync_engine, source_id) == 1 -@pytest.mark.asyncio -async def test_drift_fails_loud(source_id, sync_engine, tmp_path): - client = _FakeClient([], raise_exc=PatreonDriftError("missing top-level 'data' key")) - downloader = _FakeDownloader(tmp_path) - ing = _ingester(sync_engine, tmp_path, client, downloader) - - result = ing.run( +def _run_with_client_error(source_id, sync_engine, tmp_path, exc): + client = _FakeClient([], raise_exc=exc) + ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path)) + return ing.run( source_id=source_id, campaign_id="c1", artist_slug="ingest", url="https://patreon.com/ingest", mode="tick", ) + + +@pytest.mark.asyncio +async def test_drift_maps_to_api_drift_and_fails_loud(source_id, sync_engine, tmp_path): + result = _run_with_client_error( + source_id, sync_engine, tmp_path, + PatreonDriftError("missing top-level 'data' key"), + ) assert result.success is False + assert result.error_type == ErrorType.API_DRIFT assert "Patreon API changed" in (result.error_message or "") +@pytest.mark.asyncio +async def test_auth_error_maps_to_auth_error(source_id, sync_engine, tmp_path): + result = _run_with_client_error( + source_id, sync_engine, tmp_path, + PatreonAuthError("HTTP 403 — auth rejected", status_code=403), + ) + assert result.success is False + assert result.error_type == ErrorType.AUTH_ERROR + + +@pytest.mark.asyncio +async def test_rate_limit_status_maps_to_rate_limited(source_id, sync_engine, tmp_path): + result = _run_with_client_error( + source_id, sync_engine, tmp_path, + PatreonAPIError("HTTP 429", status_code=429), + ) + assert result.error_type == ErrorType.RATE_LIMITED + + +@pytest.mark.asyncio +async def test_not_found_status_maps_to_not_found(source_id, sync_engine, tmp_path): + result = _run_with_client_error( + source_id, sync_engine, tmp_path, + PatreonAPIError("HTTP 404", status_code=404), + ) + assert result.error_type == ErrorType.NOT_FOUND + + +@pytest.mark.asyncio +async def test_transport_error_maps_to_network_error(source_id, sync_engine, tmp_path): + result = _run_with_client_error( + source_id, sync_engine, tmp_path, + PatreonAPIError("connection reset"), # no status_code → transport + ) + assert result.error_type == ErrorType.NETWORK_ERROR + + def test_ledger_key_video_has_no_filehash(): video = MediaItem( url="https://stream.mux.com/abc.m3u8", filename="clip.mp4", -- 2.52.0 From ec43e823e170cef2fdb356bb6f112eaf7020b1c9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 22:18:03 -0400 Subject: [PATCH 07/26] =?UTF-8?q?feat(patreon):=20recovery=20UI=20+=20gall?= =?UTF-8?q?ery-dl=20cutover=20=E2=80=94=20build=20step=205=20(plan=20#697)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Final step of the native Patreon ingester: a first-class Recovery action, and removal of the now-dead gallery-dl Patreon path. Recovery (rules #23/#24/#27 — full product, with UI): - source_service.start_recovery arms the #693 backfill state machine PLUS `_backfill_bypass_seen`, flipping download mode to recovery (bypass the seen-ledger to re-fetch dropped-and-deleted near-dups and re-evaluate under the current pHash threshold). Stop via the shared stop_backfill. - SourceRecord exposes backfill_bypass_seen; POST /sources/{id}/backfill gains action="recover". - Frontend: Recovery button (Patreon-only, mdi-backup-restore) on SourceRow + SourceCard; the running badge labels "Recovering (N)" vs "Backfilling (N)"; the Stop tooltip says "Stop recovery". sources.js recoverSource + SubscriptionsTab onRecover. Cutover (rule #22 — no legacy): - gallery_dl: removed PLATFORM_DEFAULTS["patreon"], the patreon files/cursor branch in _build_config_for_source, and the patreon/Mux yt-dlp Referer/Origin block (was patreon-specific and wrongly tagged the other platforms' yt-dlp fetches; native ingester owns it now). - download_service: removed the dead campaign-id-retry helpers (_looks_like_campaign_id_failure / _CAMPAIGN_ID_FAILURE_PATTERN) and _effective_url. Vanity→campaign resolution + resume_cursor + the cursor/PARTIAL lifecycle stay — they serve the native ingester. Tests: removed the two obsolete patreon-gallery-dl config tests (yt-dlp Referer, resume-cursor); repointed the generic skip-value config tests to subscribestar; added start_recovery + recover-endpoint coverage (backfill_bypass_seen). gallery-dl stays for the other 5 platforms. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/api/sources.py | 31 ++++--- backend/app/services/download_service.py | 20 ++--- backend/app/services/gallery_dl.py | 83 ++++++------------- backend/app/services/patreon_downloader.py | 9 +- backend/app/services/source_service.py | 33 ++++++++ .../components/subscriptions/SourceCard.vue | 19 ++++- .../components/subscriptions/SourceRow.vue | 17 +++- .../subscriptions/SubscriptionsTab.vue | 18 ++++ frontend/src/stores/sources.js | 9 ++ tests/test_api_sources.py | 25 ++++++ tests/test_gallery_dl_service.py | 41 +++------ tests/test_source_service.py | 32 +++++++ 12 files changed, 211 insertions(+), 126 deletions(-) diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index d33e317..56c5cd6 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -122,23 +122,30 @@ async def delete_source(source_id: int): @sources_bp.route("//backfill", methods=["POST"]) async def set_backfill(source_id: int): - """Plan #693: start or stop a run-until-done backfill. Body: - `{"action": "start" | "stop"}` (default "start"). 'start' walks the full - post history in time-boxed chunks until it reaches the bottom (then the - source shows 'complete'); 'stop' cancels back to tick mode. Returns the - updated source dict (incl. backfill_state / backfill_chunks).""" + """Plan #693/#697: start/stop a run-until-done backfill, or start a recovery. + Body: `{"action": "start" | "stop" | "recover"}` (default "start"). 'start' + walks the full post history in time-boxed chunks until it reaches the bottom + (then the source shows 'complete'); 'recover' is the same walk but bypasses + the Patreon seen-ledger to re-fetch dropped-and-deleted near-dups under the + current pHash threshold; 'stop' cancels either back to tick mode. Returns the + updated source dict (incl. backfill_state / backfill_chunks / + backfill_bypass_seen).""" payload = await request.get_json(silent=True) or {} action = payload.get("action", "start") - if action not in ("start", "stop"): - return _bad("invalid_action", detail="action must be 'start' or 'stop'") + if action not in ("start", "stop", "recover"): + return _bad( + "invalid_action", + detail="action must be 'start', 'stop', or 'recover'", + ) async with get_session() as session: try: svc = SourceService(session) - record = ( - await svc.start_backfill(source_id) - if action == "start" - else await svc.stop_backfill(source_id) - ) + if action == "start": + record = await svc.start_backfill(source_id) + elif action == "recover": + record = await svc.start_recovery(source_id) + else: + record = await svc.stop_backfill(source_id) except LookupError: return _bad("not_found", status=404) return jsonify(record.to_dict()) diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index c84d657..7a86917 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -44,11 +44,14 @@ from .scheduler_service import set_platform_cooldown log = logging.getLogger(__name__) +# Vanity → campaign-id resolution is still needed by the native Patreon +# ingester (phase 2 resolves the campaign id before the walk). gallery-dl's +# reactive campaign-id retry + the `id:` effective-URL rewrite were removed at +# the #697 cutover (Patreon no longer flows through gallery-dl). _PATREON_VANITY_RE = re.compile( r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)", re.IGNORECASE, ) -_CAMPAIGN_ID_FAILURE_PATTERN = "failed to extract campaign id" def _extract_patreon_vanity(url: str) -> str | None: @@ -56,16 +59,6 @@ def _extract_patreon_vanity(url: str) -> str | None: return m.group(1) if m else None -def _looks_like_campaign_id_failure(stdout: str, stderr: str) -> bool: - return _CAMPAIGN_ID_FAILURE_PATTERN in f"{stdout}\n{stderr}".lower() - - -def _effective_url(platform: str, source_url: str, overrides: dict) -> str: - if platform == "patreon" and overrides.get("patreon_campaign_id"): - return f"https://www.patreon.com/id:{overrides['patreon_campaign_id']}" - return source_url - - class DownloadService: """Async orchestrator. The Celery task runs `asyncio.run(svc.download_source(N))`. @@ -160,11 +153,8 @@ class DownloadService: ctx, source_config, mode, ) else: - effective_url = _effective_url( - ctx["platform"], ctx["url"], ctx["config_overrides"] or {} - ) dl_result = await self.gdl.download( - url=effective_url, + url=ctx["url"], artist_slug=ctx["artist_slug"], platform=ctx["platform"], source_config=source_config, diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 9d3dc92..bab1d63 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -107,12 +107,14 @@ class SourceConfig: `resume_cursor` is RUNTIME state, not persisted operator config: the cursor-paged backfill checkpoint (see parse_last_cursor + the - download_service backfill lifecycle). When set on a patreon backfill - run, gallery-dl resumes its newest→oldest walk from that pagination - cursor instead of restarting from the top. It is threaded by - download_service from config_overrides["_backfill_cursor"]; SourceConfig - deliberately does NOT read it in from_dict (config_overrides also holds - operator config, and resume_cursor must only apply in backfill mode). + download_service backfill lifecycle). It is consumed only by the native + Patreon ingester (the sole cursor-paged platform; gallery-dl's Patreon path + was removed at the #697 cutover): on a backfill/recovery run the ingester + resumes its newest→oldest walk from that pagination cursor instead of + restarting from the top. download_service threads it from + config_overrides["_backfill_cursor"]; SourceConfig deliberately does NOT read + it in from_dict (config_overrides also holds operator config, and + resume_cursor must only apply in backfill/recovery mode). """ content_types: list[str] = field(default_factory=lambda: ["all"]) sleep: float | None = None @@ -170,13 +172,14 @@ def _summarize_validation_failures(failures: list[dict]) -> str: return f"{n} files quarantined ({top_count}× {top_reason}, mixed)" -# gallery-dl logs its pagination cursor (when extractor.*.cursor is truthy) -# as `Cursor: ` — once per fetched page, at debug verbosity. The LAST -# such line is the furthest-progressed page, i.e. the resume point. We scan -# both streams (the debug log lands on stderr, but be defensive) and take the -# final match; passing it back as extractor.patreon.cursor resumes the walk. -# Survives a timed-out run too: the TimeoutExpired path returns the partial -# stdout/stderr, so an interrupted backfill still yields its last cursor. +# The native Patreon ingester emits its pagination cursor as `Cursor: ` +# — one line per fetched page — into the DownloadResult stdout (mirroring the +# convention gallery-dl used before the #697 cutover, so the backfill lifecycle +# stayed unchanged). The LAST such line is the furthest-progressed page, i.e. +# the resume point. We scan both streams (be defensive) and take the final +# match; download_service checkpoints it as the next chunk's resume_cursor. +# Survives a time-boxed chunk too: the ingester emits the cursor for a page when +# it STARTS it, so an interrupted walk still yields its last cursor. _CURSOR_RE = re.compile(r"Cursor:\s*(\S+)") @@ -218,16 +221,10 @@ class GalleryDLService: "permission denied", "tier required", "pledge required", ] - # Per-platform defaults. Lifted from GS — same six platforms FC supports. + # Per-platform defaults for the gallery-dl-backed platforms. Patreon was + # removed at the plan-#697 cutover — it now uses the native ingester + # (services/patreon_ingester.py), not gallery-dl. PLATFORM_DEFAULTS = { - "patreon": { - "content_types": ["images", "image_large", "attachments", "postfile", "content"], - "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], - "filename": "{num:>02}_{filename}.{extension}", - "videos": True, - "embeds": True, - "cursor": True, - }, "subscribestar": { "content_types": ["all"], "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], @@ -312,25 +309,11 @@ class GalleryDLService: # transfer cap — large GIFs that keep streaming are unaffected. "retries": 2, "timeout": 60.0, - # Forward Patreon as Referer/Origin to yt-dlp when it - # fetches video manifests. Operator-flagged 2026-06-01 - # (DaferQ patreon): video posts hosted on Mux carry a JWT - # playback restriction that checks Referer/Origin on every - # request — not just the token signature. gallery-dl's - # HEAD probe to stream.mux.com returns 200 (the token is - # valid), but yt-dlp's actual GET-with-Range to fetch the - # m3u8 manifest 403s because yt-dlp sends its own default - # Referer, which Mux's policy rejects. Forcing the right - # headers fixes the headers-only case; Mux IP-range - # restrictions are unfixable from here. - "ytdl": { - "raw-options": { - "http_headers": { - "Referer": "https://www.patreon.com/", - "Origin": "https://www.patreon.com", - }, - }, - }, + # NOTE: the Patreon/Mux yt-dlp Referer/Origin forwarding lived + # here until the plan-#697 cutover. It was Patreon-specific (and + # would have wrongly tagged the other platforms' yt-dlp fetches); + # Patreon video is now handled by the native ingester's + # downloader (patreon_downloader._VIDEO_HEADERS), so it's gone. }, "output": {"progress": True}, } @@ -382,23 +365,7 @@ class GalleryDLService: platform_section = config["extractor"].setdefault(platform, {}) - if platform == "patreon": - if "all" in source_config.content_types: - platform_section["files"] = [ - "images", "image_large", "attachments", "postfile", "content", - ] - else: - platform_section["files"] = source_config.content_types - # Cursor-paged backfill: resume the newest→oldest walk from the - # checkpoint saved by the previous run instead of restarting from - # the top. Without this, a large catalog (Anduo's weekly - # image-dense Reports) never reaches its frontier inside the - # subprocess budget — the HEAD walk alone times out, 0 files - # written, no progress (event #40411). The PLATFORM_DEFAULTS leave - # `cursor: True` (log-only) for the first/fresh run. - if source_config.resume_cursor: - platform_section["cursor"] = source_config.resume_cursor - elif platform == "hentaifoundry": + if platform == "hentaifoundry": if "pictures" in source_config.content_types or "all" in source_config.content_types: platform_section["include"] = "all" diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index bd920ce..7103ec8 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -50,10 +50,11 @@ _TITLE_MAX = 40 _TIMEOUT_SECONDS = 120.0 _CHUNK = 1 << 16 -# Same Referer/Origin gallery-dl forwards to yt-dlp for Mux-hosted Patreon -# video (gallery_dl.py _get_default_config -> downloader.ytdl.raw-options). -# Mux's JWT playback policy checks Referer/Origin on every request, so yt-dlp -# must send Patreon's, not its own default. +# Referer/Origin yt-dlp must send for Mux-hosted Patreon video. Mux's JWT +# playback policy checks Referer/Origin on every request, so yt-dlp must send +# Patreon's, not its own default. (gallery-dl forwarded the same headers before +# the #697 cutover removed its Patreon path; this is now the only place they +# live.) _VIDEO_HEADERS = { "Referer": "https://www.patreon.com/", "Origin": "https://www.patreon.com", diff --git a/backend/app/services/source_service.py b/backend/app/services/source_service.py index 0192a9c..67b2cf2 100644 --- a/backend/app/services/source_service.py +++ b/backend/app/services/source_service.py @@ -67,6 +67,9 @@ class SourceRecord: # plan #693: derived from config_overrides for the UI badge. backfill_state: str | None # "running" | "complete" | "stalled" | None (idle) backfill_chunks: int + # plan #697: a running deep-walk that bypasses the Patreon seen-ledger + # (recovery) vs. a normal backfill. Lets the badge label it "Recovering". + backfill_bypass_seen: bool def to_dict(self) -> dict: return { @@ -87,6 +90,7 @@ class SourceRecord: "backfill_runs_remaining": self.backfill_runs_remaining, "backfill_state": self.backfill_state, "backfill_chunks": self.backfill_chunks, + "backfill_bypass_seen": self.backfill_bypass_seen, } @@ -162,6 +166,7 @@ class SourceService: backfill_runs_remaining=source.backfill_runs_remaining or 0, backfill_state=co.get("_backfill_state"), backfill_chunks=int(co.get("_backfill_chunks", 0)), + backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")), ) async def _row_to_record(self, source: Source) -> SourceRecord: @@ -319,6 +324,34 @@ class SourceService: await self.session.commit() return await self._row_to_record(source) + async def start_recovery(self, source_id: int) -> SourceRecord: + """Plan #697: arm a RECOVERY walk — a backfill that bypasses the Patreon + seen-ledger so deliberately-dropped-and-deleted near-dups get re-fetched + and re-evaluated under the CURRENT pHash threshold (tier-2 disk still + spares files we kept). Reuses the entire #693 backfill state machine + (time-boxed chunks, cursor checkpoint, complete/stall lifecycle) plus the + `_backfill_bypass_seen` flag that flips download mode to recovery. Clears + any prior cursor/chunk/stall state so it walks fresh from the top. The + flag is cleared on completion (download_service) and on stop. + + Recovery is Patreon-only (the seen-ledger is Patreon's); for other + platforms the flag is inert (download_service ignores it) and the walk + runs as a plain backfill. The UI gates the action to Patreon sources.""" + source = (await self.session.execute( + select(Source).where(Source.id == source_id) + )).scalar_one_or_none() + if source is None: + raise LookupError(f"source id={source_id} not found") + co = dict(source.config_overrides or {}) + co["_backfill_state"] = "running" + co["_backfill_bypass_seen"] = True + for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks"): + co.pop(k, None) + source.config_overrides = co + source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS + await self.session.commit() + return await self._row_to_record(source) + async def stop_backfill(self, source_id: int) -> SourceRecord: """Plan #693: cancel an in-progress backfill — back to idle/tick mode. Clears the running state + cursor/chunk/stall bookkeeping.""" diff --git a/frontend/src/components/subscriptions/SourceCard.vue b/frontend/src/components/subscriptions/SourceCard.vue index 7b8f9bc..4312d8f 100644 --- a/frontend/src/components/subscriptions/SourceCard.vue +++ b/frontend/src/components/subscriptions/SourceCard.vue @@ -27,7 +27,8 @@ Backfilling{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }} + >{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling' + }}{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }} {{ source.backfill_state === 'running' ? 'mdi-stop' : 'mdi-magnify-scan' }} - {{ source.backfill_state === 'running' ? 'Stop backfill' : 'Backfill full history' }} + {{ source.backfill_state === 'running' + ? (source.backfill_bypass_seen ? 'Stop recovery' : 'Stop backfill') + : 'Backfill full history' }} + + + + mdi-backup-restore + + Recover — re-fetch dropped near-dups & re-evaluate under the current threshold @@ -83,7 +96,7 @@ const props = defineProps({ checking: { type: Boolean, default: false }, warningThreshold: { type: Number, default: 5 }, }) -const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill']) +const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover']) function onToggleEnabled(value) { emit('toggle', { source: props.source, enabled: value }) diff --git a/frontend/src/components/subscriptions/SourceRow.vue b/frontend/src/components/subscriptions/SourceRow.vue index 9fe412e..d74caaf 100644 --- a/frontend/src/components/subscriptions/SourceRow.vue +++ b/frontend/src/components/subscriptions/SourceRow.vue @@ -34,7 +34,8 @@ Backfilling{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }} + >{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling' + }}{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }} {{ source.backfill_state === 'running' ? 'mdi-stop' : 'mdi-magnify-scan' }} {{ source.backfill_state === 'running' - ? 'Stop backfill' + ? (source.backfill_bypass_seen ? 'Stop recovery' : 'Stop backfill') : 'Backfill — walk the full post history until complete' }} + + mdi-backup-restore + + Recover — re-fetch dropped near-dups & re-evaluate under the current threshold + + @@ -251,6 +252,7 @@ @toggle="toggleSourceEnabled" @check="onCheck" @backfill="onBackfill" + @recover="onRecover" />
No sources yet. Tap + to add one. @@ -554,6 +556,22 @@ async function onBackfill(source) { } } +// Plan #697: arm a recovery walk (Patreon-only) — re-fetches dropped-and-deleted +// near-dups and re-evaluates them under the current pHash threshold. Reuses the +// backfill lifecycle/badge; stop via the same Stop control (onBackfill). +async function onRecover(source) { + try { + await store.recoverSource(source.id, source.artist_id) + toast({ text: `Recovery started for ${source.artist_name}`, type: 'success' }) + await store.loadAll() + } catch (e) { + toast({ + text: `Recovery start failed: ${e?.detail || e?.message || e}`, + type: 'error', + }) + } +} + async function checkAll(group) { let ok = 0 let conflict = 0 diff --git a/frontend/src/stores/sources.js b/frontend/src/stores/sources.js index 5eb90fe..e7916bc 100644 --- a/frontend/src/stores/sources.js +++ b/frontend/src/stores/sources.js @@ -98,6 +98,14 @@ export const useSourcesStore = defineStore('sources', () => { _invalidate(artistIdHint ?? body.artist_id) return body } + // Plan #697: arm a recovery walk — a backfill that bypasses the Patreon + // seen-ledger to re-fetch dropped-and-deleted near-dups and re-evaluate them + // under the current pHash threshold. Stop with stopBackfill (shared lifecycle). + async function recoverSource(id, artistIdHint = null) { + const body = await api.post(`/api/sources/${id}/backfill`, { body: { action: 'recover' } }) + _invalidate(artistIdHint ?? body.artist_id) + return body + } function sourcesByArtistGrouped() { // returns [{artist: {id,name,slug}, sources: [...]}, ...] @@ -127,6 +135,7 @@ export const useSourcesStore = defineStore('sources', () => { checkNow, startBackfill, stopBackfill, + recoverSource, findOrCreateArtist, autocompleteArtist, loadScheduleStatus, sourcesByArtistGrouped, diff --git a/tests/test_api_sources.py b/tests/test_api_sources.py index 3a13d97..438b305 100644 --- a/tests/test_api_sources.py +++ b/tests/test_api_sources.py @@ -250,3 +250,28 @@ async def test_backfill_endpoint_rejects_bad_action(client, artist, db): async def test_backfill_endpoint_404_when_source_missing(client): resp = await client.post("/api/sources/999999/backfill", json={"action": "start"}) assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_backfill_endpoint_recover_arms_bypass(client, artist, db): + """Plan #697: action='recover' arms a backfill that bypasses the Patreon + seen-ledger; the response exposes backfill_bypass_seen=True so the UI badge + can label it 'Recovering'. Stop clears it.""" + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-recover", enabled=True, + ) + db.add(src) + await db.commit() + sid = src.id + + resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "recover"}) + assert resp.status_code == 200 + body = await resp.get_json() + assert body["backfill_state"] == "running" + assert body["backfill_bypass_seen"] is True + + stopped = await client.post(f"/api/sources/{sid}/backfill", json={"action": "stop"}) + stopped_body = await stopped.get_json() + assert stopped_body["backfill_state"] is None + assert stopped_body["backfill_bypass_seen"] is False diff --git a/tests/test_gallery_dl_service.py b/tests/test_gallery_dl_service.py index 5eadee9..be06108 100644 --- a/tests/test_gallery_dl_service.py +++ b/tests/test_gallery_dl_service.py @@ -260,7 +260,7 @@ def test_build_config_emits_tick_skip_value(gdl): scans once 20 contiguous archived items are seen.""" from backend.app.services.gallery_dl import TICK_SKIP_VALUE cfg = gdl._build_config_for_source( - platform="patreon", + platform="subscribestar", source_config=SourceConfig(), artist_slug="alice", skip_value=TICK_SKIP_VALUE, @@ -273,7 +273,7 @@ def test_build_config_emits_backfill_skip_value(gdl): full post history.""" from backend.app.services.gallery_dl import BACKFILL_SKIP_VALUE cfg = gdl._build_config_for_source( - platform="patreon", + platform="subscribestar", source_config=SourceConfig(), artist_slug="alice", skip_value=BACKFILL_SKIP_VALUE, @@ -318,23 +318,17 @@ def test_categorize_tier_limited_wins_over_partial(gdl): assert etype == ErrorType.TIER_LIMITED -def test_default_config_forwards_patreon_referer_to_ytdl(gdl): - """Operator-flagged 2026-06-01: Mux video playback restrictions reject - yt-dlp's manifest fetch when Referer/Origin don't match Patreon. The - fix is a static `downloader.ytdl.raw-options.http_headers` block, so - it's enough to assert the keys land in the default config.""" - cfg = gdl._get_default_config() - headers = cfg["downloader"]["ytdl"]["raw-options"]["http_headers"] - assert headers["Referer"] == "https://www.patreon.com/" - assert headers["Origin"] == "https://www.patreon.com" - - -# --- Cursor-paged backfill (plan #689) ------------------------------------- +# --- Cursor parsing (plan #689 / native ingester #697) --------------------- +# parse_last_cursor is retained post-cutover: the native Patreon ingester emits +# `Cursor: ` per page into DownloadResult.stdout and download_service +# checkpoints the last one. (The gallery-dl Patreon `files`/`cursor` config and +# the Mux yt-dlp Referer block were removed at the #697 cutover, along with the +# tests that pinned them.) def test_parse_last_cursor_returns_last_match_across_streams(): - # gallery-dl logs `Cursor: ` per page (debug → stderr); the last - # is the furthest-progressed resume point. + # One `Cursor: ` line per page; the last is the furthest-progressed + # resume point. stderr = ( "[patreon][debug] Cursor: 03:AAAA:bbb\n" "[urllib3] GET ...\n" @@ -347,18 +341,3 @@ def test_parse_last_cursor_scans_stdout_too_and_none_when_absent(): assert parse_last_cursor("Cursor: 99:ZZZ", "") == "99:ZZZ" assert parse_last_cursor("no marker here", "still nothing") is None assert parse_last_cursor("", "") is None - - -def test_build_config_patreon_resume_cursor_overrides_default(gdl): - # No resume cursor → PLATFORM_DEFAULTS leaves the log-only `cursor: True`. - fresh = gdl._build_config_for_source( - "patreon", SourceConfig(), "alice", skip_value=True, - ) - assert fresh["extractor"]["patreon"]["cursor"] is True - - # Resume cursor set → gallery-dl restarts the walk from that page. - resumed = gdl._build_config_for_source( - "patreon", SourceConfig(resume_cursor="03:CCCC:ddd"), "alice", - skip_value=True, - ) - assert resumed["extractor"]["patreon"]["cursor"] == "03:CCCC:ddd" diff --git a/tests/test_source_service.py b/tests/test_source_service.py index 7e53850..ce75349 100644 --- a/tests/test_source_service.py +++ b/tests/test_source_service.py @@ -225,6 +225,38 @@ async def test_stop_backfill_returns_to_idle(db): assert "_backfill_state" not in (co or {}) +@pytest.mark.asyncio +async def test_start_recovery_arms_bypass_flag(db): + """Plan #697: start_recovery arms the backfill state machine PLUS the + _backfill_bypass_seen flag (recovery), surfaced on the record so the UI badge + can label it 'Recovering'. Clears any prior checkpoint state.""" + from backend.app.services.source_service import BACKFILL_MAX_CHUNKS + + artist = await _artist(db, "Alice") + svc = SourceService(db) + rec = await svc.create( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice", + ) + updated = await svc.start_recovery(rec.id) + assert updated.backfill_state == "running" + assert updated.backfill_bypass_seen is True + assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS + + co = (await db.execute( + select(Source.config_overrides).where(Source.id == rec.id) + )).scalar_one() + assert co.get("_backfill_bypass_seen") is True + + # Stop clears the bypass flag too (shared lifecycle). + stopped = await svc.stop_backfill(rec.id) + assert stopped.backfill_bypass_seen is False + co2 = (await db.execute( + select(Source.config_overrides).where(Source.id == rec.id) + )).scalar_one() + assert "_backfill_bypass_seen" not in (co2 or {}) + + @pytest.mark.asyncio async def test_start_backfill_raises_when_source_missing(db): svc = SourceService(db) -- 2.52.0 From 218bfebb92d098d3e23e1ba7352b58f5603dccb8 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 22:49:43 -0400 Subject: [PATCH 08/26] feat(downloads): native Patreon verify + uniform backend dispatch (plan #697) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The credential Verify button still ran gallery-dl --simulate for Patreon after the cutover — testing the wrong path (and prone to the vanity "Failed to extract campaign ID" the native resolver fixes). Wire it to the native ingester, behind a DRY dispatch so callers never branch on platform. - services/download_backends.py (new): the ONE place that knows which platforms are native vs gallery-dl. `uses_native_ingester(platform)` is the shared predicate; `verify_source_credential(...)` is the uniform probe (same (ok|None, message) contract for both backends). As a platform migrates, it moves into NATIVE_INGESTER_PLATFORMS here and BOTH download routing and verify switch together. - PatreonClient.verify_auth(campaign_id): one authenticated /api/posts fetch → True (valid) / False (401/403/HTML-login) / None (drift or network — inconclusive, not a credential verdict). - patreon_ingester.verify_patreon_credential(): resolve campaign id, then verify_auth — the verify counterpart to the download path. - patreon_resolver.resolve_campaign_id_for_source(): extracted the override / id:-URL / vanity resolution into ONE helper now shared by the download ingester and verify (download_service no longer carries its own copy + regex; −`import re`). - download_service: routes on uses_native_ingester() instead of inline `== "patreon"` (3 sites); uses the shared resolver. - api/credentials: calls verify_source_credential — no platform branch. Tests: verify_auth mapping, resolve_campaign_id_for_source (override/id:/ vanity/none), the dispatch predicate, verify_patreon_credential glue, credentials endpoint proves Patreon uses the native path (gallery-dl verify asserted not-called); repointed the gallery-dl verify test to subscribestar. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/api/credentials.py | 22 +++---- backend/app/services/download_backends.py | 71 +++++++++++++++++++++++ backend/app/services/download_service.py | 44 ++++---------- backend/app/services/patreon_client.py | 24 ++++++++ backend/app/services/patreon_ingester.py | 25 ++++++++ backend/app/services/patreon_resolver.py | 45 ++++++++++++++ tests/test_api_credentials.py | 45 ++++++++++++-- tests/test_download_backends.py | 23 ++++++++ tests/test_download_service.py | 6 +- tests/test_patreon_client.py | 37 ++++++++++++ tests/test_patreon_ingester.py | 29 +++++++++ tests/test_patreon_resolver.py | 50 +++++++++++++++- 12 files changed, 371 insertions(+), 50 deletions(-) create mode 100644 backend/app/services/download_backends.py create mode 100644 tests/test_download_backends.py diff --git a/backend/app/api/credentials.py b/backend/app/api/credentials.py index f719b51..a7136dc 100644 --- a/backend/app/api/credentials.py +++ b/backend/app/api/credentials.py @@ -121,13 +121,15 @@ async def delete_credential(platform: str): @credentials_bp.route("//verify", methods=["POST"]) async def verify_credential(platform: str): - """Test the stored credential by running gallery-dl --simulate - against one of the platform's enabled sources. On success stamps - last_verified. Returns {valid: bool|null, reason, last_verified?}. - valid=null means "couldn't test" (no credential, or no enabled - source to point at).""" + """Test the stored credential against one of the platform's enabled sources, + WITHOUT downloading. Routes through the platform's backend + (download_backends.verify_credential) — native ingester for Patreon, an + authenticated API page; gallery-dl --simulate for the rest. On success + stamps last_verified. Returns {valid: bool|null, reason, last_verified?}; + valid=null means "couldn't test" (no credential, no enabled source, or an + inconclusive network/drift result).""" from ..models import Artist, Source - from ..services.gallery_dl import GalleryDLService, SourceConfig + from ..services.download_backends import verify_source_credential async with get_session() as session: if not await _ext_key_ok(session): @@ -154,14 +156,14 @@ async def verify_credential(platform: str): cookies_path = await svc.get_cookies_path(platform) auth_token = await svc.get_token(platform) - gdl = GalleryDLService(images_root=Path("/images")) - ok, message = await gdl.verify( + ok, message = await verify_source_credential( + platform=platform, url=source.url, artist_slug=artist.slug, - platform=platform, - source_config=SourceConfig.from_dict(source.config_overrides or {}), + config_overrides=source.config_overrides or {}, cookies_path=str(cookies_path) if cookies_path else None, auth_token=auth_token, + images_root=Path("/images"), ) last_verified = None diff --git a/backend/app/services/download_backends.py b/backend/app/services/download_backends.py new file mode 100644 index 0000000..737da4d --- /dev/null +++ b/backend/app/services/download_backends.py @@ -0,0 +1,71 @@ +"""Platform → download-backend dispatch (one place that knows which platforms +are served by the native FC ingester vs. the gallery-dl subprocess). + +gallery-dl wasn't built to be driven by an automated scheduler — no native +checkpoint/resume, no structured logs, per-file HEADs that dominate wall-clock. +The native ingester (services/patreon_ingester.py, plan #697) replaces it for +Patreon and is the path we grow as more platforms migrate. To keep that +migration DRY, every caller that has to behave differently per backend — +download routing, the credential-verify probe, cursor handling — asks THIS +module instead of testing ``platform == "patreon"`` inline. When a platform gets +a native ingester, it moves into ``NATIVE_INGESTER_PLATFORMS`` here and both the +download path and verify switch over together. + +The backend surfaces share a UNIFORM signature so a caller invokes the same +function regardless of platform: + - verify_credential(...) → (ok: bool|None, message: str) + - (download stays in download_service for now; uses_native_ingester() is the + shared predicate it routes on, so the decision lives here too.) +""" + +from __future__ import annotations + +from pathlib import Path + +# Platforms whose download + verify go through the native ingester rather than +# gallery-dl. gallery-dl still serves every other platform (subscribestar, +# hentaifoundry, discord, pixiv, deviantart) unchanged. +NATIVE_INGESTER_PLATFORMS = frozenset({"patreon"}) + + +def uses_native_ingester(platform: str) -> bool: + """True when `platform` is served by the native ingester (not gallery-dl). + The single predicate the download path and verify both route on.""" + return platform in NATIVE_INGESTER_PLATFORMS + + +async def verify_source_credential( + *, + platform: str, + url: str, + artist_slug: str, + config_overrides: dict | None, + cookies_path: str | None, + auth_token: str | None, + images_root: Path, +) -> tuple[bool | None, str]: + """Uniform credential probe across backends. Returns `(ok, message)`: + True = authenticated, False = rejected, None = inconclusive (drift / + network / nothing to test). Callers don't branch on platform — they call + this and render the result. + """ + if uses_native_ingester(platform): + # Native ingester platforms verify via their own lightweight auth probe + # (resolve campaign id + one authenticated API page). Patreon today. + from .patreon_ingester import verify_patreon_credential + + return await verify_patreon_credential(url, cookies_path, config_overrides) + + # gallery-dl platforms: --simulate one item; the extractor errors before it + # can list if auth is bad. + from .gallery_dl import GalleryDLService, SourceConfig + + gdl = GalleryDLService(images_root=images_root) + return await gdl.verify( + url=url, + artist_slug=artist_slug, + platform=platform, + source_config=SourceConfig.from_dict(config_overrides or {}), + cookies_path=cookies_path, + auth_token=auth_token, + ) diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 7a86917..118ccfb 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -13,7 +13,6 @@ from __future__ import annotations import asyncio import logging -import re from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -25,6 +24,7 @@ from sqlalchemy.orm import joinedload from ..models import Artist, DownloadEvent, Source from .credential_service import CredentialService +from .download_backends import uses_native_ingester from .gallery_dl import ( BACKFILL_CHUNK_SECONDS, BACKFILL_SKIP_VALUE, @@ -37,28 +37,13 @@ from .gallery_dl import ( ) from .importer import Importer from .patreon_ingester import PatreonIngester -from .patreon_resolver import resolve_campaign_id +from .patreon_resolver import resolve_campaign_id_for_source from .platforms import auth_type_for from .scheduler_service import set_platform_cooldown log = logging.getLogger(__name__) -# Vanity → campaign-id resolution is still needed by the native Patreon -# ingester (phase 2 resolves the campaign id before the walk). gallery-dl's -# reactive campaign-id retry + the `id:` effective-URL rewrite were removed at -# the #697 cutover (Patreon no longer flows through gallery-dl). -_PATREON_VANITY_RE = re.compile( - r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)", - re.IGNORECASE, -) - - -def _extract_patreon_vanity(url: str) -> str | None: - m = _PATREON_VANITY_RE.match(url) - return m.group(1) if m else None - - class DownloadService: """Async orchestrator. The Celery task runs `asyncio.run(svc.download_source(N))`. @@ -131,13 +116,13 @@ class DownloadService: skip_value: bool | str = BACKFILL_SKIP_VALUE source_config.timeout = BACKFILL_CHUNK_SECONDS pending_cursor = overrides.get("_backfill_cursor") - if ctx["platform"] == "patreon" and pending_cursor: + if uses_native_ingester(ctx["platform"]) and pending_cursor: source_config.resume_cursor = pending_cursor else: skip_value = TICK_SKIP_VALUE resolved_campaign_id: str | None = None - if ctx["platform"] == "patreon": + if uses_native_ingester(ctx["platform"]): # Native ingester (plan #697) fully replaces gallery-dl for Patreon # in phase 2 — zero per-file HEADs, native cursor/resume, loud drift # detection. Returns a DownloadResult-shaped object so phase 3 is @@ -198,19 +183,12 @@ class DownloadService: silent empty success. """ overrides = ctx["config_overrides"] or {} - campaign_id = overrides.get("patreon_campaign_id") - resolved_campaign_id: str | None = None - if not campaign_id: - # A `.../id:` URL already carries the campaign id — no lookup - # needed (and the vanity regex deliberately excludes the id: form). - id_match = re.search(r"/id:(\d+)", ctx["url"]) - if id_match: - campaign_id = id_match.group(1) - else: - vanity = _extract_patreon_vanity(ctx["url"]) - if vanity: - campaign_id = await resolve_campaign_id(vanity, ctx["cookies_path"]) - resolved_campaign_id = campaign_id + # Shared resolution path (override / id: URL / vanity lookup) — the same + # helper the credential-verify probe uses. resolved_campaign_id is + # non-None only when a vanity lookup ran, so phase 3 caches it. + campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source( + ctx["url"], ctx["cookies_path"], overrides + ) if not campaign_id: return ( @@ -540,7 +518,7 @@ class DownloadService: # top), so they advance only by the download archive growing. new_cursor = ( parse_last_cursor(dl_result.stdout, dl_result.stderr) - if ctx["platform"] == "patreon" else None + if uses_native_ingester(ctx["platform"]) else None ) advanced = bool( (new_cursor and new_cursor != old_cursor) diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index a3685a9..87fb0b5 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -489,6 +489,30 @@ class PatreonClient: return current_cursor = next_cursor + # -- verify ------------------------------------------------------------ + + def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]: + """Cheap auth probe: fetch the first `/api/posts` page and report whether + the credential authenticated, WITHOUT downloading anything. + + Returns `(ok, message)` matching the credential-verify contract: + - True — authenticated (the feed returned a valid JSON:API page). + - False — the credential was rejected (PatreonAuthError: 401/403, or an + HTML login page → cookies expired / tier insufficient). + - None — inconclusive: API drift (our parser is stale, not a cred + problem) or a transient network/HTTP error. + """ + try: + response = self._fetch(campaign_id, None) + self._validate_response(response) + except PatreonAuthError as exc: + return False, f"Patreon rejected the credential — {exc}" + except PatreonDriftError as exc: + return None, f"Couldn't verify — Patreon's API shape changed: {exc}" + except PatreonAPIError as exc: + return None, f"Couldn't verify (network/HTTP issue): {exc}" + return True, "Credentials valid — the Patreon feed authenticated." + def _dedup_by_filehash(items: list[MediaItem]) -> list[MediaItem]: """Drop later items sharing a filehash with an earlier one (first wins). diff --git a/backend/app/services/patreon_ingester.py b/backend/app/services/patreon_ingester.py index 5409e81..69a0b17 100644 --- a/backend/app/services/patreon_ingester.py +++ b/backend/app/services/patreon_ingester.py @@ -38,6 +38,7 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. from __future__ import annotations +import asyncio import logging import time from collections.abc import Callable @@ -56,6 +57,7 @@ from .patreon_client import ( PatreonDriftError, ) from .patreon_downloader import PatreonDownloader +from .patreon_resolver import resolve_campaign_id_for_source log = logging.getLogger(__name__) @@ -377,3 +379,26 @@ class PatreonIngester: ) session.execute(stmt) session.commit() + + +async def verify_patreon_credential( + url: str, + cookies_path: str | None, + overrides: dict | None, +) -> tuple[bool | None, str]: + """Native Patreon credential probe — the verify counterpart to the ingester's + download path, sharing its campaign-id resolution. Resolves the campaign id + (override / id: URL / vanity) then does ONE authenticated `/api/posts` fetch + via PatreonClient.verify_auth. Returns the uniform `(ok, message)` contract + (True / False / None) so download_backends.verify_credential can treat it + interchangeably with the gallery-dl probe. No download, no DB. + """ + campaign_id, _ = await resolve_campaign_id_for_source(url, cookies_path, overrides) + if not campaign_id: + return None, ( + "Couldn't resolve the Patreon campaign id from the source URL — " + "can't verify (cookies expired, or the creator moved/renamed?)." + ) + client = PatreonClient(cookies_path) + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, client.verify_auth, campaign_id) diff --git a/backend/app/services/patreon_resolver.py b/backend/app/services/patreon_resolver.py index 64797c9..fb29661 100644 --- a/backend/app/services/patreon_resolver.py +++ b/backend/app/services/patreon_resolver.py @@ -19,12 +19,22 @@ import asyncio import http.cookiejar import logging import os +import re import requests log = logging.getLogger(__name__) _CAMPAIGNS_URL = "https://www.patreon.com/api/campaigns" + +# A source URL of the form `.../id:` already carries the campaign id +# (no lookup needed). The vanity regex deliberately EXCLUDES the id: form so the +# two paths don't overlap. +_ID_URL_RE = re.compile(r"/id:(\d+)") +_VANITY_RE = re.compile( + r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)", + re.IGNORECASE, +) _USER_AGENT = ( "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" @@ -100,3 +110,38 @@ async def resolve_campaign_id( Never raises.""" loop = asyncio.get_running_loop() return await loop.run_in_executor(None, _sync_lookup, vanity, cookies_path) + + +def extract_vanity(url: str) -> str | None: + """The vanity slug from a Patreon creator URL, or None for an `id:` URL.""" + m = _VANITY_RE.match(url or "") + return m.group(1) if m else None + + +async def resolve_campaign_id_for_source( + url: str, + cookies_path: str | None, + overrides: dict | None, +) -> tuple[str | None, str | None]: + """Resolve a Patreon source to its campaign id — the single resolution path + shared by the download ingester and the credential-verify probe. + + Order: cached `patreon_campaign_id` override → an `id:` URL → a + vanity lookup against the campaigns API. Returns + `(campaign_id, newly_resolved_id)`: `newly_resolved_id` is non-None ONLY when + a vanity lookup actually ran, so the caller knows to cache it on the source + (the override/id: paths needed no lookup). `(None, None)` when unresolvable. + Never raises. + """ + overrides = overrides or {} + cached = overrides.get("patreon_campaign_id") + if cached: + return cached, None + id_match = _ID_URL_RE.search(url or "") + if id_match: + return id_match.group(1), None + vanity = extract_vanity(url) + if vanity: + resolved = await resolve_campaign_id(vanity, cookies_path) + return resolved, resolved + return None, None diff --git a/tests/test_api_credentials.py b/tests/test_api_credentials.py index fa042b4..43a4853 100644 --- a/tests/test_api_credentials.py +++ b/tests/test_api_credentials.py @@ -155,6 +155,8 @@ async def test_verify_no_enabled_source_is_untestable(client): @pytest.mark.asyncio async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypatch): + """A gallery-dl platform routes through GalleryDLService.verify; success + stamps last_verified (platform-agnostic endpoint behavior).""" from backend.app.models import Artist, Source from backend.app.services import gallery_dl as gdl_mod @@ -163,6 +165,45 @@ async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypa return (True, "Credentials valid — the feed authenticated.") monkeypatch.setattr(gdl_mod.GalleryDLService, "verify", _fake_verify) + await client.post("/api/credentials", json={ + "platform": "subscribestar", "credential_type": "cookies", "data": _NETSCAPE, + }) + artist = Artist(name="Maewix", slug="maewix") + db.add(artist) + await db.flush() + db.add(Source( + artist_id=artist.id, platform="subscribestar", + url="https://www.subscribestar.com/maewix", enabled=True, config_overrides={}, + )) + await db.commit() + + resp = await client.post("/api/credentials/subscribestar/verify") + body = await resp.get_json() + assert body["valid"] is True + assert body["last_verified"] is not None + + # The stamp is persisted on the credential record. + rec = await (await client.get("/api/credentials/subscribestar")).get_json() + assert rec["last_verified"] is not None + + +@pytest.mark.asyncio +async def test_verify_patreon_uses_native_ingester_not_gallery_dl(client, db, monkeypatch): + """plan #697 cutover: Patreon credential verify routes through the native + ingester (verify_patreon_credential), NOT gallery-dl --simulate.""" + from backend.app.models import Artist, Source + from backend.app.services import gallery_dl as gdl_mod + from backend.app.services import patreon_ingester as pi_mod + + async def _native_verify(url, cookies_path, overrides): + return (True, "Credentials valid — the Patreon feed authenticated.") + monkeypatch.setattr(pi_mod, "verify_patreon_credential", _native_verify) + + # gallery-dl must NOT be consulted for Patreon. + async def _boom(self, *args, **kwargs): + raise AssertionError("gallery-dl verify must not run for patreon") + monkeypatch.setattr(gdl_mod.GalleryDLService, "verify", _boom) + await client.post("/api/credentials", json={ "platform": "patreon", "credential_type": "cookies", "data": _NETSCAPE, }) @@ -180,10 +221,6 @@ async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypa assert body["valid"] is True assert body["last_verified"] is not None - # The stamp is persisted on the credential record. - rec = await (await client.get("/api/credentials/patreon")).get_json() - assert rec["last_verified"] is not None - @pytest.mark.asyncio async def test_verify_reports_auth_failure(client, db, monkeypatch): diff --git a/tests/test_download_backends.py b/tests/test_download_backends.py new file mode 100644 index 0000000..3c67fd9 --- /dev/null +++ b/tests/test_download_backends.py @@ -0,0 +1,23 @@ +"""download_backends — the single predicate that routes a platform to the +native ingester vs. gallery-dl. Pure, no DB.""" + +from backend.app.services.download_backends import ( + NATIVE_INGESTER_PLATFORMS, + uses_native_ingester, +) + + +def test_patreon_is_native(): + assert uses_native_ingester("patreon") is True + assert "patreon" in NATIVE_INGESTER_PLATFORMS + + +def test_gallery_dl_platforms_are_not_native(): + # The five platforms still served by gallery-dl must NOT route to the + # native ingester — guards an accidental over-broad migration. + for platform in ("subscribestar", "hentaifoundry", "discord", "pixiv", "deviantart"): + assert uses_native_ingester(platform) is False + + +def test_unknown_platform_is_not_native(): + assert uses_native_ingester("nonsense") is False diff --git a/tests/test_download_service.py b/tests/test_download_service.py index 5e44df9..0befed6 100644 --- a/tests/test_download_service.py +++ b/tests/test_download_service.py @@ -288,7 +288,8 @@ async def test_run_patreon_ingester_resolves_vanity_and_runs( _artist, source = seed_artist_and_source monkeypatch.setattr( - dl_mod, "resolve_campaign_id", AsyncMock(return_value="4242"), + dl_mod, "resolve_campaign_id_for_source", + AsyncMock(return_value=("4242", "4242")), ) run_kwargs = {} @@ -335,7 +336,8 @@ async def test_run_patreon_ingester_unresolvable_fails_loud( _artist, source = seed_artist_and_source monkeypatch.setattr( - dl_mod, "resolve_campaign_id", AsyncMock(return_value=None), + dl_mod, "resolve_campaign_id_for_source", + AsyncMock(return_value=(None, None)), ) svc = DownloadService( async_session=db, sync_session=db_sync, diff --git a/tests/test_patreon_client.py b/tests/test_patreon_client.py index 268c3bb..f1574ad 100644 --- a/tests/test_patreon_client.py +++ b/tests/test_patreon_client.py @@ -238,3 +238,40 @@ def test_fetch_other_status_raises_api_error_with_code(monkeypatch, status): def test_fetch_ok_returns_payload(monkeypatch): client = _client_returning(monkeypatch, _FakeResp(200, json_data={"data": []})) assert client._fetch("5555", None) == {"data": []} + + +# -- verify_auth (credential probe; (True/False/None, message) contract) --- + + +def test_verify_auth_ok_when_page_authenticates(monkeypatch): + client = _client_returning(monkeypatch, _FakeResp(200, json_data={"data": []})) + ok, msg = client.verify_auth("5555") + assert ok is True + assert "valid" in msg.lower() + + +@pytest.mark.parametrize("status", [401, 403]) +def test_verify_auth_false_when_rejected(monkeypatch, status): + client = _client_returning(monkeypatch, _FakeResp(status)) + ok, _msg = client.verify_auth("5555") + assert ok is False + + +def test_verify_auth_inconclusive_on_drift(monkeypatch): + # 200 + JSON but missing top-level 'data' → drift → inconclusive, NOT a + # credential verdict (our parser is stale, the cookie may be fine). + client = _client_returning(monkeypatch, _FakeResp(200, json_data={"included": []})) + ok, msg = client.verify_auth("5555") + assert ok is None + assert "api shape changed" in msg.lower() + + +def test_verify_auth_inconclusive_on_network(monkeypatch): + import requests + client = PatreonClient(cookies_path=None) + + def _raise(*a, **k): + raise requests.ConnectionError("boom") + monkeypatch.setattr(client._session, "get", _raise) + ok, _msg = client.verify_auth("5555") + assert ok is None diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index ef356d7..15b8824 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -370,3 +370,32 @@ def test_ledger_key_video_has_no_filehash(): kind="postfile", filehash=None, post_id="p9", ) assert _ledger_key(video) == "p9:clip.mp4" + + +# --- verify_patreon_credential (the verify counterpart, plan #697) --------- + + +@pytest.mark.asyncio +async def test_verify_patreon_credential_unresolvable_is_inconclusive(): + from backend.app.services.patreon_ingester import verify_patreon_credential + + # No campaign id resolvable from a non-Patreon URL → can't test → None. + ok, msg = await verify_patreon_credential("not-a-patreon-url", None, {}) + assert ok is None + assert "campaign id" in msg.lower() + + +@pytest.mark.asyncio +async def test_verify_patreon_credential_delegates_to_client(monkeypatch): + import backend.app.services.patreon_ingester as mod + + async def _fake_resolve(url, cookies_path, overrides): + return "123", None + monkeypatch.setattr(mod, "resolve_campaign_id_for_source", _fake_resolve) + monkeypatch.setattr( + mod.PatreonClient, "verify_auth", + lambda self, campaign_id: (True, f"ok:{campaign_id}"), + ) + ok, msg = await mod.verify_patreon_credential("https://patreon.com/x", None, {}) + assert ok is True + assert msg == "ok:123" diff --git a/tests/test_patreon_resolver.py b/tests/test_patreon_resolver.py index ecfe165..bbf40e4 100644 --- a/tests/test_patreon_resolver.py +++ b/tests/test_patreon_resolver.py @@ -2,7 +2,10 @@ from unittest.mock import MagicMock, patch import pytest -from backend.app.services.patreon_resolver import resolve_campaign_id +from backend.app.services.patreon_resolver import ( + resolve_campaign_id, + resolve_campaign_id_for_source, +) @pytest.mark.asyncio @@ -86,3 +89,48 @@ async def test_loads_cookies_file_when_present(tmp_path): assert result == "9" _, kwargs = mock_get.call_args assert kwargs.get("cookies") is not None + + +# -- resolve_campaign_id_for_source (shared by download + verify) ---------- + + +@pytest.mark.asyncio +async def test_for_source_uses_cached_override_without_lookup(): + with patch("backend.app.services.patreon_resolver.requests.get") as mock_get: + cid, resolved = await resolve_campaign_id_for_source( + "https://patreon.com/alice", None, {"patreon_campaign_id": "999"}, + ) + assert cid == "999" + assert resolved is None # cached → no newly-resolved id to cache + mock_get.assert_not_called() + + +@pytest.mark.asyncio +async def test_for_source_extracts_id_url_without_lookup(): + with patch("backend.app.services.patreon_resolver.requests.get") as mock_get: + cid, resolved = await resolve_campaign_id_for_source( + "https://www.patreon.com/id:4242", None, {}, + ) + assert cid == "4242" + assert resolved is None + mock_get.assert_not_called() + + +@pytest.mark.asyncio +async def test_for_source_resolves_vanity_and_reports_it(): + fake = MagicMock() + fake.status_code = 200 + fake.json.return_value = {"data": [{"id": "777", "type": "campaign"}]} + with patch("backend.app.services.patreon_resolver.requests.get", return_value=fake): + cid, resolved = await resolve_campaign_id_for_source( + "https://patreon.com/alice", None, {}, + ) + assert cid == "777" + assert resolved == "777" # newly resolved → caller caches it + + +@pytest.mark.asyncio +async def test_for_source_unresolvable_returns_none(): + cid, resolved = await resolve_campaign_id_for_source("not-a-patreon-url", None, {}) + assert cid is None + assert resolved is None -- 2.52.0 From 3b2f7a41c399100618ec57685ca20b519ca54cfd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 23:13:46 -0400 Subject: [PATCH 09/26] =?UTF-8?q?feat(patreon):=20ingester=20rate-limit=20?= =?UTF-8?q?resilience=20=E2=80=94=20#703=20step=201?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single 429 mid-walk used to fail the run → RATE_LIMITED → platform-wide cooldown → every Patreon source dark ("testing dead"). The native path also ignored the operator's existing politeness setting. Fixed both: - Pacing (avoid 429s): honor download_rate_limit_seconds (gallery-dl's `rate_limit`, read off self.gdl) as a pre-download sleep on real media downloads only (skips don't pace); pace /api/posts page fetches with the per-source sleep_request override, defaulting to max(0.5, rate_limit/4) — the same API-pacing default gallery-dl used for `sleep-request`. - 429 backoff (ride out transient limits): PatreonClient._fetch retries a 429 with backoff (honor Retry-After, else exponential 2·2^(n-1), capped 30s, ≤3 tries); only a PERSISTENT 429 propagates as terminal RATE_LIMITED. Light 2-retry on a media-GET 429 too. Threaded via PatreonIngester(rate_limit=, request_sleep=) → PatreonClient/PatreonDownloader; download_service sources them. Injected test client/downloader are unaffected (carry their own pacing). Tests mock time.sleep (no real sleeping): retry-then-success, persistent 429 raises after N, Retry-After honored, request_sleep paces, media pacing per real download, skips don't pace, media 429 retried. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/download_service.py | 15 ++++ backend/app/services/patreon_client.py | 79 +++++++++++++++++++--- backend/app/services/patreon_downloader.py | 37 +++++++++- backend/app/services/patreon_ingester.py | 14 +++- tests/test_patreon_client.py | 64 +++++++++++++++++- tests/test_patreon_downloader.py | 63 ++++++++++++++++- 6 files changed, 254 insertions(+), 18 deletions(-) diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 118ccfb..b33e5b4 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -207,11 +207,26 @@ class DownloadService: None, ) + # Honor the operator's existing rate-limit knobs on the native path + # (plan #703): the global download_rate_limit_seconds (gallery-dl's + # `rate_limit`, here on self.gdl) paces media downloads. Page fetches use + # the per-source sleep_request override, else `max(0.5, rate_limit/4)` — + # the SAME API-pacing default gallery-dl applied as its `sleep-request` + # (gallery_dl._get_default_config), so the rate-limited /api/posts + # endpoint stays politely paced without over-throttling. + rate_limit = self.gdl._rate_limit + request_sleep = ( + source_config.sleep_request + if source_config.sleep_request is not None + else max(0.5, rate_limit / 4) + ) ingester = PatreonIngester( images_root=self.gdl.images_root, cookies_path=ctx["cookies_path"], session_factory=self.sync_session_factory, validate=self.gdl._validate_files, + rate_limit=rate_limit, + request_sleep=request_sleep, ) loop = asyncio.get_running_loop() dl_result = await loop.run_in_executor( diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index 87fb0b5..5d1ef2b 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -30,6 +30,7 @@ import http.cookiejar import logging import os import re +import time from collections.abc import Iterator from dataclasses import dataclass from html import unescape @@ -47,6 +48,34 @@ _USER_AGENT = ( ) _TIMEOUT_SECONDS = 30.0 +# 429 backoff (plan #703): ride out a transient API rate-limit instead of +# failing the whole walk (which would stamp RATE_LIMITED → platform-wide +# cooldown → every Patreon source dark). Honor the server's `Retry-After`; +# otherwise exponential base·2^(n-1), capped. Only after the retries are +# exhausted does the 429 propagate as terminal RATE_LIMITED. +_MAX_429_RETRIES = 3 +_BACKOFF_BASE_SECONDS = 2.0 +_BACKOFF_CAP_SECONDS = 30.0 + + +def _retry_after_seconds( + resp: requests.Response, + attempt: int, + *, + base: float = _BACKOFF_BASE_SECONDS, + cap: float = _BACKOFF_CAP_SECONDS, +) -> float: + """Backoff delay for a 429: the `Retry-After` seconds header if present and + numeric, else exponential `base·2^(attempt-1)`, both capped. (HTTP-date form + of Retry-After is rare here and falls through to exponential.)""" + header = resp.headers.get("Retry-After") + if header: + try: + return min(float(header), cap) + except (TypeError, ValueError): + pass + return min(base * (2 ** max(0, attempt - 1)), cap) + # JSON:API request contract (observed from real traffic — see module plan). _INCLUDE = ( "campaign,access_rules,attachments,attachments_media,audio,images,media," @@ -205,9 +234,19 @@ class PatreonClient: requests.Session; no secure-context APIs are used. """ - def __init__(self, cookies_path: str | Path | None): + def __init__( + self, + cookies_path: str | Path | None, + *, + request_sleep: float = 0.0, + max_retries: int = _MAX_429_RETRIES, + ): self.cookies_path = str(cookies_path) if cookies_path else None self._session = _load_session(cookies_path) + # Politeness: seconds to sleep before each /api/posts page fetch (paces + # the rate-limited API endpoint). 0 = no pacing. plan #703. + self._request_sleep = request_sleep or 0.0 + self._max_retries = max_retries # -- request ----------------------------------------------------------- @@ -228,16 +267,34 @@ class PatreonClient: return params def _fetch(self, campaign_id: str, cursor: str | None) -> dict: - try: - resp = self._session.get( - _POSTS_URL, - params=self._params(campaign_id, cursor), - timeout=_TIMEOUT_SECONDS, - ) - except requests.RequestException as exc: - raise PatreonAPIError( - f"Patreon posts request failed (campaign_id={campaign_id}): {exc}" - ) from exc + if self._request_sleep > 0: + time.sleep(self._request_sleep) # pace the API endpoint + attempt = 0 + while True: + try: + resp = self._session.get( + _POSTS_URL, + params=self._params(campaign_id, cursor), + timeout=_TIMEOUT_SECONDS, + ) + except requests.RequestException as exc: + raise PatreonAPIError( + f"Patreon posts request failed (campaign_id={campaign_id}): {exc}" + ) from exc + + # Transient rate-limit: back off and retry rather than failing the + # whole walk. Only a PERSISTENT 429 (retries exhausted) falls + # through to the terminal RATE_LIMITED raise below. + if resp.status_code == 429 and attempt < self._max_retries: + attempt += 1 + delay = _retry_after_seconds(resp, attempt) + log.warning( + "Patreon 429 (campaign_id=%s) — backing off %.1fs (retry %d/%d)", + campaign_id, delay, attempt, self._max_retries, + ) + time.sleep(delay) + continue + break if resp.status_code in (401, 403): # Auth rejected — expired/missing cookies or an insufficient tier. diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index 7103ec8..ed46d25 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -33,6 +33,7 @@ import logging import os import shutil import subprocess +import time from collections.abc import Callable from dataclasses import dataclass from datetime import datetime @@ -42,13 +43,16 @@ from urllib.parse import urlsplit import requests from .file_validator import is_validatable, validate_file -from .patreon_client import _load_session +from .patreon_client import _load_session, _retry_after_seconds log = logging.getLogger(__name__) _TITLE_MAX = 40 _TIMEOUT_SECONDS = 120.0 _CHUNK = 1 << 16 +# A CDN media GET rarely 429s, but if it does, a couple of backoff retries keep +# one throttled file from becoming a per-item error (plan #703). +_MAX_MEDIA_429_RETRIES = 2 # Referer/Origin yt-dlp must send for Mux-hosted Patreon video. Mux's JWT # playback policy checks Referer/Origin on every request, so yt-dlp must send @@ -148,11 +152,17 @@ class PatreonDownloader: cookies_path: str | None = None, *, validate: bool = True, + rate_limit: float = 0.0, session: requests.Session | None = None, ): self.images_root = Path(images_root) self.cookies_path = str(cookies_path) if cookies_path else None self._validate = validate + # Politeness: seconds to sleep before each actual media download (paces + # the CDN; honors ImportSettings.download_rate_limit_seconds, the same + # value gallery-dl used as its between-downloads `sleep`). 0 = no pacing. + # Applied only to real downloads, not to seen/disk skips. plan #703. + self._rate_limit = rate_limit or 0.0 # Build a cookie-loaded session the same way patreon_client does, so the # CDN GETs carry the creator's auth. self.session = session if session is not None else _load_session(cookies_path) @@ -227,6 +237,10 @@ class PatreonDownloader: post_dir.mkdir(parents=True, exist_ok=True) + # Pace real downloads only (the skips above already returned). plan #703. + if self._rate_limit > 0: + time.sleep(self._rate_limit) + if _is_video_url(media.url): out_path = self._run_ytdlp(media.url, media_path, _VIDEO_HEADERS) if out_path is None or not Path(out_path).exists(): @@ -267,8 +281,25 @@ class PatreonDownloader: return dest def _fetch_to_file(self, url: str, dest: Path) -> None: - """Stream a non-video URL to `dest` via the (stubbable) session.""" - resp = self.session.get(url, stream=True, timeout=_TIMEOUT_SECONDS) + """Stream a non-video URL to `dest` via the (stubbable) session. + + Retries a transient 429 with backoff (plan #703); any other non-2xx + falls through to raise_for_status → HTTPError, which download_post turns + into a resilient per-item error outcome. + """ + attempt = 0 + while True: + resp = self.session.get(url, stream=True, timeout=_TIMEOUT_SECONDS) + if resp.status_code == 429 and attempt < _MAX_MEDIA_429_RETRIES: + attempt += 1 + delay = _retry_after_seconds(resp, attempt) + log.warning( + "Patreon media 429 (%s) — backing off %.1fs (retry %d/%d)", + url, delay, attempt, _MAX_MEDIA_429_RETRIES, + ) + time.sleep(delay) + continue + break resp.raise_for_status() with open(dest, "wb") as fh: for chunk in resp.iter_content(chunk_size=_CHUNK): diff --git a/backend/app/services/patreon_ingester.py b/backend/app/services/patreon_ingester.py index 69a0b17..0a40fb6 100644 --- a/backend/app/services/patreon_ingester.py +++ b/backend/app/services/patreon_ingester.py @@ -104,18 +104,28 @@ class PatreonIngester: session_factory: Callable[[], object], *, validate: bool = True, + rate_limit: float = 0.0, + request_sleep: float = 0.0, client: PatreonClient | None = None, downloader: PatreonDownloader | None = None, ): self.images_root = Path(images_root) self.cookies_path = str(cookies_path) if cookies_path else None self.session_factory = session_factory - self.client = client if client is not None else PatreonClient(cookies_path) + # Pacing (plan #703): request_sleep paces the API page fetches, + # rate_limit paces the media downloads. Injected client/downloader (in + # tests) already carry their own pacing, so these only apply to the + # default-constructed ones. + self.client = ( + client + if client is not None + else PatreonClient(cookies_path, request_sleep=request_sleep) + ) self.downloader = ( downloader if downloader is not None else PatreonDownloader( - self.images_root, cookies_path, validate=validate + self.images_root, cookies_path, validate=validate, rate_limit=rate_limit ) ) diff --git a/tests/test_patreon_client.py b/tests/test_patreon_client.py index f1574ad..f2939e2 100644 --- a/tests/test_patreon_client.py +++ b/tests/test_patreon_client.py @@ -10,12 +10,14 @@ from pathlib import Path import pytest +from backend.app.services import patreon_client as pc_mod from backend.app.services.patreon_client import ( MediaItem, PatreonAPIError, PatreonAuthError, PatreonClient, PatreonDriftError, + _retry_after_seconds, parse_cursor_from_url, ) @@ -192,10 +194,11 @@ def test_media_referenced_but_absent_raises_drift(client): class _FakeResp: - def __init__(self, status_code, *, json_data=None, raise_json=False): + def __init__(self, status_code, *, json_data=None, raise_json=False, headers=None): self.status_code = status_code self._json_data = json_data self._raise_json = raise_json + self.headers = headers or {} def json(self): if self._raise_json: @@ -275,3 +278,62 @@ def test_verify_auth_inconclusive_on_network(monkeypatch): monkeypatch.setattr(client._session, "get", _raise) ok, _msg = client.verify_auth("5555") assert ok is None + + +# -- 429 backoff + pacing (plan #703) -------------------------------------- + + +def test_retry_after_seconds_honors_numeric_header(): + assert _retry_after_seconds(_FakeResp(429, headers={"Retry-After": "7"}), 1) == 7.0 + + +def test_retry_after_seconds_exponential_without_header(): + resp = _FakeResp(429) + assert _retry_after_seconds(resp, 1) == 2.0 + assert _retry_after_seconds(resp, 2) == 4.0 + assert _retry_after_seconds(resp, 3) == 8.0 + + +def test_retry_after_seconds_caps(): + assert _retry_after_seconds(_FakeResp(429), 10) == 30.0 + assert _retry_after_seconds(_FakeResp(429, headers={"Retry-After": "999"}), 1) == 30.0 + + +def _client_with_sequence(monkeypatch, responses): + """A client whose session yields `responses` in order; time.sleep stubbed so + backoff never really sleeps. Returns (client, slept_list).""" + client = PatreonClient(cookies_path=None) + it = iter(responses) + monkeypatch.setattr(client._session, "get", lambda *a, **k: next(it)) + slept: list[float] = [] + monkeypatch.setattr(pc_mod.time, "sleep", lambda s: slept.append(s)) + return client, slept + + +def test_fetch_retries_then_succeeds_on_429(monkeypatch): + client, slept = _client_with_sequence(monkeypatch, [ + _FakeResp(429, headers={"Retry-After": "5"}), + _FakeResp(200, json_data={"data": []}), + ]) + assert client._fetch("5555", None) == {"data": []} + assert 5.0 in slept # backed off once before the retry + + +def test_fetch_persistent_429_raises_after_retries(monkeypatch): + client, _slept = _client_with_sequence(monkeypatch, [_FakeResp(429)] * 10) + with pytest.raises(PatreonAPIError) as ei: + client._fetch("5555", None) + assert ei.value.status_code == 429 + assert not isinstance(ei.value, PatreonAuthError) + + +def test_request_sleep_paces_before_fetch(monkeypatch): + client = PatreonClient(cookies_path=None, request_sleep=1.5) + monkeypatch.setattr( + client._session, "get", + lambda *a, **k: _FakeResp(200, json_data={"data": []}), + ) + slept: list[float] = [] + monkeypatch.setattr(pc_mod.time, "sleep", lambda s: slept.append(s)) + client._fetch("5555", None) + assert slept == [1.5] diff --git a/tests/test_patreon_downloader.py b/tests/test_patreon_downloader.py index 6b79975..f96a6fb 100644 --- a/tests/test_patreon_downloader.py +++ b/tests/test_patreon_downloader.py @@ -10,7 +10,9 @@ from __future__ import annotations from pathlib import Path import pytest +import requests +from backend.app.services import patreon_downloader as pd_mod from backend.app.services.patreon_client import MediaItem from backend.app.services.patreon_downloader import ( MediaOutcome, @@ -26,10 +28,14 @@ _PNG_BYTES = _PNG_HEAD + (b"\x00" * 16) + _PNG_TAIL class _FakeResponse: - def __init__(self, payload: bytes): + def __init__(self, payload: bytes, status_code: int = 200, headers=None): self._payload = payload + self.status_code = status_code + self.headers = headers or {} def raise_for_status(self): + if self.status_code >= 400: + raise requests.HTTPError(f"HTTP {self.status_code}") return None def iter_content(self, chunk_size=65536): @@ -49,6 +55,18 @@ class _FakeSession: return _FakeResponse(self.payloads.get(url, _PNG_BYTES)) +class _SeqSession: + """Returns the queued responses in order (for retry tests).""" + + def __init__(self, responses: list[_FakeResponse]): + self._responses = list(responses) + self.calls: list[str] = [] + + def get(self, url, stream=False, timeout=None): + self.calls.append(url) + return self._responses.pop(0) + + def _post(post_id="1001", title="My Post Title", published="2026-05-01T12:00:00Z"): return { "id": post_id, @@ -250,3 +268,46 @@ def test_media_outcome_shape(): o = MediaOutcome(media=None, status="downloaded", path=Path("/tmp/x"), error=None) assert o.status == "downloaded" assert o.path == Path("/tmp/x") + + +# -- pacing + 429 backoff (plan #703) -------------------------------------- + + +def test_rate_limit_paces_each_real_download(tmp_path, monkeypatch): + slept: list[float] = [] + monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s)) + dl = PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, + rate_limit=2.0, session=_FakeSession(), + ) + dl.download_post(_post(), [_img("a.png"), _img("b.png")], "artist-x") + # One pacing sleep per actual download (two media downloaded). + assert slept == [2.0, 2.0] + + +def test_skips_do_not_pace(tmp_path, monkeypatch): + slept: list[float] = [] + monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s)) + dl = PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, + rate_limit=2.0, session=_FakeSession(), + ) + # is_seen → skipped_seen, never reaches the download/pacing path. + dl.download_post(_post(), [_img("a.png")], "artist-x", is_seen=lambda m: True) + assert slept == [] + + +def test_media_429_retried_then_succeeds(tmp_path, monkeypatch): + slept: list[float] = [] + monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s)) + session = _SeqSession([ + _FakeResponse(b"", status_code=429, headers={"Retry-After": "4"}), + _FakeResponse(_PNG_BYTES, status_code=200), + ]) + dl = PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, session=session, + ) + outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") + assert outcomes[0].status == "downloaded" + assert 4.0 in slept # backed off before the retry + assert len(session.calls) == 2 -- 2.52.0 From d6c15f4ea0478bc009db41a07a9ab94f29b5a006 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 23:21:05 -0400 Subject: [PATCH 10/26] =?UTF-8?q?test(patreon):=20fake=20gdl=20needs=20rea?= =?UTF-8?q?l=20=5Frate=5Flimit=20for=20native=20pacing=20=E2=80=94=20#703?= =?UTF-8?q?=20step=201=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _run_patreon_ingester reads self.gdl._rate_limit for the native pacing config (max(0.5, rate_limit/4)); the MagicMock fake gdl broke the arithmetic. Give it real _rate_limit/_validate_files. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_download_service.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_download_service.py b/tests/test_download_service.py index 0befed6..0cd6cbe 100644 --- a/tests/test_download_service.py +++ b/tests/test_download_service.py @@ -84,6 +84,11 @@ def _make_fake_dl_result( def _fake_gdl_with_result(result): fake = MagicMock() fake.download = AsyncMock(return_value=result) + # Real values for the attrs _run_patreon_ingester reads off the gdl service + # (the native pacing config, plan #703) — a MagicMock would break the + # arithmetic (max(0.5, rate_limit/4)). + fake._rate_limit = 3.0 + fake._validate_files = True fake._compute_run_stats = lambda *a, **k: { "exit_code": 0, "downloaded_count": len(result.written_paths), "skipped_count": 0, "per_item_failures": 0, -- 2.52.0 From 5b615b7ded694aab1343be9ee85e6d98126a8e11 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 23:21:05 -0400 Subject: [PATCH 11/26] =?UTF-8?q?feat(sources):=20pre-flight=20credential?= =?UTF-8?q?=20verify=20on=20backfill/recovery=20arm=20=E2=80=94=20#703=20s?= =?UTF-8?q?tep=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before arming a deep walk on a native-ingester platform (Patreon — where verify is one cheap API page), POST /sources/{id}/backfill {start|recover} runs the shared verify_source_credential first and REFUSES (409 + reason) only on a definitive rejection (verify→False, e.g. expired cookies). It proceeds on valid (True) or inconclusive (None — a network blip must not block). Gated to native platforms: gallery-dl verify is a slow --simulate subprocess, too heavy for an arm action. The credential read happens in a session that's CLOSED before the verify network call (no held conn). Frontend: onBackfill/onRecover now read e.body.detail (ApiError carries the reason in .body, not .detail) so the rejection text surfaces in the toast. Tests: arm blocked on rejection (409, source not armed), proceeds on inconclusive, stop never pre-flights, gallery-dl platform skips pre-flight. An autouse fixture stubs verify to 'valid' for the existing backfill endpoint tests so they stay network-free. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/api/sources.py | 40 +++++++ .../subscriptions/SubscriptionsTab.vue | 4 +- tests/test_api_sources.py | 103 ++++++++++++++++++ 3 files changed, 145 insertions(+), 2 deletions(-) diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index 56c5cd6..19d47ce 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -130,6 +130,15 @@ async def set_backfill(source_id: int): current pHash threshold; 'stop' cancels either back to tick mode. Returns the updated source dict (incl. backfill_state / backfill_chunks / backfill_bypass_seen).""" + from pathlib import Path + + from ..services.credential_service import CredentialService + from ..services.download_backends import ( + uses_native_ingester, + verify_source_credential, + ) + from .credentials import _get_crypto + payload = await request.get_json(silent=True) or {} action = payload.get("action", "start") if action not in ("start", "stop", "recover"): @@ -137,6 +146,37 @@ async def set_backfill(source_id: int): "invalid_action", detail="action must be 'start', 'stop', or 'recover'", ) + + # Pre-flight (plan #703 #2): before arming a deep walk on a native-ingester + # platform (where verify is one cheap API page), refuse if the credential is + # DEFINITIVELY rejected — don't burn chunks against expired cookies. Proceed + # on valid OR inconclusive (a network blip shouldn't block). Gated to native + # platforms: gallery-dl verify is a slow --simulate subprocess, too heavy for + # an arm action. The credential read happens in a session that's CLOSED + # before the verify network call (don't hold a DB conn across the request). + if action in ("start", "recover"): + async with get_session() as session: + rec = await SourceService(session).get(source_id) + if rec is None: + return _bad("not_found", status=404) + native = uses_native_ingester(rec.platform) + if native: + cred = CredentialService(session, _get_crypto()) + cookies_path = await cred.get_cookies_path(rec.platform) + auth_token = await cred.get_token(rec.platform) + if native: + ok, message = await verify_source_credential( + platform=rec.platform, + url=rec.url, + artist_slug=rec.artist_slug, + config_overrides=rec.config_overrides or {}, + cookies_path=str(cookies_path) if cookies_path else None, + auth_token=auth_token, + images_root=Path("/images"), + ) + if ok is False: + return _bad("credential_rejected", detail=message, status=409) + async with get_session() as session: try: svc = SourceService(session) diff --git a/frontend/src/components/subscriptions/SubscriptionsTab.vue b/frontend/src/components/subscriptions/SubscriptionsTab.vue index cafd618..d804c7a 100644 --- a/frontend/src/components/subscriptions/SubscriptionsTab.vue +++ b/frontend/src/components/subscriptions/SubscriptionsTab.vue @@ -550,7 +550,7 @@ async function onBackfill(source) { await store.loadAll() } catch (e) { toast({ - text: `Backfill ${running ? 'stop' : 'start'} failed: ${e?.detail || e?.message || e}`, + text: `Backfill ${running ? 'stop' : 'start'} failed: ${e?.body?.detail || e?.message || e}`, type: 'error', }) } @@ -566,7 +566,7 @@ async function onRecover(source) { await store.loadAll() } catch (e) { toast({ - text: `Recovery start failed: ${e?.detail || e?.message || e}`, + text: `Recovery start failed: ${e?.body?.detail || e?.message || e}`, type: 'error', }) } diff --git a/tests/test_api_sources.py b/tests/test_api_sources.py index 438b305..71f4dde 100644 --- a/tests/test_api_sources.py +++ b/tests/test_api_sources.py @@ -5,6 +5,20 @@ from backend.app.models import Artist, Source pytestmark = pytest.mark.integration +@pytest.fixture(autouse=True) +def _stub_preflight_verify(monkeypatch): + """Backfill/recover arms run a pre-flight credential verify (plan #703 #2) + which for Patreon would hit the network. Default it to 'valid' so the + endpoint tests stay network-free; the dedicated pre-flight tests override + this with a rejection/inconclusive verdict.""" + from backend.app.services import download_backends as db_mod + + async def _ok(**kwargs): + return (True, "Credentials valid") + + monkeypatch.setattr(db_mod, "verify_source_credential", _ok) + + @pytest.fixture async def artist(db): a = Artist(name="Alice", slug="alice") @@ -275,3 +289,92 @@ async def test_backfill_endpoint_recover_arms_bypass(client, artist, db): stopped_body = await stopped.get_json() assert stopped_body["backfill_state"] is None assert stopped_body["backfill_bypass_seen"] is False + + +# --- Plan #703 #2: pre-flight credential verify on backfill/recover arm ----- + + +@pytest.mark.asyncio +async def test_arm_blocked_when_credential_rejected(client, artist, db, monkeypatch): + """A definitively-rejected credential refuses the arm (409 + reason) instead + of starting a doomed walk; the source is NOT armed.""" + from backend.app.services import download_backends as db_mod + + async def _reject(**kwargs): + return (False, "Patreon rejected the credential — cookies expired") + monkeypatch.setattr(db_mod, "verify_source_credential", _reject) + + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-preflight", enabled=True, + ) + db.add(src) + await db.commit() + sid = src.id + + resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "recover"}) + assert resp.status_code == 409 + assert "expired" in ((await resp.get_json()).get("detail") or "") + + # Not armed. + one = await (await client.get(f"/api/sources/{sid}")).get_json() + assert one["backfill_state"] is None + + +@pytest.mark.asyncio +async def test_arm_proceeds_when_credential_inconclusive(client, artist, db, monkeypatch): + """An inconclusive verify (network blip / drift) must NOT block the arm.""" + from backend.app.services import download_backends as db_mod + + async def _inconclusive(**kwargs): + return (None, "couldn't verify (network)") + monkeypatch.setattr(db_mod, "verify_source_credential", _inconclusive) + + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-incon", enabled=True, + ) + db.add(src) + await db.commit() + + resp = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "start"}) + assert resp.status_code == 200 + assert (await resp.get_json())["backfill_state"] == "running" + + +@pytest.mark.asyncio +async def test_stop_never_pre_flights(client, artist, db, monkeypatch): + from backend.app.services import download_backends as db_mod + + async def _boom(**kwargs): + raise AssertionError("stop must not pre-flight verify") + monkeypatch.setattr(db_mod, "verify_source_credential", _boom) + + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-stop", enabled=True, + ) + db.add(src) + await db.commit() + resp = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "stop"}) + assert resp.status_code == 200 + + +@pytest.mark.asyncio +async def test_gallery_dl_platform_arm_skips_pre_flight(client, artist, db, monkeypatch): + """Pre-flight is gated to native-ingester platforms (cheap verify); a + gallery-dl source arms without the slow --simulate probe.""" + from backend.app.services import download_backends as db_mod + + async def _boom(**kwargs): + raise AssertionError("gallery-dl arm must not pre-flight verify") + monkeypatch.setattr(db_mod, "verify_source_credential", _boom) + + src = Source( + artist_id=artist.id, platform="subscribestar", + url="https://www.subscribestar.com/x", enabled=True, + ) + db.add(src) + await db.commit() + resp = await client.post(f"/api/sources/{src.id}/backfill", json={"action": "start"}) + assert resp.status_code == 200 -- 2.52.0 From e53f8959afd4ace129c6d0b7c9051d7b43a547f3 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 23:39:28 -0400 Subject: [PATCH 12/26] =?UTF-8?q?feat(patreon):=20structured=20ingester=20?= =?UTF-8?q?results=20+=20quarantine=20surfacing=20=E2=80=94=20#704=20step?= =?UTF-8?q?=201?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The native ingester faked gallery-dl stdout (`Cursor:` lines, summary) and phase 3 regex-scraped it back — so Patreon run-stats were mostly zero and quarantine stats blank. We own the ingester, so it now RETURNS structured data and phase 3 reads it directly. - DownloadResult gains run_stats/cursor/posts_processed (None/0 on the gallery-dl path, which keeps the text route). - Ingester builds real run_stats from per-media outcome counts, sets the checkpoint cursor structurally (no fake `Cursor:` stdout), and counts posts processed. download_service phase 3 uses dl_result.run_stats when present; the backfill lifecycle + TIMEOUT→PARTIAL block checkpoint dl_result.cursor instead of parse_last_cursor(stdout). - #4 quarantine: PatreonDownloader reports a distinct "quarantined" MediaOutcome (with the _quarantine dest); the ingester surfaces a real files_quarantined + quarantined_paths + run_stats.quarantined_count (was hardcoded 0). Quarantined media isn't written or marked seen. - Cleanup: parse_last_cursor + _CURSOR_RE (and the now-unused `import re`) removed from gallery_dl — the structured cursor replaced the scrape. Tests: ingester result carries real run_stats/cursor/posts_processed + quarantine counts; downloader quarantines an invalid file as "quarantined"; backfill cursor tests pass cursor= structurally; dropped the parse_last_cursor tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/download_service.py | 24 ++++++----- backend/app/services/gallery_dl.py | 26 +++++------- backend/app/services/patreon_downloader.py | 43 +++++++++++-------- backend/app/services/patreon_ingester.py | 48 ++++++++++++++++++---- tests/test_download_service.py | 17 +++++--- tests/test_gallery_dl_service.py | 29 ++----------- tests/test_patreon_downloader.py | 16 ++++++++ tests/test_patreon_ingester.py | 45 +++++++++++++++++--- 8 files changed, 162 insertions(+), 86 deletions(-) diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index b33e5b4..7139134 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -33,7 +33,6 @@ from .gallery_dl import ( ErrorType, GalleryDLService, SourceConfig, - parse_last_cursor, ) from .importer import Importer from .patreon_ingester import PatreonIngester @@ -155,7 +154,7 @@ class DownloadService: # timed out with NO progress stays TIMEOUT and feeds phase 3's # stall-guard. Leave RATE_LIMITED alone so the platform-cooldown fires. if in_backfill and dl_result.error_type == ErrorType.TIMEOUT: - new_cursor = parse_last_cursor(dl_result.stdout, dl_result.stderr) + new_cursor = dl_result.cursor # plan #704: structured, not scraped advanced = bool( (new_cursor and new_cursor != overrides.get("_backfill_cursor")) or dl_result.files_downloaded > 0 @@ -439,9 +438,14 @@ class DownloadService: select(DownloadEvent).where(DownloadEvent.id == event_id) )).scalar_one() - run_stats = self.gdl._compute_run_stats( - dl_result.return_code, dl_result.stdout, dl_result.stderr - ) + # plan #704: the native ingester returns structured run_stats; only the + # gallery-dl path needs the regex-over-stdout reconstruction. + if dl_result.run_stats is not None: + run_stats = dict(dl_result.run_stats) + else: + run_stats = self.gdl._compute_run_stats( + dl_result.return_code, dl_result.stdout, dl_result.stderr + ) run_stats["quarantined_count"] = dl_result.files_quarantined stderr_summary = self.gdl._extract_errors_warnings(dl_result.stderr) @@ -528,12 +532,12 @@ class DownloadService: src.backfill_runs_remaining = 0 return - # Did not finish. Patreon checkpoints + resumes via cursor; other - # platforms have no resumable cursor (every chunk re-walks from the - # top), so they advance only by the download archive growing. + # Did not finish. The native ingester checkpoints + resumes via cursor + # (carried structurally on the result, plan #704); gallery-dl platforms + # have no resumable cursor (every chunk re-walks from the top), so they + # advance only by the download archive growing. new_cursor = ( - parse_last_cursor(dl_result.stdout, dl_result.stderr) - if uses_native_ingester(ctx["platform"]) else None + dl_result.cursor if uses_native_ingester(ctx["platform"]) else None ) advanced = bool( (new_cursor and new_cursor != old_cursor) diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index bab1d63..f4c9967 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -13,7 +13,6 @@ from __future__ import annotations import asyncio import json import logging -import re import subprocess import sys import tempfile @@ -156,6 +155,14 @@ class DownloadResult: duration_seconds: float = 0.0 started_at: str | None = None completed_at: str | None = None + # Plan #704 — structured fields the NATIVE ingester populates directly (None + # on the gallery-dl path, which keeps the regex-over-stdout route). When set, + # phase 3 reads these instead of scraping the text it would otherwise have to + # reconstruct: `run_stats` mirrors _compute_run_stats' shape; `cursor` is the + # backfill checkpoint the ingester knows exactly (no parse_last_cursor). + run_stats: dict | None = None + cursor: str | None = None + posts_processed: int = 0 def _summarize_validation_failures(failures: list[dict]) -> str: @@ -172,20 +179,9 @@ def _summarize_validation_failures(failures: list[dict]) -> str: return f"{n} files quarantined ({top_count}× {top_reason}, mixed)" -# The native Patreon ingester emits its pagination cursor as `Cursor: ` -# — one line per fetched page — into the DownloadResult stdout (mirroring the -# convention gallery-dl used before the #697 cutover, so the backfill lifecycle -# stayed unchanged). The LAST such line is the furthest-progressed page, i.e. -# the resume point. We scan both streams (be defensive) and take the final -# match; download_service checkpoints it as the next chunk's resume_cursor. -# Survives a time-boxed chunk too: the ingester emits the cursor for a page when -# it STARTS it, so an interrupted walk still yields its last cursor. -_CURSOR_RE = re.compile(r"Cursor:\s*(\S+)") - - -def parse_last_cursor(stdout: str, stderr: str) -> str | None: - matches = _CURSOR_RE.findall(f"{stdout or ''}\n{stderr or ''}") - return matches[-1] if matches else None +# (parse_last_cursor was removed in plan #704: the native ingester now carries +# its checkpoint cursor as a structured DownloadResult.cursor field, so there is +# no log text to scrape — and gallery-dl platforms never had a cursor.) class GalleryDLService: diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index ed46d25..9dda0f4 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -124,11 +124,12 @@ def _post_dir_name(post: dict) -> str: class MediaOutcome: """Per-media result of a download_post pass. - status is one of: "downloaded", "skipped_seen", "skipped_disk", "error". - `path` is the final on-disk path for "downloaded" (the actual yt-dlp output - for video), or the path that already existed for "skipped_disk"; None for - "skipped_seen" and (usually) "error". `error` carries the failure reason for - "error", else None. + status is one of: "downloaded", "skipped_seen", "skipped_disk", + "quarantined", "error". `path` is the final on-disk path for "downloaded" + (the actual yt-dlp output for video), the path that already existed for + "skipped_disk", or the _quarantine destination for "quarantined"; None for + "skipped_seen" and (usually) "error". `error` carries the failure/validation + reason for "error"/"quarantined", else None. """ media: object # MediaItem (avoid importing the name for a bare annotation) @@ -253,10 +254,13 @@ class PatreonDownloader: out_path = Path(out_path) else: out_path = self._fetch_get(media.url, media_path) - invalid = self._validate_path(out_path, artist_slug) - if invalid is not None: + reason, quarantine_dest = self._validate_path(out_path, artist_slug) + if reason is not None: + # Quarantined (corrupt/invalid) — distinct from a download error + # so the run can report a real files_quarantined count + paths. return MediaOutcome( - media=media, status="error", path=None, error=invalid + media=media, status="quarantined", + path=quarantine_dest, error=reason, ) self._write_sidecar(post, out_path) @@ -358,23 +362,29 @@ class PatreonDownloader: # -- validation -------------------------------------------------------- - def _validate_path(self, path: Path, artist_slug: str) -> str | None: - """Validate a freshly-written file; quarantine + return reason if bad. + def _validate_path( + self, path: Path, artist_slug: str + ) -> tuple[str | None, Path | None]: + """Validate a freshly-written file; quarantine if bad. Mirrors gallery_dl._validate_and_quarantine: fail-open for unknown - formats, move the corrupt file to _quarantine//patreon, and return - the failure reason string (None when ok / not validatable / disabled). + formats, move the corrupt file to _quarantine//patreon. Returns + `(reason, quarantine_dest)` when quarantined (dest is the original path if + the move itself failed), else `(None, None)` (ok / not validatable / + disabled). plan #704: the dest is surfaced so the run reports a real + quarantined-paths list instead of a blank count. """ if not self._validate or not is_validatable(path): - return None + return None, None try: result = validate_file(path) except Exception as exc: log.warning("Validator raised on %s: %s", path, exc) - return None + return None, None if result.ok: - return None + return None, None quarantine_root = self.images_root / "_quarantine" / artist_slug / "patreon" + dest = path try: quarantine_root.mkdir(parents=True, exist_ok=True) dest = quarantine_root / path.name @@ -385,7 +395,8 @@ class PatreonDownloader: shutil.move(str(path), str(dest)) except OSError as exc: log.error("Failed to quarantine %s: %s. File left in place.", path, exc) - return result.reason or "validation failed" + dest = path + return (result.reason or "validation failed"), dest # -- sidecar ----------------------------------------------------------- diff --git a/backend/app/services/patreon_ingester.py b/backend/app/services/patreon_ingester.py index 0a40fb6..07186b7 100644 --- a/backend/app/services/patreon_ingester.py +++ b/backend/app/services/patreon_ingester.py @@ -156,8 +156,12 @@ class PatreonIngester: start = time.monotonic() log_lines: list[str] = [] written: list[str] = [] + quarantined_paths: list[str] = [] downloaded = 0 errors = 0 + quarantined = 0 + skipped_count = 0 + posts_processed = 0 consecutive_seen = 0 emitted_cursor: str | None = None reached_bottom = False @@ -168,14 +172,17 @@ class PatreonIngester: *, success: bool, return_code: int, error_type: ErrorType | None, error_message: str | None, ) -> DownloadResult: + # plan #704: return STRUCTURED data — phase 3 reads run_stats/cursor + # directly instead of regex-scraping a reconstructed stdout. stdout + # stays a human-readable summary (no fake `Cursor:` lines). return DownloadResult( success=success, url=url, artist_slug=artist_slug, platform="patreon", files_downloaded=downloaded, - files_quarantined=0, - quarantined_paths=[], + files_quarantined=quarantined, + quarantined_paths=list(quarantined_paths), written_paths=written, stdout="\n".join(log_lines), stderr="", @@ -183,18 +190,28 @@ class PatreonIngester: error_type=error_type, error_message=error_message, duration_seconds=time.monotonic() - start, + cursor=emitted_cursor, + posts_processed=posts_processed, + run_stats={ + "exit_code": return_code, + "downloaded_count": downloaded, + "skipped_count": skipped_count, + "per_item_failures": errors, + "warning_count": 0, + "tier_gated_count": 0, + "quarantined_count": quarantined, + }, ) try: for post, included, page_cursor in self.client.iter_posts( campaign_id, cursor=resume_cursor ): - # Checkpoint: emit the cursor that FETCHED this page once, the - # moment we START it — so a chunk cut mid-page resumes the page, - # not the one after it (matches the gallery-dl cursor semantics - # download_service's lifecycle already depends on). + # Checkpoint the cursor that FETCHED this page the moment we + # START it — so a chunk cut mid-page resumes the page, not the one + # after it. Carried as DownloadResult.cursor (plan #704); no fake + # `Cursor:` stdout line to regex back out. if page_cursor and page_cursor != emitted_cursor: - log_lines.append(f"Cursor: {page_cursor}") emitted_cursor = page_cursor # Time-box check at the post boundary (coarse, like a gallery-dl @@ -203,6 +220,7 @@ class PatreonIngester: budget_hit = True break + posts_processed += 1 media = self.client.extract_media(post, included) if not media: continue @@ -236,9 +254,20 @@ class PatreonIngester: # do NOT re-feed it to phase 3 — attach_in_place would see # the duplicate sha256 and unlink the on-disk copy. to_mark.append((key, media_item.post_id)) + skipped_count += 1 consecutive_seen += 1 elif outcome.status == "skipped_seen": + skipped_count += 1 consecutive_seen += 1 + elif outcome.status == "quarantined": + # New content that failed validation (corrupt) — counted + # distinctly so the run surfaces a real quarantined total. + # Not marked seen (a later walk may re-fetch a fixed file); + # it IS new content, so it breaks the run-of-seen. + quarantined += 1 + if outcome.path is not None: + quarantined_paths.append(str(outcome.path)) + consecutive_seen = 0 elif outcome.status == "error": errors += 1 # An error neither advances nor resets the run-of-seen. @@ -262,9 +291,12 @@ class PatreonIngester: if errors: log_lines.append(f"{errors} media item(s) failed") + if quarantined: + log_lines.append(f"{quarantined} media item(s) quarantined (invalid)") log_lines.append( f"Patreon ingest ({mode}): {downloaded} downloaded, " - f"{errors} error(s)" + f"{skipped_count} skipped, {quarantined} quarantined, " + f"{errors} error(s), {posts_processed} post(s)" + (", reached end" if reached_bottom else "") + (", time-boxed" if budget_hit else "") ) diff --git a/tests/test_download_service.py b/tests/test_download_service.py index 0cd6cbe..4d59fbb 100644 --- a/tests/test_download_service.py +++ b/tests/test_download_service.py @@ -59,7 +59,7 @@ def _make_jpg(path: Path, split: str = "h"): def _make_fake_dl_result( *, success=True, written_paths=None, quarantined_paths=None, files_downloaded=0, error_type=None, error_message=None, - stdout="", stderr="", + stdout="", stderr="", cursor=None, run_stats=None, posts_processed=0, ): return SimpleNamespace( success=success, @@ -78,6 +78,11 @@ def _make_fake_dl_result( duration_seconds=1.23, started_at="2026-05-20T14:00:00+00:00", completed_at="2026-05-20T14:01:00+00:00", + # plan #704: native ingester now returns the cursor + run_stats + # structurally (no more stderr `Cursor:` scraping). + cursor=cursor, + run_stats=run_stats, + posts_processed=posts_processed, ) @@ -505,7 +510,7 @@ async def test_backfill_chunk_progress_advances_cursor( svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result( success=False, written_paths=[], files_downloaded=0, - stderr="[patreon][debug] Cursor: 03:PAGE2:xyz\n", + cursor="03:PAGE2:xyz", )) await svc.download_source(source.id) @@ -534,7 +539,7 @@ async def test_backfill_state_running_selects_backfill_mode_and_resumes( svc, calls = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result( success=False, written_paths=[], - stderr="[patreon][debug] Cursor: 03:RESUME2:next\n", + cursor="03:RESUME2:next", )) await svc.download_source(source.id) @@ -587,7 +592,7 @@ async def test_backfill_cap_exhaustion_stalls( svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result( success=False, written_paths=[], - stderr="[patreon][debug] Cursor: 03:MORE:left\n", + cursor="03:MORE:left", )) await svc.download_source(source.id) @@ -623,7 +628,7 @@ async def test_backfill_stuck_guard_stalls_after_two_no_advance( "config_overrides": {"_backfill_state": "running", "_backfill_cursor": "stuck"}, "backfill_runs_remaining": 5, } - stuck = _make_fake_dl_result(success=False, stderr="Cursor: stuck\n") + stuck = _make_fake_dl_result(success=False, cursor="stuck") await svc._apply_backfill_lifecycle(ctx, stuck) await db.commit() @@ -661,7 +666,7 @@ async def test_backfill_timeout_chunk_reclassified_to_ok( success=False, files_downloaded=3, error_type=ErrorType.TIMEOUT, error_message="Download timed out", - stderr="[patreon][debug] Cursor: 03:NEXT:page\n", + cursor="03:NEXT:page", )) event_id = await svc.download_source(source.id) diff --git a/tests/test_gallery_dl_service.py b/tests/test_gallery_dl_service.py index be06108..76f0997 100644 --- a/tests/test_gallery_dl_service.py +++ b/tests/test_gallery_dl_service.py @@ -8,7 +8,6 @@ from backend.app.services.gallery_dl import ( ErrorType, GalleryDLService, SourceConfig, - parse_last_cursor, ) @@ -316,28 +315,6 @@ def test_categorize_tier_limited_wins_over_partial(gdl): return_code=1, stdout=stdout, stderr=stderr, ) assert etype == ErrorType.TIER_LIMITED - - -# --- Cursor parsing (plan #689 / native ingester #697) --------------------- -# parse_last_cursor is retained post-cutover: the native Patreon ingester emits -# `Cursor: ` per page into DownloadResult.stdout and download_service -# checkpoints the last one. (The gallery-dl Patreon `files`/`cursor` config and -# the Mux yt-dlp Referer block were removed at the #697 cutover, along with the -# tests that pinned them.) - - -def test_parse_last_cursor_returns_last_match_across_streams(): - # One `Cursor: ` line per page; the last is the furthest-progressed - # resume point. - stderr = ( - "[patreon][debug] Cursor: 03:AAAA:bbb\n" - "[urllib3] GET ...\n" - "[patreon][debug] Cursor: 03:CCCC:ddd\n" - ) - assert parse_last_cursor("", stderr) == "03:CCCC:ddd" - - -def test_parse_last_cursor_scans_stdout_too_and_none_when_absent(): - assert parse_last_cursor("Cursor: 99:ZZZ", "") == "99:ZZZ" - assert parse_last_cursor("no marker here", "still nothing") is None - assert parse_last_cursor("", "") is None +# (The cursor-parsing tests were removed in plan #704: parse_last_cursor is gone +# — the native ingester now carries its checkpoint cursor as a structured +# DownloadResult.cursor field instead of a `Cursor:` log line to scrape.) diff --git a/tests/test_patreon_downloader.py b/tests/test_patreon_downloader.py index f96a6fb..8b5d4ff 100644 --- a/tests/test_patreon_downloader.py +++ b/tests/test_patreon_downloader.py @@ -165,6 +165,22 @@ def test_skip_seen(tmp_path): assert not (tmp_path / "artist-x").exists() +def test_invalid_file_is_quarantined(tmp_path): + """plan #704 (#4): a downloaded file that fails validation is moved to + _quarantine and reported as a 'quarantined' outcome carrying that path — + distinct from a download 'error'.""" + bad_url = "https://cdn.patreon.com/bad.png" + session = _FakeSession({bad_url: b"this is not a valid PNG"}) + dl = _downloader(tmp_path, session=session, validate=True) + outcomes = dl.download_post( + _post(), [_img("bad.png", url=bad_url)], "artist-x" + ) + assert outcomes[0].status == "quarantined" + assert outcomes[0].path is not None + assert "_quarantine" in str(outcomes[0].path) + assert outcomes[0].path.is_file() + + def test_skip_disk(tmp_path): dl = _downloader(tmp_path) post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title" diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index 15b8824..7f40c0c 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -63,11 +63,13 @@ class _FakeClient: class _FakeDownloader: """Stub PatreonDownloader. Honors the injected is_seen (tier-1); media in - `on_disk` report skipped_disk (tier-2); everything else downloads.""" + `on_disk` report skipped_disk (tier-2); media in `quarantine` report + quarantined; everything else downloads.""" - def __init__(self, tmp_path, on_disk=None): + def __init__(self, tmp_path, on_disk=None, quarantine=None): self.tmp_path = tmp_path self.on_disk = set(on_disk or ()) + self.quarantine = set(quarantine or ()) self.download_calls = 0 def download_post(self, post, media_items, artist_slug, *, is_seen): @@ -78,6 +80,10 @@ class _FakeDownloader: elif _ledger_key(m) in self.on_disk: p = self.tmp_path / f"{m.post_id}_{m.filename}" outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None)) + elif _ledger_key(m) in self.quarantine: + self.download_calls += 1 + q = self.tmp_path / "_quarantine" / f"{m.post_id}_{m.filename}" + outcomes.append(MediaOutcome(media=m, status="quarantined", path=q, error="bad image")) else: self.download_calls += 1 p = self.tmp_path / f"{m.post_id}_{m.filename}" @@ -137,9 +143,36 @@ async def test_tick_downloads_unseen_and_marks_seen(source_id, sync_engine, tmp_ assert result.return_code == 0 assert result.files_downloaded == 2 assert len(result.written_paths) == 2 + # plan #704: structured run_stats carry the real counts. + assert result.run_stats["downloaded_count"] == 2 + assert result.posts_processed == 1 assert _count_ledger(sync_engine, source_id) == 2 +@pytest.mark.asyncio +async def test_quarantined_media_surfaced_in_result(source_id, sync_engine, tmp_path): + """plan #704 (#4): a media that fails validation is reported as quarantined — + a real files_quarantined count + paths + run_stats — not a blank 0, and is + NOT written or marked seen.""" + m1, m2 = _media("p1", 1), _media("p1", 2) + client = _FakeClient([(None, [("p1", [m1, m2])])]) + downloader = _FakeDownloader(tmp_path, quarantine={_ledger_key(m2)}) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + assert result.files_downloaded == 1 # only m1 + assert result.files_quarantined == 1 # m2 + assert len(result.quarantined_paths) == 1 + assert result.run_stats["quarantined_count"] == 1 + assert result.run_stats["downloaded_count"] == 1 + assert len(result.written_paths) == 1 # quarantined NOT written + # Quarantined media is NOT marked seen (a fixed file may be re-fetched). + assert _count_ledger(sync_engine, source_id) == 1 + + @pytest.mark.asyncio async def test_tick_skips_seen_via_ledger(source_id, sync_engine, tmp_path): m1 = _media("p1", 1) @@ -206,8 +239,10 @@ async def test_backfill_reaches_bottom_signals_complete(source_id, sync_engine, assert result.return_code == 0 assert result.error_type is None assert result.files_downloaded == 2 - # The page-2 cursor was emitted for checkpointing. - assert "Cursor: CUR2" in result.stdout + # The page-2 cursor is carried structurally for checkpointing (plan #704). + assert result.cursor == "CUR2" + assert result.posts_processed == 2 + assert result.run_stats["downloaded_count"] == 2 @pytest.mark.asyncio @@ -245,7 +280,7 @@ async def test_backfill_budget_cut_returns_partial_with_progress( assert result.return_code == -1 assert result.files_downloaded == 1 # post1 downloaded before the cut assert downloader.download_calls == 1 # post2's body never ran - assert "Cursor: CUR1" in result.stdout + assert result.cursor == "CUR1" # structured checkpoint (plan #704) # --- recovery ------------------------------------------------------------- -- 2.52.0 From b2e59e7e1708e62e86804748fbe94e592124da37 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 23:43:23 -0400 Subject: [PATCH 13/26] =?UTF-8?q?feat(subscriptions):=20live=20posts-proce?= =?UTF-8?q?ssed=20progress=20on=20backfill/recovery=20=E2=80=94=20#704=20s?= =?UTF-8?q?tep=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The running badge only showed the chunk counter; now it shows posts walked — real walk progress. The ingester already reports posts_processed per chunk (step 1); the backfill lifecycle accumulates it into config_overrides._backfill_posts across chunks. SourceRecord exposes backfill_posts; start_backfill/start_recovery clear it (fresh walk); stop clears it too. SourceRow/SourceCard badge renders "Recovering · 45 posts" (falls back to "(N)" chunks before any posts are counted). Per-chunk accumulation (no mid-walk DB write) — simple and race-free; a small over-count from each chunk re-walking its resumed page is fine for a progress indicator. Tests: lifecycle accumulates posts_processed across chunks; start clears a prior _backfill_posts and the record exposes it. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/download_service.py | 7 ++++ backend/app/services/source_service.py | 13 +++++-- .../components/subscriptions/SourceCard.vue | 4 ++- .../components/subscriptions/SourceRow.vue | 4 ++- tests/test_download_service.py | 34 +++++++++++++++++++ tests/test_source_service.py | 10 ++++++ 6 files changed, 67 insertions(+), 5 deletions(-) diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 7139134..47cd4c0 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -515,6 +515,13 @@ class DownloadService: new_overrides = dict(src.config_overrides or {}) chunks = int(new_overrides.get("_backfill_chunks", 0)) + 1 new_overrides["_backfill_chunks"] = chunks + # plan #704 (#5): accumulate posts-processed across chunks so the badge + # shows live walk progress, not just the chunk count. Cleared on + # (re)start. (Slight over-count: each chunk re-walks the resumed page — + # fine for a progress indicator.) + new_overrides["_backfill_posts"] = int( + new_overrides.get("_backfill_posts", 0) + ) + int(getattr(dl_result, "posts_processed", 0) or 0) completed = ( dl_result.success diff --git a/backend/app/services/source_service.py b/backend/app/services/source_service.py index 67b2cf2..073b4d8 100644 --- a/backend/app/services/source_service.py +++ b/backend/app/services/source_service.py @@ -70,6 +70,9 @@ class SourceRecord: # plan #697: a running deep-walk that bypasses the Patreon seen-ledger # (recovery) vs. a normal backfill. Lets the badge label it "Recovering". backfill_bypass_seen: bool + # plan #704: cumulative posts processed across the walk's chunks — live + # progress for the badge. + backfill_posts: int def to_dict(self) -> dict: return { @@ -91,6 +94,7 @@ class SourceRecord: "backfill_state": self.backfill_state, "backfill_chunks": self.backfill_chunks, "backfill_bypass_seen": self.backfill_bypass_seen, + "backfill_posts": self.backfill_posts, } @@ -167,6 +171,7 @@ class SourceService: backfill_state=co.get("_backfill_state"), backfill_chunks=int(co.get("_backfill_chunks", 0)), backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")), + backfill_posts=int(co.get("_backfill_posts", 0)), ) async def _row_to_record(self, source: Source) -> SourceRecord: @@ -317,7 +322,8 @@ class SourceService: raise LookupError(f"source id={source_id} not found") co = dict(source.config_overrides or {}) co["_backfill_state"] = "running" - for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks"): + for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks", + "_backfill_posts"): co.pop(k, None) source.config_overrides = co source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS @@ -345,7 +351,8 @@ class SourceService: co = dict(source.config_overrides or {}) co["_backfill_state"] = "running" co["_backfill_bypass_seen"] = True - for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks"): + for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks", + "_backfill_posts"): co.pop(k, None) source.config_overrides = co source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS @@ -362,7 +369,7 @@ class SourceService: raise LookupError(f"source id={source_id} not found") co = dict(source.config_overrides or {}) for k in ("_backfill_state", "_backfill_cursor", "_backfill_cursor_stalls", - "_backfill_chunks", "_backfill_bypass_seen"): + "_backfill_chunks", "_backfill_bypass_seen", "_backfill_posts"): co.pop(k, None) source.config_overrides = co source.backfill_runs_remaining = 0 diff --git a/frontend/src/components/subscriptions/SourceCard.vue b/frontend/src/components/subscriptions/SourceCard.vue index 4312d8f..c0c092f 100644 --- a/frontend/src/components/subscriptions/SourceCard.vue +++ b/frontend/src/components/subscriptions/SourceCard.vue @@ -28,7 +28,9 @@ v-else-if="source.backfill_state === 'running'" size="x-small" color="info" variant="tonal" label >{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling' - }}{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }} + }}{{ source.backfill_posts + ? ` · ${source.backfill_posts} posts` + : (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }} {{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling' - }}{{ source.backfill_chunks ? ` (${source.backfill_chunks})` : '' }} + }}{{ source.backfill_posts + ? ` · ${source.backfill_posts} posts` + : (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }} Date: Fri, 5 Jun 2026 23:46:02 -0400 Subject: [PATCH 14/26] =?UTF-8?q?test(patreon):=20fix=20#704=20=E2=80=94?= =?UTF-8?q?=20quarantine=20status=20+=20budget-cut=20cursor=20assertions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two test breaks from the structured-results change: - An existing downloader test pinned a corrupt file to status "error"; it's now the distinct "quarantined" status (the new behavior). Updated it + removed the duplicate I'd added. - The budget-cut ingester test asserted the checkpoint cursor was the last FULLY-processed page (CUR1); it's actually the page we were cut on (CUR2, entered + cursor emitted before the budget check), matching the prior parse_last_cursor(last) semantics. Corrected the assertion. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_patreon_downloader.py | 25 ++++++------------------- tests/test_patreon_ingester.py | 5 ++++- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/tests/test_patreon_downloader.py b/tests/test_patreon_downloader.py index 8b5d4ff..ea89752 100644 --- a/tests/test_patreon_downloader.py +++ b/tests/test_patreon_downloader.py @@ -165,22 +165,6 @@ def test_skip_seen(tmp_path): assert not (tmp_path / "artist-x").exists() -def test_invalid_file_is_quarantined(tmp_path): - """plan #704 (#4): a downloaded file that fails validation is moved to - _quarantine and reported as a 'quarantined' outcome carrying that path — - distinct from a download 'error'.""" - bad_url = "https://cdn.patreon.com/bad.png" - session = _FakeSession({bad_url: b"this is not a valid PNG"}) - dl = _downloader(tmp_path, session=session, validate=True) - outcomes = dl.download_post( - _post(), [_img("bad.png", url=bad_url)], "artist-x" - ) - assert outcomes[0].status == "quarantined" - assert outcomes[0].path is not None - assert "_quarantine" in str(outcomes[0].path) - assert outcomes[0].path.is_file() - - def test_skip_disk(tmp_path): dl = _downloader(tmp_path) post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title" @@ -248,14 +232,17 @@ def test_one_failure_isolated(tmp_path): def test_validation_quarantines_corrupt(tmp_path): - # A .png whose bytes are not a valid PNG → validator fails → error + - # quarantine move; with validate=False it would pass. + # A .png whose bytes are not a valid PNG → validator fails → "quarantined" + # outcome (distinct from a download error) carrying the dest path + + # quarantine move; with validate=False it would pass. plan #704 (#4). bad = b"not a real png" session = _FakeSession({"https://cdn.patreon.com/corrupt.png": bad}) dl = _downloader(tmp_path, session=session, validate=True) item = _img("corrupt.png", url="https://cdn.patreon.com/corrupt.png") outcomes = dl.download_post(_post(), [item], "artist-x") - assert outcomes[0].status == "error" + assert outcomes[0].status == "quarantined" + assert outcomes[0].path is not None + assert "_quarantine" in str(outcomes[0].path) post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title" assert not (post_dir / "01_corrupt.png").exists() quarantine = tmp_path / "_quarantine" / "artist-x" / "patreon" diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index 7f40c0c..b557740 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -280,7 +280,10 @@ async def test_backfill_budget_cut_returns_partial_with_progress( assert result.return_code == -1 assert result.files_downloaded == 1 # post1 downloaded before the cut assert downloader.download_calls == 1 # post2's body never ran - assert result.cursor == "CUR1" # structured checkpoint (plan #704) + # The checkpoint is the page we were CUT ON (CUR2 — entered, cursor emitted, + # then the budget check broke before processing it); the next chunk resumes + # there. Structured (plan #704), matching the old parse_last_cursor(last). + assert result.cursor == "CUR2" # --- recovery ------------------------------------------------------------- -- 2.52.0 From 4bb11ce7dc20bfe873caa56748b37824d8864bc4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 23:58:16 -0400 Subject: [PATCH 15/26] =?UTF-8?q?feat(patreon):=20incremental=20cursor=20c?= =?UTF-8?q?heckpoint=20mid-walk=20=E2=80=94=20#705=20step=201=20(#6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A worker SIGKILL (hard-time-limit backstop) mid-chunk lost the whole chunk's walk — the cursor was only persisted at chunk boundaries by phase 3, so the next tick re-walked from the chunk start. Now the ingester checkpoints _backfill_cursor to the DB at each page boundary (backfill/ recovery only) via an ATOMIC single-key UPDATE (config_overrides::jsonb → jsonb_set('{_backfill_cursor}') → ::json), so it never clobbers operator config or other backfill keys. On a crash the last mid-walk cursor survives → the next chunk resumes near the crash, not the chunk start. phase 3 still writes the final cursor (same value); this is the safety net. Tests: a backfill walk leaves the last page's cursor in the DB (written by the ingester, before any phase 3); a tick never checkpoints. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/patreon_ingester.py | 33 ++++++++++++++++++- tests/test_patreon_ingester.py | 41 ++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/backend/app/services/patreon_ingester.py b/backend/app/services/patreon_ingester.py index 07186b7..5207ff6 100644 --- a/backend/app/services/patreon_ingester.py +++ b/backend/app/services/patreon_ingester.py @@ -44,7 +44,7 @@ import time from collections.abc import Callable from pathlib import Path -from sqlalchemy import select +from sqlalchemy import select, text from sqlalchemy.dialects.postgresql import insert as pg_insert from ..models import PatreonSeenMedia @@ -153,6 +153,9 @@ class PatreonIngester: A client-level failure (drift / auth / network) fails the whole run loud. """ bypass_seen = mode == "recovery" + # Only deep walks checkpoint their cursor mid-flight (plan #705 #6); a + # tick has no resumable backfill state. + checkpoint = mode in ("backfill", "recovery") start = time.monotonic() log_lines: list[str] = [] written: list[str] = [] @@ -213,6 +216,12 @@ class PatreonIngester: # `Cursor:` stdout line to regex back out. if page_cursor and page_cursor != emitted_cursor: emitted_cursor = page_cursor + # plan #705 #6: persist the cursor at each page boundary so a + # worker SIGKILL mid-chunk resumes near the crash, not the + # chunk start. (phase 3 still writes the final cursor — same + # value; this is the crash-safety net.) + if checkpoint: + self._checkpoint_cursor(source_id, emitted_cursor) # Time-box check at the post boundary (coarse, like a gallery-dl # chunk). Backfill/recovery resume from emitted_cursor next chunk. @@ -394,6 +403,28 @@ class PatreonIngester: ).scalars().all() return set(rows) + def _checkpoint_cursor(self, source_id: int, cursor: str) -> None: + """Persist the in-progress backfill cursor mid-walk (plan #705 #6). + + ATOMIC, single-key UPDATE: cast the JSON column to jsonb, set just + `_backfill_cursor`, cast back — so it never clobbers operator config or + the other backfill keys (no read-modify-write race). The in-flight guard + means only this source's one download runs at a time; a concurrent + operator stop is benign (a stray cursor with no `_backfill_state` is + ignored by tick mode and cleared on the next start). + """ + with self.session_factory() as session: + session.execute( + text( + "UPDATE source SET config_overrides = jsonb_set(" + " coalesce(config_overrides::jsonb, '{}'::jsonb)," + " '{_backfill_cursor}', to_jsonb(cast(:cur AS text))" + ")::json WHERE id = :sid" + ), + {"cur": cursor, "sid": source_id}, + ) + session.commit() + def _mark_seen(self, source_id: int, items: list[tuple[str, str]]) -> None: """Idempotent upsert of (filehash, post_id) ledger rows for a page. diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index b557740..e5a0015 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -245,6 +245,47 @@ async def test_backfill_reaches_bottom_signals_complete(source_id, sync_engine, assert result.run_stats["downloaded_count"] == 2 +@pytest.mark.asyncio +async def test_backfill_checkpoints_cursor_mid_walk(source_id, sync_engine, tmp_path, db): + """plan #705 #6: the cursor is persisted to the DB at each page boundary + DURING the walk (not just at the end via phase 3), so a worker SIGKILL + mid-chunk resumes near the crash. After the walk the source carries the last + page's cursor — written by the ingester, not phase 3 (which the unit run + never reaches).""" + pages = [ + (None, [("p1", [_media("p1", 1)])]), + ("CUR2", [("p2", [_media("p2", 1)])]), + ("CUR3", [("p3", [_media("p3", 1)])]), + ] + ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path)) + ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="backfill", + ) + co = (await db.execute( + select(Source.config_overrides).where(Source.id == source_id) + )).scalar_one() + assert co.get("_backfill_cursor") == "CUR3" + + +@pytest.mark.asyncio +async def test_tick_does_not_checkpoint_cursor(source_id, sync_engine, tmp_path, db): + """A tick has no resumable backfill state, so it must not write a cursor.""" + ing = _ingester( + sync_engine, tmp_path, + _FakeClient([("CUR2", [("p1", [_media("p1", 1)])])]), + _FakeDownloader(tmp_path), + ) + ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + co = (await db.execute( + select(Source.config_overrides).where(Source.id == source_id) + )).scalar_one() + assert "_backfill_cursor" not in (co or {}) + + @pytest.mark.asyncio async def test_backfill_budget_cut_returns_partial_with_progress( source_id, sync_engine, tmp_path, monkeypatch, -- 2.52.0 From 7a872a3619e1fdd6535063266d933b2bcff3f9ee Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 00:04:24 -0400 Subject: [PATCH 16/26] =?UTF-8?q?feat(patreon):=20dead-letter=20ledger=20f?= =?UTF-8?q?or=20permanently-failing=20media=20=E2=80=94=20#705=20step=202?= =?UTF-8?q?=20(#7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A media that fails every walk (404'd CDN, deleted post, geo-blocked Mux, persistently-corrupt bytes) used to re-error forever and re-burn chunks. New `patreon_failed_media` table (alembic 0038, chains 0037) records per-media attempts; once attempts reach DEAD_LETTER_THRESHOLD (3) the ingester skips it on routine tick/backfill walks (tier-1.5, folded into the seen/skip predicate). Recovery BYPASSES it (the operator's "try everything again" re-attempts dead media). A clean download clears the row (recovered); errors/quarantines upsert-increment it. Surfaced as run_stats.dead_lettered_count. - New PatreonFailedMedia model + migration; ingester _dead_keys / _record_failures (on_conflict increment) / _clear_failures. - skip = seen | dead (empty in recovery); failures recorded post-fetch on short sessions (same pattern as the seen-ledger). Tests: a media erroring 3× is dead-lettered + skipped (no download attempt); recovery re-attempts a dead media and clears it on success; a clean download clears a sub-threshold failure. Co-Authored-By: Claude Opus 4.8 (1M context) --- alembic/versions/0038_patreon_failed_media.py | 58 +++++++++ backend/app/models/__init__.py | 2 + backend/app/models/patreon_failed_media.py | 45 +++++++ backend/app/services/patreon_ingester.py | 110 ++++++++++++++++-- tests/test_patreon_ingester.py | 106 ++++++++++++++++- 5 files changed, 311 insertions(+), 10 deletions(-) create mode 100644 alembic/versions/0038_patreon_failed_media.py create mode 100644 backend/app/models/patreon_failed_media.py diff --git a/alembic/versions/0038_patreon_failed_media.py b/alembic/versions/0038_patreon_failed_media.py new file mode 100644 index 0000000..e907ae1 --- /dev/null +++ b/alembic/versions/0038_patreon_failed_media.py @@ -0,0 +1,58 @@ +"""patreon_failed_media: per-source dead-letter ledger for failing Patreon media + +Revision ID: 0038 +Revises: 0037 +Create Date: 2026-06-06 + +Plan #705 (#7). Media that keeps failing to download/validate (404'd CDN, +deleted post, geo-blocked Mux, persistently-corrupt bytes) gets recorded here +with an attempt counter; once it crosses the dead-letter threshold the ingester +skips it on routine walks (recovery still re-attempts). A clean download clears +the row. UNIQUE (source_id, filehash) is the upsert key (same media key the +seen-ledger uses). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0038" +down_revision: Union[str, None] = "0037" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "patreon_failed_media", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "source_id", + sa.Integer, + sa.ForeignKey("source.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("filehash", sa.String(128), nullable=False), + sa.Column("attempts", sa.Integer, nullable=False, server_default="1"), + sa.Column("last_error", sa.Text, nullable=True), + sa.Column( + "first_failed_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.Column( + "last_failed_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.UniqueConstraint( + "source_id", "filehash", name="uq_patreon_failed_media_source_id" + ), + ) + + +def downgrade() -> None: + op.drop_table("patreon_failed_media") diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 1445788..b8854b1 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -14,6 +14,7 @@ from .import_settings import ImportSettings from .import_task import ImportTask from .library_audit_run import LibraryAuditRun from .ml_settings import MLSettings +from .patreon_failed_media import PatreonFailedMedia from .patreon_seen_media import PatreonSeenMedia from .post import Post from .post_attachment import PostAttachment @@ -34,6 +35,7 @@ __all__ = [ "BackupRun", "Source", "Credential", + "PatreonFailedMedia", "PatreonSeenMedia", "Post", "PostAttachment", diff --git a/backend/app/models/patreon_failed_media.py b/backend/app/models/patreon_failed_media.py new file mode 100644 index 0000000..79976ef --- /dev/null +++ b/backend/app/models/patreon_failed_media.py @@ -0,0 +1,45 @@ +"""PatreonFailedMedia — per-source dead-letter ledger of Patreon media that +keeps failing to download/validate. + +Plan #705 (#7). A media that fails every walk (404'd CDN URL, deleted post, +geo-blocked Mux stream, persistently-corrupt bytes) would otherwise re-error +forever and re-burn backfill chunks. After ``attempts`` reaches the dead-letter +threshold the ingester skips it on routine tick/backfill walks (recovery still +re-attempts it — the operator's "try everything again"). A later clean download +clears the row (the media recovered). + +`filehash` is the same per-media key the seen-ledger uses (32-hex CDN MD5, or a +``video:`` / ``post:filename`` synthesized key) — hence String(128). UNIQUE +(source_id, filehash) is the upsert key. +""" + +from datetime import datetime + +from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import DateTime + +from .base import Base + + +class PatreonFailedMedia(Base): + __tablename__ = "patreon_failed_media" + __table_args__ = ( + UniqueConstraint( + "source_id", "filehash", name="uq_patreon_failed_media_source_id" + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source_id: Mapped[int] = mapped_column( + ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + ) + filehash: Mapped[str] = mapped_column(String(128), nullable=False) + attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + last_error: Mapped[str | None] = mapped_column(Text, nullable=True) + first_failed_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + last_failed_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/services/patreon_ingester.py b/backend/app/services/patreon_ingester.py index 5207ff6..206b555 100644 --- a/backend/app/services/patreon_ingester.py +++ b/backend/app/services/patreon_ingester.py @@ -44,10 +44,10 @@ import time from collections.abc import Callable from pathlib import Path -from sqlalchemy import select, text +from sqlalchemy import delete, func, select, text from sqlalchemy.dialects.postgresql import insert as pg_insert -from ..models import PatreonSeenMedia +from ..models import PatreonFailedMedia, PatreonSeenMedia from .gallery_dl import DownloadResult, ErrorType from .patreon_client import ( MediaItem, @@ -72,6 +72,14 @@ _TICK_SEEN_THRESHOLD = 20 # synthesized key so a pathologically long file_name can't overflow the column. _LEDGER_KEY_MAX = 128 +# plan #705 #7: after this many failed download/validate attempts a media is +# "dead-lettered" and skipped on routine tick/backfill walks (recovery still +# re-attempts it). Stops a permanently-broken media (404'd CDN, deleted post) +# re-erroring forever and re-burning chunks. +DEAD_LETTER_THRESHOLD = 3 +# last_error is Text but bound it so a giant traceback doesn't bloat the row. +_ERROR_MAX = 1000 + def _ledger_key(media: MediaItem) -> str: """Stable per-media identity for the cross-run seen-ledger. @@ -163,6 +171,7 @@ class PatreonIngester: downloaded = 0 errors = 0 quarantined = 0 + dead_lettered = 0 skipped_count = 0 posts_processed = 0 consecutive_seen = 0 @@ -203,6 +212,7 @@ class PatreonIngester: "warning_count": 0, "tier_gated_count": 0, "quarantined_count": quarantined, + "dead_lettered_count": dead_lettered, }, ) @@ -235,27 +245,37 @@ class PatreonIngester: continue keys = [_ledger_key(m) for m in media] + # Recovery bypasses BOTH the seen-ledger AND the dead-letter + # ledger (the operator's "try everything again"); routine walks + # skip seen + dead media (tier-1 + tier-1.5, plan #705 #7). + dead = set() if bypass_seen else self._dead_keys(source_id, keys) seen = ( set() if bypass_seen else self._seen_keys(source_id, keys) ) + skip = seen | dead - def _is_seen(m: MediaItem, _seen=seen) -> bool: - return _ledger_key(m) in _seen + def _is_skip(m: MediaItem, _skip=skip) -> bool: + return _ledger_key(m) in _skip outcomes = self.downloader.download_post( - post, media, artist_slug, is_seen=_is_seen + post, media, artist_slug, is_seen=_is_skip ) to_mark: list[tuple[str, str]] = [] + to_clear: list[str] = [] # recovered → drop any dead-letter row + to_fail: list[tuple[str, str, str]] = [] # (key, post_id, error) for media_item, outcome in zip(media, outcomes, strict=False): key = _ledger_key(media_item) + if key in dead: + dead_lettered += 1 # skipped because previously dead if outcome.status == "downloaded": downloaded += 1 if outcome.path is not None: written.append(str(outcome.path)) to_mark.append((key, media_item.post_id)) + to_clear.append(key) consecutive_seen = 0 elif outcome.status == "skipped_disk": # Already on disk (a prior run). Reconcile the ledger so a @@ -263,6 +283,7 @@ class PatreonIngester: # do NOT re-feed it to phase 3 — attach_in_place would see # the duplicate sha256 and unlink the on-disk copy. to_mark.append((key, media_item.post_id)) + to_clear.append(key) skipped_count += 1 consecutive_seen += 1 elif outcome.status == "skipped_seen": @@ -272,22 +293,31 @@ class PatreonIngester: # New content that failed validation (corrupt) — counted # distinctly so the run surfaces a real quarantined total. # Not marked seen (a later walk may re-fetch a fixed file); - # it IS new content, so it breaks the run-of-seen. + # it IS new content, so it breaks the run-of-seen. Counts + # toward the dead-letter ledger (plan #705 #7). quarantined += 1 if outcome.path is not None: quarantined_paths.append(str(outcome.path)) + to_fail.append((key, media_item.post_id, outcome.error or "quarantined")) consecutive_seen = 0 elif outcome.status == "error": errors += 1 + to_fail.append((key, media_item.post_id, outcome.error or "error")) # An error neither advances nor resets the run-of-seen. if mode == "tick" and consecutive_seen >= seen_threshold: early_out = True break - # Mark seen AFTER the network fetch, on its own short session. + # Persist ledger changes AFTER the network fetch, on short + # sessions: mark downloaded/on-disk seen, clear any dead-letter + # for recovered media, and record failures (plan #705 #7). if to_mark: self._mark_seen(source_id, to_mark) + if to_clear: + self._clear_failures(source_id, to_clear) + if to_fail: + self._record_failures(source_id, to_fail) if early_out: break @@ -302,10 +332,13 @@ class PatreonIngester: log_lines.append(f"{errors} media item(s) failed") if quarantined: log_lines.append(f"{quarantined} media item(s) quarantined (invalid)") + if dead_lettered: + log_lines.append(f"{dead_lettered} media item(s) skipped (dead-lettered)") log_lines.append( f"Patreon ingest ({mode}): {downloaded} downloaded, " f"{skipped_count} skipped, {quarantined} quarantined, " - f"{errors} error(s), {posts_processed} post(s)" + f"{dead_lettered} dead-lettered, {errors} error(s), " + f"{posts_processed} post(s)" + (", reached end" if reached_bottom else "") + (", time-boxed" if budget_hit else "") ) @@ -453,6 +486,67 @@ class PatreonIngester: session.execute(stmt) session.commit() + # -- dead-letter ledger (plan #705 #7) --------------------------------- + + def _dead_keys(self, source_id: int, keys: list[str]) -> set[str]: + """Which of `keys` have failed >= DEAD_LETTER_THRESHOLD times (dead). + One short SELECT; recovery never calls this (it re-attempts dead media).""" + if not keys: + return set() + with self.session_factory() as session: + rows = session.execute( + select(PatreonFailedMedia.filehash).where( + PatreonFailedMedia.source_id == source_id, + PatreonFailedMedia.filehash.in_(keys), + PatreonFailedMedia.attempts >= DEAD_LETTER_THRESHOLD, + ) + ).scalars().all() + return set(rows) + + def _record_failures( + self, source_id: int, items: list[tuple[str, str, str]] + ) -> None: + """Upsert-increment the dead-letter ledger for failed media. On conflict + bump `attempts` and refresh last_error/last_failed_at (UNIQUE backs the + upsert — no check-then-insert, [[scalar_one_or_none-duplicates]]). De-dup + the batch locally (one row per key, last error wins).""" + by_key: dict[str, str] = {} + for key, _post_id, err in items: + by_key[key] = (err or "")[:_ERROR_MAX] + if not by_key: + return + values = [ + {"source_id": source_id, "filehash": k, "attempts": 1, "last_error": e} + for k, e in by_key.items() + ] + with self.session_factory() as session: + stmt = pg_insert(PatreonFailedMedia).values(values) + stmt = stmt.on_conflict_do_update( + constraint="uq_patreon_failed_media_source_id", + set_={ + "attempts": PatreonFailedMedia.attempts + 1, + "last_error": stmt.excluded.last_error, + "last_failed_at": func.now(), + }, + ) + session.execute(stmt) + session.commit() + + def _clear_failures(self, source_id: int, keys: list[str]) -> None: + """Drop dead-letter rows for media that just downloaded cleanly — they + recovered. A no-op DELETE for keys that were never failing.""" + unique = list(dict.fromkeys(keys)) + if not unique: + return + with self.session_factory() as session: + session.execute( + delete(PatreonFailedMedia).where( + PatreonFailedMedia.source_id == source_id, + PatreonFailedMedia.filehash.in_(unique), + ) + ) + session.commit() + async def verify_patreon_credential( url: str, diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index e5a0015..0b3df18 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -11,7 +11,12 @@ import pytest from sqlalchemy import func, select from sqlalchemy.orm import sessionmaker -from backend.app.models import Artist, PatreonSeenMedia, Source +from backend.app.models import ( + Artist, + PatreonFailedMedia, + PatreonSeenMedia, + Source, +) from backend.app.services.gallery_dl import ErrorType from backend.app.services.patreon_client import ( MediaItem, @@ -66,10 +71,11 @@ class _FakeDownloader: `on_disk` report skipped_disk (tier-2); media in `quarantine` report quarantined; everything else downloads.""" - def __init__(self, tmp_path, on_disk=None, quarantine=None): + def __init__(self, tmp_path, on_disk=None, quarantine=None, error=None): self.tmp_path = tmp_path self.on_disk = set(on_disk or ()) self.quarantine = set(quarantine or ()) + self.error = set(error or ()) self.download_calls = 0 def download_post(self, post, media_items, artist_slug, *, is_seen): @@ -80,6 +86,9 @@ class _FakeDownloader: elif _ledger_key(m) in self.on_disk: p = self.tmp_path / f"{m.post_id}_{m.filename}" outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None)) + elif _ledger_key(m) in self.error: + self.download_calls += 1 + outcomes.append(MediaOutcome(media=m, status="error", path=None, error="boom")) elif _ledger_key(m) in self.quarantine: self.download_calls += 1 q = self.tmp_path / "_quarantine" / f"{m.post_id}_{m.filename}" @@ -370,6 +379,99 @@ async def test_recovery_tier2_disk_still_skips(source_id, sync_engine, tmp_path) assert _count_ledger(sync_engine, source_id) == 1 +# --- dead-letter ledger (plan #705 #7) ------------------------------------ + + +def _count_failed(sync_engine, source_id): + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + return s.execute( + select(func.count(PatreonFailedMedia.id)).where( + PatreonFailedMedia.source_id == source_id + ) + ).scalar_one() + + +def _seed_failed(sync_engine, source_id, key, attempts): + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + s.add(PatreonFailedMedia(source_id=source_id, filehash=key, attempts=attempts)) + s.commit() + + +@pytest.mark.asyncio +async def test_repeated_failures_dead_letter_then_skip(source_id, sync_engine, tmp_path): + """A media that errors DEAD_LETTER_THRESHOLD times is recorded; the next walk + skips it (no download attempt) and counts it dead-lettered.""" + from backend.app.services.patreon_ingester import DEAD_LETTER_THRESHOLD + + m1 = _media("p1", 1) + + def _run(): + ing = _ingester( + sync_engine, tmp_path, + _FakeClient([(None, [("p1", [m1])])]), + _FakeDownloader(tmp_path, error={_ledger_key(m1)}), + ) + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + return result, ing.downloader + + # Fail it up to the threshold — each run attempts the download and errors. + for _ in range(DEAD_LETTER_THRESHOLD): + result, dl = _run() + assert dl.download_calls == 1 + assert _count_failed(sync_engine, source_id) == 1 + + # Now attempts == threshold → dead: the next walk skips it without trying. + result, dl = _run() + assert dl.download_calls == 0 + assert result.run_stats["dead_lettered_count"] == 1 + + +@pytest.mark.asyncio +async def test_recovery_reattempts_dead_lettered_and_clears_on_success( + source_id, sync_engine, tmp_path, +): + """Recovery bypasses the dead-letter ledger (re-attempts dead media); a clean + download clears the row (it recovered).""" + from backend.app.services.patreon_ingester import DEAD_LETTER_THRESHOLD + + m1 = _media("p1", 1) + _seed_failed(sync_engine, source_id, _ledger_key(m1), DEAD_LETTER_THRESHOLD) + + downloader = _FakeDownloader(tmp_path) # succeeds + ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader) + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="recovery", + ) + assert downloader.download_calls == 1 # re-attempted despite being dead + assert result.files_downloaded == 1 + assert _count_failed(sync_engine, source_id) == 0 # cleared on success + + +@pytest.mark.asyncio +async def test_clean_download_clears_below_threshold_failure( + source_id, sync_engine, tmp_path, +): + """A media with a sub-threshold failure that now downloads cleanly (tick) has + its dead-letter row cleared.""" + m1 = _media("p1", 1) + _seed_failed(sync_engine, source_id, _ledger_key(m1), 1) # not yet dead + + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader) + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + assert result.files_downloaded == 1 + assert _count_failed(sync_engine, source_id) == 0 + + # --- ledger + drift ------------------------------------------------------- -- 2.52.0 From d592e0ca029273041501b447fdc8a0ae74e95ca3 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 00:15:48 -0400 Subject: [PATCH 17/26] =?UTF-8?q?feat(patreon):=20within-pass=20transient?= =?UTF-8?q?=20retry=20for=20media=20GETs=20=E2=80=94=20#8=20(was=20oversta?= =?UTF-8?q?ted=20as=20done)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Honest completion of roadmap #8. Previously a NON-429 media failure (a connection reset, a timeout, a truncated stream, a 5xx) was an immediate terminal "error" for the pass — only retried on the NEXT walk. Now _fetch_to_file retries TRANSIENT failures in-place with backoff (transport blips incl. mid-download, 429 honoring Retry-After, and 5xx; up to 3 tries), while PERMANENT failures (404 gone / 403 forbidden) fail fast straight to the error → dead-letter path — re-fetching them is pointless. This makes the transient-vs-permanent split explicit instead of leaning on the next-tick cycle. (#1's 429 backoff + #7's dead-letter covered most of #8's value; this is the missing in-pass transient piece I'd loosely marked "folded".) Tests: a connection blip / a 5xx is retried then succeeds; a 404 errors with NO retry; an exhausted transient becomes a terminal error. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/patreon_downloader.py | 73 ++++++++++++++++------ tests/test_patreon_downloader.py | 61 +++++++++++++++++- 2 files changed, 111 insertions(+), 23 deletions(-) diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index 9dda0f4..7125a7e 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -43,16 +43,30 @@ from urllib.parse import urlsplit import requests from .file_validator import is_validatable, validate_file -from .patreon_client import _load_session, _retry_after_seconds +from .patreon_client import ( + _BACKOFF_CAP_SECONDS, + _load_session, + _retry_after_seconds, +) log = logging.getLogger(__name__) _TITLE_MAX = 40 _TIMEOUT_SECONDS = 120.0 _CHUNK = 1 << 16 -# A CDN media GET rarely 429s, but if it does, a couple of backoff retries keep -# one throttled file from becoming a per-item error (plan #703). -_MAX_MEDIA_429_RETRIES = 2 +# Retry a media GET that hits a TRANSIENT failure within the same pass (plan +# #705 #8): a transport blip (connection reset / timeout / truncated stream), a +# 429, or a 5xx. PERMANENT failures (404 gone, 403 forbidden) fail fast straight +# to the error/dead-letter path — no point re-fetching them. Keeps a momentary +# network hiccup from becoming a per-item error that waits for the next walk. +_MAX_MEDIA_RETRIES = 3 +# requests transport errors worth retrying (vs. an HTTPError, which is a real +# server response and is classified by status code). +_TRANSIENT_TRANSPORT_EXC = ( + requests.ConnectionError, + requests.Timeout, + requests.exceptions.ChunkedEncodingError, +) # Referer/Origin yt-dlp must send for Mux-hosted Patreon video. Mux's JWT # playback policy checks Referer/Origin on every request, so yt-dlp must send @@ -285,30 +299,49 @@ class PatreonDownloader: return dest def _fetch_to_file(self, url: str, dest: Path) -> None: - """Stream a non-video URL to `dest` via the (stubbable) session. + """Stream a non-video URL to `dest` via the (stubbable) session, retrying + TRANSIENT failures within the same pass (plan #705 #8). - Retries a transient 429 with backoff (plan #703); any other non-2xx - falls through to raise_for_status → HTTPError, which download_post turns - into a resilient per-item error outcome. + Retried (backoff): transport blips (connection reset / timeout / + truncated stream — incl. mid-download), HTTP 429 (honoring Retry-After), + and 5xx. Failed fast (no retry → HTTPError → per-item error → dead-letter + path): 4xx other than 429 (404 gone, 403 forbidden) — re-fetching a + permanent failure is pointless. """ attempt = 0 while True: - resp = self.session.get(url, stream=True, timeout=_TIMEOUT_SECONDS) - if resp.status_code == 429 and attempt < _MAX_MEDIA_429_RETRIES: + try: + resp = self.session.get(url, stream=True, timeout=_TIMEOUT_SECONDS) + if (resp.status_code == 429 or resp.status_code >= 500) \ + and attempt < _MAX_MEDIA_RETRIES: + attempt += 1 + delay = _retry_after_seconds(resp, attempt) + log.warning( + "Patreon media transient HTTP %d (%s) — backing off " + "%.1fs (retry %d/%d)", + resp.status_code, url, delay, attempt, _MAX_MEDIA_RETRIES, + ) + time.sleep(delay) + continue + # 2xx → ok; 4xx-non-429 (or an exhausted 429/5xx) → HTTPError + # (permanent for this pass) → not caught below → per-item error. + resp.raise_for_status() + with open(dest, "wb") as fh: + for chunk in resp.iter_content(chunk_size=_CHUNK): + if chunk: + fh.write(chunk) + return + except _TRANSIENT_TRANSPORT_EXC as exc: + if attempt >= _MAX_MEDIA_RETRIES: + raise # exhausted → terminal error outcome attempt += 1 - delay = _retry_after_seconds(resp, attempt) + delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS) log.warning( - "Patreon media 429 (%s) — backing off %.1fs (retry %d/%d)", - url, delay, attempt, _MAX_MEDIA_429_RETRIES, + "Patreon media transport error (%s) — backing off %.1fs " + "(retry %d/%d): %s", + url, delay, attempt, _MAX_MEDIA_RETRIES, exc, ) time.sleep(delay) - continue - break - resp.raise_for_status() - with open(dest, "wb") as fh: - for chunk in resp.iter_content(chunk_size=_CHUNK): - if chunk: - fh.write(chunk) def _run_ytdlp(self, url: str, dest: Path, headers: dict) -> Path | None: """Invoke yt-dlp to fetch a Mux/HLS stream to (around) `dest`. diff --git a/tests/test_patreon_downloader.py b/tests/test_patreon_downloader.py index ea89752..1d42285 100644 --- a/tests/test_patreon_downloader.py +++ b/tests/test_patreon_downloader.py @@ -56,15 +56,19 @@ class _FakeSession: class _SeqSession: - """Returns the queued responses in order (for retry tests).""" + """Returns the queued items in order (for retry tests). An Exception item is + RAISED by get() instead of returned (simulates a transport error).""" - def __init__(self, responses: list[_FakeResponse]): + def __init__(self, responses): self._responses = list(responses) self.calls: list[str] = [] def get(self, url, stream=False, timeout=None): self.calls.append(url) - return self._responses.pop(0) + item = self._responses.pop(0) + if isinstance(item, Exception): + raise item + return item def _post(post_id="1001", title="My Post Title", published="2026-05-01T12:00:00Z"): @@ -314,3 +318,54 @@ def test_media_429_retried_then_succeeds(tmp_path, monkeypatch): assert outcomes[0].status == "downloaded" assert 4.0 in slept # backed off before the retry assert len(session.calls) == 2 + + +def _dl(tmp_path, session, monkeypatch): + monkeypatch.setattr(pd_mod.time, "sleep", lambda s: None) + return PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, session=session, + ) + + +def test_transient_transport_error_retried_within_pass(tmp_path, monkeypatch): + """plan #705 #8: a connection blip on the GET is retried in-place, not a + terminal error that waits for the next walk.""" + session = _SeqSession([ + requests.ConnectionError("connection reset"), + _FakeResponse(_PNG_BYTES, status_code=200), + ]) + dl = _dl(tmp_path, session, monkeypatch) + outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") + assert outcomes[0].status == "downloaded" + assert len(session.calls) == 2 + + +def test_5xx_retried_within_pass(tmp_path, monkeypatch): + session = _SeqSession([ + _FakeResponse(b"", status_code=503), + _FakeResponse(_PNG_BYTES, status_code=200), + ]) + dl = _dl(tmp_path, session, monkeypatch) + outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") + assert outcomes[0].status == "downloaded" + assert len(session.calls) == 2 + + +def test_404_fails_fast_no_retry(tmp_path, monkeypatch): + """A permanent 4xx (gone/forbidden) is NOT retried — it errors immediately + (and the ingester's dead-letter ledger takes over across walks).""" + session = _SeqSession([ + _FakeResponse(b"", status_code=404), + _FakeResponse(_PNG_BYTES, status_code=200), # would succeed if it retried + ]) + dl = _dl(tmp_path, session, monkeypatch) + outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") + assert outcomes[0].status == "error" + assert len(session.calls) == 1 # no retry on a permanent failure + + +def test_exhausted_transient_becomes_error(tmp_path, monkeypatch): + session = _SeqSession([requests.Timeout("t")] * 10) # always times out + dl = _dl(tmp_path, session, monkeypatch) + outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") + assert outcomes[0].status == "error" -- 2.52.0 From 9a2cd569c39eda044fae231194c9284c39f4f0b6 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 00:24:23 -0400 Subject: [PATCH 18/26] =?UTF-8?q?refactor(ingest):=20extract=20platform-ag?= =?UTF-8?q?nostic=20Ingester=20core=20=E2=80=94=20roadmap=20#9=20(plan=20#?= =?UTF-8?q?706)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Factor the native-ingest orchestration out of PatreonIngester into a reusable ingest_core.Ingester base, parametrized by client/downloader/ledger-models/ constraints/key/platform/error_base. PatreonIngester becomes a thin adapter: it resolves the Patreon client/downloader, wires the seen/dead-letter models + UNIQUE-constraint names + _ledger_key into super().__init__, and overrides _failure_result with the Patreon exception taxonomy. Behavior-preserving — no table rename, no migration; the public surface (PatreonIngester, _ledger_key, DEAD_LETTER_THRESHOLD, verify_patreon_credential) is unchanged. This is the strategic seam: SubscribeStar/etc. now migrate by writing a ~40-line adapter, not by re-implementing the tick/backfill/recovery walk, tiered skip, checkpoint, and dead-letter logic. run() moved to ingest_core, so the budget test's monotonic patch repoints to ingest_core.time. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/ingest_core.py | 459 ++++++++++++++++++++++ backend/app/services/patreon_ingester.py | 464 ++--------------------- tests/test_patreon_ingester.py | 6 +- 3 files changed, 505 insertions(+), 424 deletions(-) create mode 100644 backend/app/services/ingest_core.py diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py new file mode 100644 index 0000000..099f634 --- /dev/null +++ b/backend/app/services/ingest_core.py @@ -0,0 +1,459 @@ +"""Platform-agnostic native-ingest core (plan #706, build on #697/#703/#704/#705). + +The orchestration that drives a native subscription walk — page a feed → +extract media → tiered skip (seen-ledger / on-disk / dead-letter) → download → +mark-seen / record-failures / checkpoint-cursor → return a gallery-dl-shaped +`DownloadResult`, across tick/backfill/recovery modes — is identical for every +platform. Only four things are platform-specific, and they're INJECTED at +construction by a thin adapter (e.g. `PatreonIngester`): + + - `client` — `.iter_posts(feed_id, cursor)` yielding `(post, included, + page_cursor)` + `.extract_media(post, included) -> [media]`. + - `downloader`— `.download_post(post, media, artist_slug, is_seen) -> + [MediaOutcome]` (status in downloaded/skipped_seen/skipped_disk + /quarantined/error; `.path`/`.error`/`.post_id`). + - ledger — `seen_model` + `failed_model` SQLAlchemy models (+ their + on-conflict UNIQUE constraint names) and a `ledger_key(media)`. + - failure map — the adapter overrides `_failure_result` (platform exception + → DownloadResult.error_type) and supplies `error_base` (the + exception type the walk catches) + `platform` (result label). + +Everything DB touches a SHORT-LIVED sync session from the injected sessionmaker — +never held across a network fetch ([[db-connection-held-across-subprocess]]). +Plain-HTTP homelab: no secure-context Web API. +""" + +from __future__ import annotations + +import logging +import time +from collections.abc import Callable + +from sqlalchemy import delete, func, select, text +from sqlalchemy.dialects.postgresql import insert as pg_insert + +from .gallery_dl import DownloadResult, ErrorType + +log = logging.getLogger(__name__) + +# Stop a tick after this many CONTIGUOUS already-have-it media (seen-ledger or +# on-disk) — the cheap native equivalent of gallery-dl's `exit:20`, now free of +# per-file HEADs. Headroom against paywalled/undownloadable items interleaving. +_TICK_SEEN_THRESHOLD = 20 + +# plan #705 #7: after this many failed download/validate attempts a media is +# "dead-lettered" and skipped on routine tick/backfill walks (recovery still +# re-attempts it). Stops a permanently-broken media re-erroring forever. +DEAD_LETTER_THRESHOLD = 3 +# last_error is Text but bound it so a giant traceback doesn't bloat the row. +_ERROR_MAX = 1000 + + +class Ingester: + """Generic native-ingest orchestration. Subclass with a platform adapter + (see the module docstring) — or construct directly with the keyword seams.""" + + def __init__( + self, + *, + client, + downloader, + session_factory: Callable[[], object], + seen_model, + failed_model, + seen_constraint: str, + failed_constraint: str, + ledger_key: Callable[[object], str], + platform: str, + error_base: type[Exception], + ): + self.client = client + self.downloader = downloader + self.session_factory = session_factory + self._seen_model = seen_model + self._failed_model = failed_model + self._seen_constraint = seen_constraint + self._failed_constraint = failed_constraint + self._ledger_key = ledger_key + self._platform = platform + self._error_base = error_base + + # -- public ------------------------------------------------------------ + + def run( + self, + *, + source_id: int, + campaign_id: str, + artist_slug: str, + url: str, + mode: str, + resume_cursor: str | None = None, + time_budget_seconds: float = 870.0, + seen_threshold: int = _TICK_SEEN_THRESHOLD, + ) -> DownloadResult: + """Walk + download for one source, returning a gallery-dl-shaped result. + + `mode` is "tick" | "backfill" | "recovery". Recovery bypasses the tier-1 + seen-ledger AND the dead-letter ledger (tier-2 disk still skips kept + files). The walk stops on: + - budget exhaustion (time_budget_seconds) → TIMEOUT / PARTIAL + - tick early-out (seen_threshold contiguous seen) → success + - reaching the bottom of the feed → success (rc 0) + A client-level failure (drift / auth / network) fails the whole run loud. + """ + bypass_seen = mode == "recovery" + # Only deep walks checkpoint their cursor mid-flight (plan #705 #6); a + # tick has no resumable backfill state. + checkpoint = mode in ("backfill", "recovery") + ledger_key = self._ledger_key + start = time.monotonic() + log_lines: list[str] = [] + written: list[str] = [] + quarantined_paths: list[str] = [] + downloaded = 0 + errors = 0 + quarantined = 0 + dead_lettered = 0 + skipped_count = 0 + posts_processed = 0 + consecutive_seen = 0 + emitted_cursor: str | None = None + reached_bottom = False + budget_hit = False + early_out = False + + def _result( + *, success: bool, return_code: int, + error_type: ErrorType | None, error_message: str | None, + ) -> DownloadResult: + # plan #704: return STRUCTURED data — phase 3 reads run_stats/cursor + # directly instead of regex-scraping a reconstructed stdout. stdout + # stays a human-readable summary (no fake `Cursor:` lines). + return DownloadResult( + success=success, + url=url, + artist_slug=artist_slug, + platform=self._platform, + files_downloaded=downloaded, + files_quarantined=quarantined, + quarantined_paths=list(quarantined_paths), + written_paths=written, + stdout="\n".join(log_lines), + stderr="", + return_code=return_code, + error_type=error_type, + error_message=error_message, + duration_seconds=time.monotonic() - start, + cursor=emitted_cursor, + posts_processed=posts_processed, + run_stats={ + "exit_code": return_code, + "downloaded_count": downloaded, + "skipped_count": skipped_count, + "per_item_failures": errors, + "warning_count": 0, + "tier_gated_count": 0, + "quarantined_count": quarantined, + "dead_lettered_count": dead_lettered, + }, + ) + + try: + for post, included, page_cursor in self.client.iter_posts( + campaign_id, cursor=resume_cursor + ): + # Checkpoint the cursor that FETCHED this page the moment we + # START it — so a chunk cut mid-page resumes the page, not the one + # after it. Carried as DownloadResult.cursor (plan #704). + if page_cursor and page_cursor != emitted_cursor: + emitted_cursor = page_cursor + # plan #705 #6: persist the cursor at each page boundary so a + # worker SIGKILL mid-chunk resumes near the crash, not the + # chunk start. (phase 3 still writes the final cursor — same + # value; this is the crash-safety net.) + if checkpoint: + self._checkpoint_cursor(source_id, emitted_cursor) + + # Time-box check at the post boundary (coarse, like a gallery-dl + # chunk). Backfill/recovery resume from emitted_cursor next chunk. + if time.monotonic() - start >= time_budget_seconds: + budget_hit = True + break + + posts_processed += 1 + media = self.client.extract_media(post, included) + if not media: + continue + + keys = [ledger_key(m) for m in media] + # Recovery bypasses BOTH the seen-ledger AND the dead-letter + # ledger (the operator's "try everything again"); routine walks + # skip seen + dead media (tier-1 + tier-1.5, plan #705 #7). + dead = set() if bypass_seen else self._dead_keys(source_id, keys) + seen = ( + set() + if bypass_seen + else self._seen_keys(source_id, keys) + ) + skip = seen | dead + + def _is_skip(m, _skip=skip) -> bool: + return ledger_key(m) in _skip + + outcomes = self.downloader.download_post( + post, media, artist_slug, is_seen=_is_skip + ) + + to_mark: list[tuple[str, str]] = [] + to_clear: list[str] = [] # recovered → drop any dead-letter row + to_fail: list[tuple[str, str, str]] = [] # (key, post_id, error) + for media_item, outcome in zip(media, outcomes, strict=False): + key = ledger_key(media_item) + if key in dead: + dead_lettered += 1 # skipped because previously dead + if outcome.status == "downloaded": + downloaded += 1 + if outcome.path is not None: + written.append(str(outcome.path)) + to_mark.append((key, media_item.post_id)) + to_clear.append(key) + consecutive_seen = 0 + elif outcome.status == "skipped_disk": + # Already on disk (a prior run). Reconcile the ledger so a + # later tick skips it at tier-1 without a disk stat, but + # do NOT re-feed it to phase 3 — attach_in_place would see + # the duplicate sha256 and unlink the on-disk copy. + to_mark.append((key, media_item.post_id)) + to_clear.append(key) + skipped_count += 1 + consecutive_seen += 1 + elif outcome.status == "skipped_seen": + skipped_count += 1 + consecutive_seen += 1 + elif outcome.status == "quarantined": + # New content that failed validation (corrupt) — counted + # distinctly so the run surfaces a real quarantined total. + # Not marked seen (a later walk may re-fetch a fixed file); + # it IS new content, so it breaks the run-of-seen. Counts + # toward the dead-letter ledger (plan #705 #7). + quarantined += 1 + if outcome.path is not None: + quarantined_paths.append(str(outcome.path)) + to_fail.append((key, media_item.post_id, outcome.error or "quarantined")) + consecutive_seen = 0 + elif outcome.status == "error": + errors += 1 + to_fail.append((key, media_item.post_id, outcome.error or "error")) + # An error neither advances nor resets the run-of-seen. + + if mode == "tick" and consecutive_seen >= seen_threshold: + early_out = True + break + + # Persist ledger changes AFTER the network fetch, on short + # sessions: mark downloaded/on-disk seen, clear any dead-letter + # for recovered media, and record failures (plan #705 #7). + if to_mark: + self._mark_seen(source_id, to_mark) + if to_clear: + self._clear_failures(source_id, to_clear) + if to_fail: + self._record_failures(source_id, to_fail) + + if early_out: + break + else: + reached_bottom = True + except self._error_base as exc: + # The platform's client-error base — _failure_result (adapter) + # maps it to a typed error. + return self._failure_result(exc, _result) + + if errors: + log_lines.append(f"{errors} media item(s) failed") + if quarantined: + log_lines.append(f"{quarantined} media item(s) quarantined (invalid)") + if dead_lettered: + log_lines.append(f"{dead_lettered} media item(s) skipped (dead-lettered)") + log_lines.append( + f"{self._platform} ingest ({mode}): {downloaded} downloaded, " + f"{skipped_count} skipped, {quarantined} quarantined, " + f"{dead_lettered} dead-lettered, {errors} error(s), " + f"{posts_processed} post(s)" + + (", reached end" if reached_bottom else "") + + (", time-boxed" if budget_hit else "") + ) + + if budget_hit: + # A chunk that hit its time-box but made forward progress is a + # NORMAL chunk boundary, not a failure (PARTIAL → status "ok"); the + # next chunk resumes from the emitted cursor. No progress → TIMEOUT, + # which feeds download_service's backfill stall-guard. rc<0 mirrors + # subprocess TimeoutExpired so completion detection stays false. + made_progress = downloaded > 0 or emitted_cursor != resume_cursor + if made_progress: + return _result( + success=False, return_code=-1, + error_type=ErrorType.PARTIAL, + error_message=( + f"Backfill chunk: {downloaded} file(s) — continuing" + ), + ) + return _result( + success=False, return_code=-1, + error_type=ErrorType.TIMEOUT, + error_message="Chunk timed out with no progress", + ) + + # Normal success: reached the bottom, or a tick that early-outed. rc 0 + + # error_type None is REQUIRED for a backfill/recovery walk that reached + # the bottom to be marked COMPLETE by + # download_service._apply_backfill_lifecycle — so we return None even + # when downloaded == 0 (a re-confirming walk that found nothing new still + # completed). success=True maps to status "ok" regardless. A tick that + # early-outed also returns here; ticks never set backfill state so the + # lifecycle is a no-op for them. + return _result( + success=True, return_code=0, + error_type=None, error_message=None, + ) + + # -- failure mapping (adapter overrides) ------------------------------- + + def _failure_result(self, exc: Exception, _result) -> DownloadResult: + """Map a platform client-error to a typed failed DownloadResult. The base + gives a safe default; adapters override with their exception taxonomy.""" + log.warning("%s ingest failed: %s", self._platform, exc) + return _result( + success=False, return_code=1, + error_type=ErrorType.UNKNOWN_ERROR, error_message=str(exc), + ) + + # -- seen-ledger (short-lived sessions) -------------------------------- + + def _seen_keys(self, source_id: int, keys: list[str]) -> set[str]: + """Which of `keys` are already in the seen-ledger for this source. + + One short SELECT on its own session — opened and closed without any + network in between (the GETs happen after, in download_post). + """ + if not keys: + return set() + with self.session_factory() as session: + rows = session.execute( + select(self._seen_model.filehash).where( + self._seen_model.source_id == source_id, + self._seen_model.filehash.in_(keys), + ) + ).scalars().all() + return set(rows) + + def _checkpoint_cursor(self, source_id: int, cursor: str) -> None: + """Persist the in-progress backfill cursor mid-walk (plan #705 #6). + + ATOMIC, single-key UPDATE: cast the JSON column to jsonb, set just + `_backfill_cursor`, cast back — so it never clobbers operator config or + the other backfill keys (no read-modify-write race). The in-flight guard + means only this source's one download runs at a time; a concurrent + operator stop is benign (a stray cursor with no `_backfill_state` is + ignored by tick mode and cleared on the next start). + """ + with self.session_factory() as session: + session.execute( + text( + "UPDATE source SET config_overrides = jsonb_set(" + " coalesce(config_overrides::jsonb, '{}'::jsonb)," + " '{_backfill_cursor}', to_jsonb(cast(:cur AS text))" + ")::json WHERE id = :sid" + ), + {"cur": cursor, "sid": source_id}, + ) + session.commit() + + def _mark_seen(self, source_id: int, items: list[tuple[str, str]]) -> None: + """Idempotent upsert of (filehash, post_id) seen-ledger rows for a page. + + ON CONFLICT DO NOTHING against the (source_id, filehash) UNIQUE so a + re-sighting — or a concurrent walk — is a harmless no-op + ([[scalar_one_or_none-duplicates]]: never check-then-insert without the + DB constraint backing it). De-dup the batch locally first so a single + page can't present the same key twice to one INSERT. + """ + seen_local: set[str] = set() + values = [] + for key, post_id in items: + if key in seen_local: + continue + seen_local.add(key) + values.append( + {"source_id": source_id, "filehash": key, "post_id": post_id} + ) + if not values: + return + with self.session_factory() as session: + stmt = pg_insert(self._seen_model).values(values) + stmt = stmt.on_conflict_do_nothing(constraint=self._seen_constraint) + session.execute(stmt) + session.commit() + + # -- dead-letter ledger (plan #705 #7) --------------------------------- + + def _dead_keys(self, source_id: int, keys: list[str]) -> set[str]: + """Which of `keys` have failed >= DEAD_LETTER_THRESHOLD times (dead). + One short SELECT; recovery never calls this (it re-attempts dead media).""" + if not keys: + return set() + with self.session_factory() as session: + rows = session.execute( + select(self._failed_model.filehash).where( + self._failed_model.source_id == source_id, + self._failed_model.filehash.in_(keys), + self._failed_model.attempts >= DEAD_LETTER_THRESHOLD, + ) + ).scalars().all() + return set(rows) + + def _record_failures( + self, source_id: int, items: list[tuple[str, str, str]] + ) -> None: + """Upsert-increment the dead-letter ledger for failed media. On conflict + bump `attempts` and refresh last_error/last_failed_at (UNIQUE backs the + upsert — no check-then-insert). De-dup the batch (one row/key, last error + wins).""" + by_key: dict[str, str] = {} + for key, _post_id, err in items: + by_key[key] = (err or "")[:_ERROR_MAX] + if not by_key: + return + values = [ + {"source_id": source_id, "filehash": k, "attempts": 1, "last_error": e} + for k, e in by_key.items() + ] + with self.session_factory() as session: + stmt = pg_insert(self._failed_model).values(values) + stmt = stmt.on_conflict_do_update( + constraint=self._failed_constraint, + set_={ + "attempts": self._failed_model.attempts + 1, + "last_error": stmt.excluded.last_error, + "last_failed_at": func.now(), + }, + ) + session.execute(stmt) + session.commit() + + def _clear_failures(self, source_id: int, keys: list[str]) -> None: + """Drop dead-letter rows for media that just downloaded cleanly — they + recovered. A no-op DELETE for keys that were never failing.""" + unique = list(dict.fromkeys(keys)) + if not unique: + return + with self.session_factory() as session: + session.execute( + delete(self._failed_model).where( + self._failed_model.source_id == source_id, + self._failed_model.filehash.in_(unique), + ) + ) + session.commit() diff --git a/backend/app/services/patreon_ingester.py b/backend/app/services/patreon_ingester.py index 206b555..3862088 100644 --- a/backend/app/services/patreon_ingester.py +++ b/backend/app/services/patreon_ingester.py @@ -1,13 +1,14 @@ -"""Native Patreon ingester — phase-2 orchestrator (build step 3). +"""Native Patreon ingester — the Patreon ADAPTER over the platform-agnostic core. -Ties build steps 1 (`patreon_client`) and 2 (`patreon_downloader` + -`patreon_seen_media`) together into a single sync walk that REPLACES the -gallery-dl subprocess for Patreon. `download_service.download_source` calls -`PatreonIngester.run(...)` from phase 2 (in a thread, via run_in_executor, since -everything here is sync `requests`/`subprocess`) and gets back a -`DownloadResult` — the exact shape gallery-dl returns — so phase 1 (DB setup) -and phase 3 (import → pHash dedup → thumbnails → ML) are untouched and cannot -tell the difference. +The orchestration that drives a native subscription walk (page a feed → extract +media → tiered skip → download → mark-seen / record-failures / checkpoint-cursor +→ return a gallery-dl-shaped `DownloadResult`, across tick/backfill/recovery) +now lives in `ingest_core.Ingester` — it's identical for every platform. This +module is the thin Patreon adapter: it wires the Patreon `client`/`downloader`/ +ledger models/constraints/key into the core and supplies the Patreon-specific +failure mapping. `download_service.download_source` calls `PatreonIngester.run` +exactly as before; the public surface (this class, `_ledger_key`, +`DEAD_LETTER_THRESHOLD`, `verify_patreon_credential`) is unchanged. Three modes (selected by `download_service` from `config_overrides` state): - tick — newest→oldest, skip seen (tier-1 ledger + tier-2 disk), early-out @@ -15,24 +16,14 @@ Three modes (selected by `download_service` from `config_overrides` state): equivalent of gallery-dl's `exit:20`, now free of per-file HEADs). - backfill — full-history walk in a time-boxed chunk, resuming from the pagination cursor checkpoint; reaches the bottom → "complete". - - recovery — like backfill but BYPASSES the tier-1 seen-ledger, so - deliberately-dropped-and-deleted near-dups get re-fetched and - re-evaluated under the current pHash threshold (tier-2 disk skip - still spares files we kept). Triggered by the same #693 backfill - state machine plus the `_backfill_bypass_seen` flag, so the whole - cursor/chunk/complete/stall lifecycle is reused verbatim. - -Cursor contract: the ingester emits gallery-dl-style ``Cursor: `` lines -into the returned `stdout`, one per page it walks. That is exactly what -`download_service`'s existing backfill lifecycle reads via -`parse_last_cursor(stdout, stderr)` — so checkpointing, the TIMEOUT→PARTIAL -reclassification, and completion detection all keep working unchanged. - -The seen-ledger lives in Postgres (`patreon_seen_media`). The ingester opens a -SHORT-LIVED sync session per page batch (via an injected sessionmaker) — never -held across a network fetch — so the multi-minute walk can't strand a checked-out -connection for the server to reap ([[db-connection-held-across-subprocess]]). + - recovery — like backfill but BYPASSES the tier-1 seen-ledger AND the + dead-letter ledger, so deliberately-dropped-and-deleted near-dups + get re-fetched and re-evaluated under the current pHash threshold + (tier-2 disk skip still spares files we kept). +The seen/dead-letter ledgers live in Postgres (`patreon_seen_media` / +`patreon_failed_media`); the core opens SHORT-LIVED sync sessions per page batch +— never held across a network fetch ([[db-connection-held-across-subprocess]]). FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. """ @@ -40,15 +31,12 @@ from __future__ import annotations import asyncio import logging -import time from collections.abc import Callable from pathlib import Path -from sqlalchemy import delete, func, select, text -from sqlalchemy.dialects.postgresql import insert as pg_insert - from ..models import PatreonFailedMedia, PatreonSeenMedia from .gallery_dl import DownloadResult, ErrorType +from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester from .patreon_client import ( MediaItem, PatreonAPIError, @@ -59,27 +47,19 @@ from .patreon_client import ( from .patreon_downloader import PatreonDownloader from .patreon_resolver import resolve_campaign_id_for_source -log = logging.getLogger(__name__) +__all__ = [ + "DEAD_LETTER_THRESHOLD", + "PatreonIngester", + "_ledger_key", + "verify_patreon_credential", +] -# gallery-dl's `exit:20` default ported over: stop a tick after this many -# CONTIGUOUS already-have-it media (seen-ledger or on-disk). Native walks have -# zero per-file HEADs, so the only cost of a higher number is a few extra cheap -# ledger lookups — 20 is operator-set headroom against paywalled/undownloadable -# items interleaving with archived ones. -_TICK_SEEN_THRESHOLD = 20 +log = logging.getLogger(__name__) # Ledger keys are stored in patreon_seen_media.filehash VARCHAR(128); bound any # synthesized key so a pathologically long file_name can't overflow the column. _LEDGER_KEY_MAX = 128 -# plan #705 #7: after this many failed download/validate attempts a media is -# "dead-lettered" and skipped on routine tick/backfill walks (recovery still -# re-attempts it). Stops a permanently-broken media (404'd CDN, deleted post) -# re-erroring forever and re-burning chunks. -DEAD_LETTER_THRESHOLD = 3 -# last_error is Text but bound it so a giant traceback doesn't bloat the row. -_ERROR_MAX = 1000 - def _ledger_key(media: MediaItem) -> str: """Stable per-media identity for the cross-run seen-ledger. @@ -96,12 +76,12 @@ def _ledger_key(media: MediaItem) -> str: return f"{media.post_id}:{media.filename}"[:_LEDGER_KEY_MAX] -class PatreonIngester: +class PatreonIngester(Ingester): """Walk a Patreon campaign's posts, download unseen media, return a - `DownloadResult`. + `DownloadResult`. A thin adapter over `ingest_core.Ingester`. Construct with the per-source `cookies_path` and a sync sessionmaker for the - seen-ledger. `client` / `downloader` are injectable seams so unit tests run + ledgers. `client` / `downloader` are injectable seams so unit tests run without network, subprocess, or a real CDN. """ @@ -119,266 +99,36 @@ class PatreonIngester: ): self.images_root = Path(images_root) self.cookies_path = str(cookies_path) if cookies_path else None - self.session_factory = session_factory # Pacing (plan #703): request_sleep paces the API page fetches, # rate_limit paces the media downloads. Injected client/downloader (in # tests) already carry their own pacing, so these only apply to the # default-constructed ones. - self.client = ( + resolved_client = ( client if client is not None else PatreonClient(cookies_path, request_sleep=request_sleep) ) - self.downloader = ( + resolved_downloader = ( downloader if downloader is not None else PatreonDownloader( self.images_root, cookies_path, validate=validate, rate_limit=rate_limit ) ) - - # -- public ------------------------------------------------------------ - - def run( - self, - *, - source_id: int, - campaign_id: str, - artist_slug: str, - url: str, - mode: str, - resume_cursor: str | None = None, - time_budget_seconds: float = 870.0, - seen_threshold: int = _TICK_SEEN_THRESHOLD, - ) -> DownloadResult: - """Walk + download for one source, returning a gallery-dl-shaped result. - - `mode` is "tick" | "backfill" | "recovery". Recovery bypasses the tier-1 - seen-ledger (tier-2 disk still skips kept files). The walk stops on: - - budget exhaustion (time_budget_seconds) → TIMEOUT / PARTIAL - - tick early-out (seen_threshold contiguous seen) → success - - reaching the bottom of the feed → success (rc 0) - A client-level failure (drift / auth / network) fails the whole run loud. - """ - bypass_seen = mode == "recovery" - # Only deep walks checkpoint their cursor mid-flight (plan #705 #6); a - # tick has no resumable backfill state. - checkpoint = mode in ("backfill", "recovery") - start = time.monotonic() - log_lines: list[str] = [] - written: list[str] = [] - quarantined_paths: list[str] = [] - downloaded = 0 - errors = 0 - quarantined = 0 - dead_lettered = 0 - skipped_count = 0 - posts_processed = 0 - consecutive_seen = 0 - emitted_cursor: str | None = None - reached_bottom = False - budget_hit = False - early_out = False - - def _result( - *, success: bool, return_code: int, - error_type: ErrorType | None, error_message: str | None, - ) -> DownloadResult: - # plan #704: return STRUCTURED data — phase 3 reads run_stats/cursor - # directly instead of regex-scraping a reconstructed stdout. stdout - # stays a human-readable summary (no fake `Cursor:` lines). - return DownloadResult( - success=success, - url=url, - artist_slug=artist_slug, - platform="patreon", - files_downloaded=downloaded, - files_quarantined=quarantined, - quarantined_paths=list(quarantined_paths), - written_paths=written, - stdout="\n".join(log_lines), - stderr="", - return_code=return_code, - error_type=error_type, - error_message=error_message, - duration_seconds=time.monotonic() - start, - cursor=emitted_cursor, - posts_processed=posts_processed, - run_stats={ - "exit_code": return_code, - "downloaded_count": downloaded, - "skipped_count": skipped_count, - "per_item_failures": errors, - "warning_count": 0, - "tier_gated_count": 0, - "quarantined_count": quarantined, - "dead_lettered_count": dead_lettered, - }, - ) - - try: - for post, included, page_cursor in self.client.iter_posts( - campaign_id, cursor=resume_cursor - ): - # Checkpoint the cursor that FETCHED this page the moment we - # START it — so a chunk cut mid-page resumes the page, not the one - # after it. Carried as DownloadResult.cursor (plan #704); no fake - # `Cursor:` stdout line to regex back out. - if page_cursor and page_cursor != emitted_cursor: - emitted_cursor = page_cursor - # plan #705 #6: persist the cursor at each page boundary so a - # worker SIGKILL mid-chunk resumes near the crash, not the - # chunk start. (phase 3 still writes the final cursor — same - # value; this is the crash-safety net.) - if checkpoint: - self._checkpoint_cursor(source_id, emitted_cursor) - - # Time-box check at the post boundary (coarse, like a gallery-dl - # chunk). Backfill/recovery resume from emitted_cursor next chunk. - if time.monotonic() - start >= time_budget_seconds: - budget_hit = True - break - - posts_processed += 1 - media = self.client.extract_media(post, included) - if not media: - continue - - keys = [_ledger_key(m) for m in media] - # Recovery bypasses BOTH the seen-ledger AND the dead-letter - # ledger (the operator's "try everything again"); routine walks - # skip seen + dead media (tier-1 + tier-1.5, plan #705 #7). - dead = set() if bypass_seen else self._dead_keys(source_id, keys) - seen = ( - set() - if bypass_seen - else self._seen_keys(source_id, keys) - ) - skip = seen | dead - - def _is_skip(m: MediaItem, _skip=skip) -> bool: - return _ledger_key(m) in _skip - - outcomes = self.downloader.download_post( - post, media, artist_slug, is_seen=_is_skip - ) - - to_mark: list[tuple[str, str]] = [] - to_clear: list[str] = [] # recovered → drop any dead-letter row - to_fail: list[tuple[str, str, str]] = [] # (key, post_id, error) - for media_item, outcome in zip(media, outcomes, strict=False): - key = _ledger_key(media_item) - if key in dead: - dead_lettered += 1 # skipped because previously dead - if outcome.status == "downloaded": - downloaded += 1 - if outcome.path is not None: - written.append(str(outcome.path)) - to_mark.append((key, media_item.post_id)) - to_clear.append(key) - consecutive_seen = 0 - elif outcome.status == "skipped_disk": - # Already on disk (a prior run). Reconcile the ledger so a - # later tick skips it at tier-1 without a disk stat, but - # do NOT re-feed it to phase 3 — attach_in_place would see - # the duplicate sha256 and unlink the on-disk copy. - to_mark.append((key, media_item.post_id)) - to_clear.append(key) - skipped_count += 1 - consecutive_seen += 1 - elif outcome.status == "skipped_seen": - skipped_count += 1 - consecutive_seen += 1 - elif outcome.status == "quarantined": - # New content that failed validation (corrupt) — counted - # distinctly so the run surfaces a real quarantined total. - # Not marked seen (a later walk may re-fetch a fixed file); - # it IS new content, so it breaks the run-of-seen. Counts - # toward the dead-letter ledger (plan #705 #7). - quarantined += 1 - if outcome.path is not None: - quarantined_paths.append(str(outcome.path)) - to_fail.append((key, media_item.post_id, outcome.error or "quarantined")) - consecutive_seen = 0 - elif outcome.status == "error": - errors += 1 - to_fail.append((key, media_item.post_id, outcome.error or "error")) - # An error neither advances nor resets the run-of-seen. - - if mode == "tick" and consecutive_seen >= seen_threshold: - early_out = True - break - - # Persist ledger changes AFTER the network fetch, on short - # sessions: mark downloaded/on-disk seen, clear any dead-letter - # for recovered media, and record failures (plan #705 #7). - if to_mark: - self._mark_seen(source_id, to_mark) - if to_clear: - self._clear_failures(source_id, to_clear) - if to_fail: - self._record_failures(source_id, to_fail) - - if early_out: - break - else: - reached_bottom = True - except PatreonAPIError as exc: - # Base of PatreonAuthError + PatreonDriftError — catches every - # client-level failure; _failure_result maps it to a typed error. - return self._failure_result(exc, _result) - - if errors: - log_lines.append(f"{errors} media item(s) failed") - if quarantined: - log_lines.append(f"{quarantined} media item(s) quarantined (invalid)") - if dead_lettered: - log_lines.append(f"{dead_lettered} media item(s) skipped (dead-lettered)") - log_lines.append( - f"Patreon ingest ({mode}): {downloaded} downloaded, " - f"{skipped_count} skipped, {quarantined} quarantined, " - f"{dead_lettered} dead-lettered, {errors} error(s), " - f"{posts_processed} post(s)" - + (", reached end" if reached_bottom else "") - + (", time-boxed" if budget_hit else "") + super().__init__( + client=resolved_client, + downloader=resolved_downloader, + session_factory=session_factory, + seen_model=PatreonSeenMedia, + failed_model=PatreonFailedMedia, + seen_constraint="uq_patreon_seen_media_source_id", + failed_constraint="uq_patreon_failed_media_source_id", + ledger_key=_ledger_key, + platform="patreon", + error_base=PatreonAPIError, ) - if budget_hit: - # A chunk that hit its time-box but made forward progress is a - # NORMAL chunk boundary, not a failure (PARTIAL → status "ok"); the - # next chunk resumes from the emitted cursor. No progress → TIMEOUT, - # which feeds download_service's backfill stall-guard. rc<0 mirrors - # subprocess TimeoutExpired so completion detection stays false. - made_progress = downloaded > 0 or emitted_cursor != resume_cursor - if made_progress: - return _result( - success=False, return_code=-1, - error_type=ErrorType.PARTIAL, - error_message=( - f"Patreon backfill chunk: {downloaded} file(s) — continuing" - ), - ) - return _result( - success=False, return_code=-1, - error_type=ErrorType.TIMEOUT, - error_message="Patreon chunk timed out with no progress", - ) - - # Normal success: reached the bottom, or a tick that early-outed. rc 0 + - # error_type None is REQUIRED for a backfill/recovery walk that reached - # the bottom to be marked COMPLETE by - # download_service._apply_backfill_lifecycle — so we return None even - # when downloaded == 0 (a re-confirming walk that found nothing new still - # completed). NO_NEW_CONTENT would read as "not finished" there and trip - # the stall guard. success=True maps to status "ok" regardless. A tick - # that early-outed also returns here; ticks never set backfill state so - # the lifecycle is a no-op for them. - return _result( - success=True, return_code=0, - error_type=None, error_message=None, - ) - - # -- failure mapping --------------------------------------------------- + # -- failure mapping (Patreon exception taxonomy) ---------------------- def _failure_result(self, exc: Exception, _result) -> DownloadResult: """Map a client-level exception to a loud, typed failed DownloadResult. @@ -417,136 +167,6 @@ class PatreonIngester: error_type=error_type, error_message=message, ) - # -- seen-ledger (short-lived sessions) -------------------------------- - - def _seen_keys(self, source_id: int, keys: list[str]) -> set[str]: - """Which of `keys` are already in the ledger for this source. - - One short SELECT on its own session — opened and closed without any - network in between (the GETs happen after, in download_post). - """ - if not keys: - return set() - with self.session_factory() as session: - rows = session.execute( - select(PatreonSeenMedia.filehash).where( - PatreonSeenMedia.source_id == source_id, - PatreonSeenMedia.filehash.in_(keys), - ) - ).scalars().all() - return set(rows) - - def _checkpoint_cursor(self, source_id: int, cursor: str) -> None: - """Persist the in-progress backfill cursor mid-walk (plan #705 #6). - - ATOMIC, single-key UPDATE: cast the JSON column to jsonb, set just - `_backfill_cursor`, cast back — so it never clobbers operator config or - the other backfill keys (no read-modify-write race). The in-flight guard - means only this source's one download runs at a time; a concurrent - operator stop is benign (a stray cursor with no `_backfill_state` is - ignored by tick mode and cleared on the next start). - """ - with self.session_factory() as session: - session.execute( - text( - "UPDATE source SET config_overrides = jsonb_set(" - " coalesce(config_overrides::jsonb, '{}'::jsonb)," - " '{_backfill_cursor}', to_jsonb(cast(:cur AS text))" - ")::json WHERE id = :sid" - ), - {"cur": cursor, "sid": source_id}, - ) - session.commit() - - def _mark_seen(self, source_id: int, items: list[tuple[str, str]]) -> None: - """Idempotent upsert of (filehash, post_id) ledger rows for a page. - - ON CONFLICT DO NOTHING against the (source_id, filehash) UNIQUE so a - re-sighting — or a concurrent walk — is a harmless no-op - ([[scalar_one_or_none-duplicates]]: never check-then-insert without the - DB constraint backing it). De-dup the batch locally first so a single - page can't present the same key twice to one INSERT. - """ - seen_local: set[str] = set() - values = [] - for key, post_id in items: - if key in seen_local: - continue - seen_local.add(key) - values.append( - {"source_id": source_id, "filehash": key, "post_id": post_id} - ) - if not values: - return - with self.session_factory() as session: - stmt = pg_insert(PatreonSeenMedia).values(values) - stmt = stmt.on_conflict_do_nothing( - constraint="uq_patreon_seen_media_source_id" - ) - session.execute(stmt) - session.commit() - - # -- dead-letter ledger (plan #705 #7) --------------------------------- - - def _dead_keys(self, source_id: int, keys: list[str]) -> set[str]: - """Which of `keys` have failed >= DEAD_LETTER_THRESHOLD times (dead). - One short SELECT; recovery never calls this (it re-attempts dead media).""" - if not keys: - return set() - with self.session_factory() as session: - rows = session.execute( - select(PatreonFailedMedia.filehash).where( - PatreonFailedMedia.source_id == source_id, - PatreonFailedMedia.filehash.in_(keys), - PatreonFailedMedia.attempts >= DEAD_LETTER_THRESHOLD, - ) - ).scalars().all() - return set(rows) - - def _record_failures( - self, source_id: int, items: list[tuple[str, str, str]] - ) -> None: - """Upsert-increment the dead-letter ledger for failed media. On conflict - bump `attempts` and refresh last_error/last_failed_at (UNIQUE backs the - upsert — no check-then-insert, [[scalar_one_or_none-duplicates]]). De-dup - the batch locally (one row per key, last error wins).""" - by_key: dict[str, str] = {} - for key, _post_id, err in items: - by_key[key] = (err or "")[:_ERROR_MAX] - if not by_key: - return - values = [ - {"source_id": source_id, "filehash": k, "attempts": 1, "last_error": e} - for k, e in by_key.items() - ] - with self.session_factory() as session: - stmt = pg_insert(PatreonFailedMedia).values(values) - stmt = stmt.on_conflict_do_update( - constraint="uq_patreon_failed_media_source_id", - set_={ - "attempts": PatreonFailedMedia.attempts + 1, - "last_error": stmt.excluded.last_error, - "last_failed_at": func.now(), - }, - ) - session.execute(stmt) - session.commit() - - def _clear_failures(self, source_id: int, keys: list[str]) -> None: - """Drop dead-letter rows for media that just downloaded cleanly — they - recovered. A no-op DELETE for keys that were never failing.""" - unique = list(dict.fromkeys(keys)) - if not unique: - return - with self.session_factory() as session: - session.execute( - delete(PatreonFailedMedia).where( - PatreonFailedMedia.source_id == source_id, - PatreonFailedMedia.filehash.in_(unique), - ) - ) - session.commit() - async def verify_patreon_credential( url: str, diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index 0b3df18..32db179 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -300,7 +300,9 @@ async def test_backfill_budget_cut_returns_partial_with_progress( source_id, sync_engine, tmp_path, monkeypatch, ): # Deterministic clock: start=0, post1 check=10 (ok), post2 check=200 (>budget). - import backend.app.services.patreon_ingester as mod + # run() lives in ingest_core now, so patch the clock there (the same global + # time module object, but we reference it through the module that uses it). + import backend.app.services.ingest_core as core ticks = iter([0.0, 10.0, 200.0, 250.0]) last = [0.0] @@ -311,7 +313,7 @@ async def test_backfill_budget_cut_returns_partial_with_progress( pass return last[0] - monkeypatch.setattr(mod.time, "monotonic", fake_monotonic) + monkeypatch.setattr(core.time, "monotonic", fake_monotonic) pages = [ ("CUR1", [("p1", [_media("p1", 1)])]), -- 2.52.0 From 697a86d31c88befe05bcc9846c09d496eb7cbd43 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 09:56:26 -0400 Subject: [PATCH 19/26] fix(ingester): close #5 within-chunk live posts + #8 video transient retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the #1–#9 ingester roadmap found two real-but-small gaps; this closes both. #5 (live posts progress) shipped at per-chunk granularity — _apply_backfill_ lifecycle accumulated DownloadResult.posts_processed AFTER each chunk, so the badge didn't move during a chunk (up to ~14.5 min) and over-counted the re-walked resume page. The plan called for within-chunk live updates. Move ownership of _backfill_posts into the ingester: ingest_core writes a monotonic absolute (posts_base + net-new) via _checkpoint_posts at each page boundary and once at the end, EXCLUDING the resumed page so it no longer inflates across chunks. download_service seeds posts_base from prior chunks and stops touching the key (the lifecycle now carries the ingester's committed value forward). #8 (per-media transient/permanent retry) covered only the plain-GET path (_fetch_to_file); the Mux/video path returned None on any yt-dlp failure with no retry. Give _run_ytdlp the same split: TimeoutExpired/OSError are transient (back off + retry up to _MAX_MEDIA_RETRIES), a non-zero exit (CalledProcessError) is permanent (yt-dlp already did its own network retries) → fail fast to the per-item/dead-letter path. Tests: live-posts absolute + resume-page exclusion + tick-doesn't-persist (test_patreon_ingester); lifecycle-leaves-posts-to-ingester rewrite (test_download_service); video transient-retry + permanent-fail-fast (test_patreon_downloader). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/download_service.py | 16 ++++--- backend/app/services/ingest_core.py | 43 +++++++++++++++++- backend/app/services/patreon_downloader.py | 52 +++++++++++++++++----- tests/test_download_service.py | 13 +++--- tests/test_patreon_downloader.py | 51 +++++++++++++++++++++ tests/test_patreon_ingester.py | 44 ++++++++++++++++++ 6 files changed, 194 insertions(+), 25 deletions(-) diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 47cd4c0..390d854 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -238,6 +238,9 @@ class DownloadService: mode=mode, resume_cursor=source_config.resume_cursor, time_budget_seconds=source_config.timeout, + # plan #704 #5: seed the live posts badge from prior chunks so the + # ingester writes a monotonic absolute mid-walk (it owns the key). + posts_base=int(overrides.get("_backfill_posts", 0)), ), ) return dl_result, resolved_campaign_id @@ -515,13 +518,12 @@ class DownloadService: new_overrides = dict(src.config_overrides or {}) chunks = int(new_overrides.get("_backfill_chunks", 0)) + 1 new_overrides["_backfill_chunks"] = chunks - # plan #704 (#5): accumulate posts-processed across chunks so the badge - # shows live walk progress, not just the chunk count. Cleared on - # (re)start. (Slight over-count: each chunk re-walks the resumed page — - # fine for a progress indicator.) - new_overrides["_backfill_posts"] = int( - new_overrides.get("_backfill_posts", 0) - ) + int(getattr(dl_result, "posts_processed", 0) or 0) + # plan #704 (#5): _backfill_posts (the live progress badge) is OWNED by the + # ingester now — it writes a monotonic absolute mid-walk at each page + # boundary (ingest_core._checkpoint_posts), so the badge climbs DURING a + # chunk instead of jumping once per chunk here, and the re-walked resume + # page is no longer double-counted. new_overrides (read fresh above) + # carries the ingester's committed value forward untouched. completed = ( dl_result.success diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 099f634..0e6fb5d 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -91,6 +91,7 @@ class Ingester: resume_cursor: str | None = None, time_budget_seconds: float = 870.0, seen_threshold: int = _TICK_SEEN_THRESHOLD, + posts_base: int = 0, ) -> DownloadResult: """Walk + download for one source, returning a gallery-dl-shaped result. @@ -117,6 +118,11 @@ class Ingester: dead_lettered = 0 skipped_count = 0 posts_processed = 0 + # Net-new posts THIS chunk for the live progress badge (plan #704 #5); + # excludes the re-walked resume page so _backfill_posts stays a monotonic + # absolute across chunks instead of an inflating sum. posts_processed + # stays the gross per-chunk count used for the run summary. + chunk_new_posts = 0 consecutive_seen = 0 emitted_cursor: str | None = None reached_bottom = False @@ -171,9 +177,12 @@ class Ingester: # plan #705 #6: persist the cursor at each page boundary so a # worker SIGKILL mid-chunk resumes near the crash, not the # chunk start. (phase 3 still writes the final cursor — same - # value; this is the crash-safety net.) + # value; this is the crash-safety net.) plan #704 #5: persist + # the live posts count alongside it so the badge climbs DURING + # the chunk, not only when it ends. if checkpoint: self._checkpoint_cursor(source_id, emitted_cursor) + self._checkpoint_posts(source_id, posts_base + chunk_new_posts) # Time-box check at the post boundary (coarse, like a gallery-dl # chunk). Backfill/recovery resume from emitted_cursor next chunk. @@ -182,6 +191,12 @@ class Ingester: break posts_processed += 1 + # The resume page (its cursor == resume_cursor) was already + # counted by the chunk that checkpointed it — don't re-count it + # into the persisted badge (plan #704 #5). First chunk has + # resume_cursor None, so everything counts. + if not (resume_cursor and page_cursor == resume_cursor): + chunk_new_posts += 1 media = self.client.extract_media(post, included) if not media: continue @@ -270,6 +285,11 @@ class Ingester: # maps it to a typed error. return self._failure_result(exc, _result) + # Final authoritative posts count for the badge — captures the last page + # after the last boundary write and the time-box break (plan #704 #5). + if checkpoint: + self._checkpoint_posts(source_id, posts_base + chunk_new_posts) + if errors: log_lines.append(f"{errors} media item(s) failed") if quarantined: @@ -371,6 +391,27 @@ class Ingester: ) session.commit() + def _checkpoint_posts(self, source_id: int, posts: int) -> None: + """Persist the live backfill posts-processed count mid-walk (plan #704 #5). + + Same atomic single-key jsonb_set dance as _checkpoint_cursor, on the + `_backfill_posts` key (cast to a JSON number) — so the progress badge + climbs DURING a chunk without clobbering operator config or the cursor. + The ingester OWNS this key now; download_service no longer accumulates it + post-chunk (which lagged a whole chunk and over-counted the resume page). + """ + with self.session_factory() as session: + session.execute( + text( + "UPDATE source SET config_overrides = jsonb_set(" + " coalesce(config_overrides::jsonb, '{}'::jsonb)," + " '{_backfill_posts}', to_jsonb(cast(:posts AS int))" + ")::json WHERE id = :sid" + ), + {"posts": posts, "sid": source_id}, + ) + session.commit() + def _mark_seen(self, source_id: int, items: list[tuple[str, str]]) -> None: """Idempotent upsert of (filehash, post_id) seen-ledger rows for a page. diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index 7125a7e..ad001c8 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -353,6 +353,14 @@ class PatreonDownloader: The output template uses `dest` without its extension; yt-dlp appends the chosen container extension. We pass Referer/Origin (Mux JWT policy) and the cookies file. + + Mirrors the GET path's transient/permanent split (plan #705 #8): a hung + fetch (TimeoutExpired) or a spawn failure (OSError) is TRANSIENT — back + off and retry. A non-zero yt-dlp exit (CalledProcessError) is treated as + PERMANENT for this pass — yt-dlp already does its OWN internal network + retries, so a non-zero exit is effectively a real failure (private/gone/ + geo-blocked), like a 4xx on the GET path: fail fast to the per-item error + → dead-letter path. """ dest = Path(dest) out_template = str(dest.with_suffix("")) + ".%(ext)s" @@ -362,17 +370,39 @@ class PatreonDownloader: if self.cookies_path and os.path.isfile(self.cookies_path): cmd += ["--cookies", self.cookies_path] cmd.append(url) - try: - subprocess.run( - cmd, - check=True, - capture_output=True, - text=True, - timeout=_TIMEOUT_SECONDS, - ) - except (OSError, subprocess.SubprocessError) as exc: - log.warning("yt-dlp failed for %s: %s", url, exc) - return None + attempt = 0 + while True: + try: + subprocess.run( + cmd, + check=True, + capture_output=True, + text=True, + timeout=_TIMEOUT_SECONDS, + ) + break + except subprocess.CalledProcessError as exc: + # Permanent for this pass — fail fast (no retry). + log.warning( + "yt-dlp failed (exit %s) for %s: %s", + exc.returncode, url, (exc.stderr or "").strip() or exc, + ) + return None + except (OSError, subprocess.TimeoutExpired) as exc: + # Transient — back off and retry, like a transport blip on a GET. + if attempt >= _MAX_MEDIA_RETRIES: + log.warning( + "yt-dlp transient failure exhausted for %s: %s", url, exc + ) + return None + attempt += 1 + delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS) + log.warning( + "yt-dlp transient failure (%s) — backing off %.1fs " + "(retry %d/%d): %s", + url, delay, attempt, _MAX_MEDIA_RETRIES, exc, + ) + time.sleep(delay) return self._existing_video_output(dest) def _existing_video_output(self, dest: Path) -> Path | None: diff --git a/tests/test_download_service.py b/tests/test_download_service.py index 253c29e..d08eb12 100644 --- a/tests/test_download_service.py +++ b/tests/test_download_service.py @@ -649,17 +649,18 @@ async def test_backfill_stuck_guard_stalls_after_two_no_advance( @pytest.mark.asyncio -async def test_backfill_accumulates_posts_processed( +async def test_backfill_lifecycle_leaves_posts_to_ingester( db, db_sync, tmp_path, seed_artist_and_source, ): - """plan #704 (#5): the lifecycle accumulates DownloadResult.posts_processed - into _backfill_posts across chunks for the live progress badge.""" + """plan #704 (#5), revised: _backfill_posts is OWNED by the ingester now (it + writes a monotonic absolute live mid-walk), so the post-chunk lifecycle must + NOT touch it — it carries the ingester's committed value forward unchanged.""" from unittest.mock import MagicMock from backend.app.services.download_service import DownloadService _artist, source = seed_artist_and_source - source.config_overrides = {"_backfill_state": "running", "_backfill_posts": 5} + source.config_overrides = {"_backfill_state": "running", "_backfill_posts": 12} source.backfill_runs_remaining = 5 await db.commit() @@ -669,7 +670,7 @@ async def test_backfill_accumulates_posts_processed( ) ctx = { "source_id": source.id, "platform": "patreon", - "config_overrides": {"_backfill_state": "running", "_backfill_posts": 5}, + "config_overrides": {"_backfill_state": "running", "_backfill_posts": 12}, "backfill_runs_remaining": 5, } await svc._apply_backfill_lifecycle( @@ -679,7 +680,7 @@ async def test_backfill_accumulates_posts_processed( co = (await db.execute( select(Source.config_overrides).where(Source.id == source.id) )).scalar_one() - assert co.get("_backfill_posts") == 15 # 5 (prior) + 10 (this chunk) + assert co.get("_backfill_posts") == 12 # untouched — the ingester owns it @pytest.mark.asyncio diff --git a/tests/test_patreon_downloader.py b/tests/test_patreon_downloader.py index 1d42285..e606489 100644 --- a/tests/test_patreon_downloader.py +++ b/tests/test_patreon_downloader.py @@ -216,6 +216,57 @@ def test_video_routes_to_ytdlp(tmp_path, monkeypatch): assert find_sidecar(outcomes[0].path) is not None +def _video_item(): + return MediaItem( + url="https://stream.mux.com/abc123.m3u8", filename="clip.mp4", + kind="postfile", filehash=None, post_id="1001", + ) + + +def test_video_transient_failure_retried_then_succeeds(tmp_path, monkeypatch): + """plan #705 #8: a TRANSIENT yt-dlp failure (TimeoutExpired) is retried within + the pass; a later success yields a downloaded outcome. (The GET path already + retried transients; the video path didn't until #8 closed the gap.)""" + calls = {"n": 0} + + def fake_run(cmd, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + raise pd_mod.subprocess.TimeoutExpired(cmd, kwargs.get("timeout")) + out = Path(cmd[cmd.index("-o") + 1].replace(".%(ext)s", ".mp4")) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_bytes(b"video-bytes") + return pd_mod.subprocess.CompletedProcess(cmd, 0, "", "") + + monkeypatch.setattr(pd_mod.subprocess, "run", fake_run) + monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a, **_k: None) + + dl = _downloader(tmp_path, session=_FakeSession()) + outcomes = dl.download_post(_post(), [_video_item()], "artist-x") + assert calls["n"] == 2 # retried once after the transient blip + assert outcomes[0].status == "downloaded" + assert outcomes[0].path.read_bytes() == b"video-bytes" + + +def test_video_permanent_failure_fails_fast(tmp_path, monkeypatch): + """plan #705 #8: a non-zero yt-dlp exit (CalledProcessError) is PERMANENT — + yt-dlp already did its own network retries — so we don't re-spawn; the item + errors straight to the per-item/dead-letter path.""" + calls = {"n": 0} + + def fake_run(cmd, **kwargs): + calls["n"] += 1 + raise pd_mod.subprocess.CalledProcessError(1, cmd, stderr="ERROR: gone") + + monkeypatch.setattr(pd_mod.subprocess, "run", fake_run) + monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a, **_k: None) + + dl = _downloader(tmp_path, session=_FakeSession()) + outcomes = dl.download_post(_post(), [_video_item()], "artist-x") + assert calls["n"] == 1 # no retry on a permanent failure + assert outcomes[0].status == "error" + + def test_one_failure_isolated(tmp_path): class _BoomSession(_FakeSession): def get(self, url, stream=False, timeout=None): diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index 32db179..3ca19a5 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -277,6 +277,50 @@ async def test_backfill_checkpoints_cursor_mid_walk(source_id, sync_engine, tmp_ assert co.get("_backfill_cursor") == "CUR3" +@pytest.mark.asyncio +async def test_backfill_persists_live_posts_count(source_id, sync_engine, tmp_path, db): + """plan #704 #5: the ingester writes a monotonic _backfill_posts absolute + DURING the walk (seeded from prior chunks via posts_base), EXCLUDING the + re-walked resume page so the count doesn't inflate across chunks.""" + # Resume from CUR2 (a prior chunk left off here): its page is re-walked and + # must NOT be re-counted. posts_base=4 stands in for the prior chunks. + pages = [ + ("CUR2", [("p1", [_media("p1", 1)])]), # the resumed page — not counted + ("CUR3", [("p2", [_media("p2", 1)])]), # net-new + ("CUR4", [("p3", [_media("p3", 1)])]), # net-new + ] + ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path)) + ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="backfill", + resume_cursor="CUR2", posts_base=4, + ) + co = (await db.execute( + select(Source.config_overrides).where(Source.id == source_id) + )).scalar_one() + # 4 prior + 2 net-new (p2, p3); p1 on the resumed CUR2 page is excluded. + assert co.get("_backfill_posts") == 6 + + +@pytest.mark.asyncio +async def test_tick_does_not_persist_posts(source_id, sync_engine, tmp_path, db): + """A tick has no resumable backfill state, so it must not write _backfill_posts + (the badge is a backfill/recovery concept).""" + ing = _ingester( + sync_engine, tmp_path, + _FakeClient([("CUR2", [("p1", [_media("p1", 1)])])]), + _FakeDownloader(tmp_path), + ) + ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + co = (await db.execute( + select(Source.config_overrides).where(Source.id == source_id) + )).scalar_one() + assert "_backfill_posts" not in (co or {}) + + @pytest.mark.asyncio async def test_tick_does_not_checkpoint_cursor(source_id, sync_engine, tmp_path, db): """A tick has no resumable backfill state, so it must not write a cursor.""" -- 2.52.0 From b21190039072d2141ab66ae9a0f5dd9ee7de6acf Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 11:11:25 -0400 Subject: [PATCH 20/26] =?UTF-8?q?refactor(downloads):=20DRY=20the=20ingest?= =?UTF-8?q?er/gallery-dl=20seam=20=E2=80=94=20A1=E2=80=93A4=20(plan=20#707?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consolidates duplication that owning the native ingester left against the still- live gallery-dl path, and fixes a parity gap the duplication hid. A1 — shared quarantine: extract file_validator.quarantine_file (move to _quarantine// + write the .quarantine.json provenance sidecar). gallery_dl._validate_and_quarantine and patreon_downloader._validate_path both call it. PARITY FIX: the native path now writes the provenance sidecar it previously skipped — threads the media url through for source_url. A2 — make_run_stats(**counts) factory in gallery_dl for the canonical run_stats key set; gallery_dl._compute_run_stats and ingest_core both build through it so the shape can't drift (gallery-dl path gains a benign dead_lettered_count=0). A3 — one safe_ext in utils/paths.py; importer._safe_ext (thin wrapper, kept for the Path call sites + memory pointer) and patreon_client both use it. Closes the double-impl of the URL-encoded-basename ext gotcha. A4 — promote gallery_dl._truncate_log/_extract_errors_warnings to module-level truncate_log/extract_errors_warnings; download_service calls them directly instead of reaching through self.gdl for native-result log shaping. The staticmethods stay as thin delegators for existing callers/tests. Behavior-preserving except the A1 sidecar parity fix. Test: native quarantine writes a .quarantine.json (test_patreon_downloader). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/download_service.py | 8 +- backend/app/services/file_validator.py | 55 ++++++++ backend/app/services/gallery_dl.py | 145 ++++++++++++--------- backend/app/services/importer.py | 31 ++--- backend/app/services/ingest_core.py | 20 ++- backend/app/services/patreon_client.py | 18 +-- backend/app/services/patreon_downloader.py | 40 +++--- backend/app/utils/paths.py | 20 +++ tests/test_patreon_downloader.py | 22 ++++ 9 files changed, 222 insertions(+), 137 deletions(-) diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 390d854..86d8324 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -33,6 +33,8 @@ from .gallery_dl import ( ErrorType, GalleryDLService, SourceConfig, + extract_errors_warnings, + truncate_log, ) from .importer import Importer from .patreon_ingester import PatreonIngester @@ -450,7 +452,7 @@ class DownloadService: dl_result.return_code, dl_result.stdout, dl_result.stderr ) run_stats["quarantined_count"] = dl_result.files_quarantined - stderr_summary = self.gdl._extract_errors_warnings(dl_result.stderr) + stderr_summary = extract_errors_warnings(dl_result.stderr) # Plan #544: PARTIAL means the run downloaded ≥1 file but the # subprocess didn't finish in budget (typically wall-clock timeout @@ -470,8 +472,8 @@ class DownloadService: ev.metadata_ = { "run_stats": run_stats, "error_type": dl_result.error_type.value if dl_result.error_type else None, - "stdout": self.gdl._truncate_log(dl_result.stdout) or None, - "stderr": self.gdl._truncate_log(dl_result.stderr) or None, + "stdout": truncate_log(dl_result.stdout) or None, + "stderr": truncate_log(dl_result.stderr) or None, "stderr_errors_warnings": stderr_summary or None, "duration_seconds": dl_result.duration_seconds, "quarantined_paths": dl_result.quarantined_paths or None, diff --git a/backend/app/services/file_validator.py b/backend/app/services/file_validator.py index ca6e40f..e007422 100644 --- a/backend/app/services/file_validator.py +++ b/backend/app/services/file_validator.py @@ -11,9 +11,15 @@ expected to write. from __future__ import annotations +import json +import logging +import shutil from dataclasses import dataclass +from datetime import UTC, datetime from pathlib import Path +log = logging.getLogger(__name__) + JPEG_HEAD = b"\xff\xd8\xff" JPEG_TAIL = b"\xff\xd9" @@ -108,3 +114,52 @@ def validate_file(path: Path) -> ValidationResult: return ValidationResult(ok=True, format="webp", size=size) return ValidationResult(ok=True, format=None, size=size) + + +def quarantine_file( + images_root: Path, + path: Path, + artist_slug: str, + platform: str, + *, + url: str | None, + result: ValidationResult, +) -> Path | None: + """Move a validation-failed file to `_quarantine//` and write + a `.quarantine.json` provenance sidecar next to it. + + Returns the destination path, or None if the move itself failed (file left + in place — the caller decides what to report). The caller has already run + `validate_file` and seen `result.ok is False`. ONE implementation for both + download backends — gallery-dl's batch post-process and the native ingester's + per-media path — so the quarantine layout + provenance sidecar can't drift. + """ + quarantine_root = images_root / "_quarantine" / artist_slug / platform + try: + quarantine_root.mkdir(parents=True, exist_ok=True) + dest = quarantine_root / path.name + counter = 1 + while dest.exists(): + dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}" + counter += 1 + shutil.move(str(path), str(dest)) + sidecar = dest.with_suffix(dest.suffix + ".quarantine.json") + sidecar.write_text( + json.dumps( + { + "original_path": str(path), + "source_url": url, + "artist_slug": artist_slug, + "platform": platform, + "format": result.format, + "reason": result.reason, + "size": result.size, + "quarantined_at": datetime.now(UTC).isoformat(), + }, + indent=2, + ) + ) + except OSError as exc: + log.error("Failed to quarantine %s: %s. File left in place.", path, exc) + return None + return dest diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index f4c9967..d9e9541 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -22,7 +22,7 @@ from datetime import UTC, datetime from enum import StrEnum from pathlib import Path -from .file_validator import is_validatable, validate_file +from .file_validator import is_validatable, quarantine_file, validate_file log = logging.getLogger(__name__) @@ -165,6 +165,39 @@ class DownloadResult: posts_processed: int = 0 +def extract_errors_warnings(stderr: str) -> str: + """Keep only the `[error]`/`[warning]` lines from a captured stderr blob. + + A generic download-log helper (module-level so the native-ingester result + path can shape its logs without reaching through a GalleryDLService instance). + """ + if not stderr: + return "" + kept = [ + line for line in stderr.splitlines() + if "][error]" in line.lower() or "][warning]" in line.lower() + ] + return "\n".join(kept) + + +def truncate_log(text: str, max_bytes: int = 500_000) -> str: + """Cap a captured log to `max_bytes`, eliding the middle (head + tail kept).""" + if not text: + return text + encoded = text.encode("utf-8") + if len(encoded) <= max_bytes: + return text + half = max_bytes // 2 + head = encoded[:half].decode("utf-8", errors="ignore") + tail = encoded[-half:].decode("utf-8", errors="ignore") + head_lines = head.count("\n") + tail_lines = tail.count("\n") + total_lines = text.count("\n") + elided = max(0, total_lines - head_lines - tail_lines) + marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n" + return head + marker + tail + + def _summarize_validation_failures(failures: list[dict]) -> str: if not failures: return "Validation failed" @@ -184,6 +217,35 @@ def _summarize_validation_failures(failures: list[dict]) -> str: # no log text to scrape — and gallery-dl platforms never had a cursor.) +def make_run_stats( + *, + exit_code: int = 0, + downloaded_count: int = 0, + skipped_count: int = 0, + per_item_failures: int = 0, + warning_count: int = 0, + tier_gated_count: int = 0, + quarantined_count: int = 0, + dead_lettered_count: int = 0, +) -> dict: + """The canonical `run_stats` dict shape, in ONE place. + + Both result producers — gallery-dl's `_compute_run_stats` (log scrape) and the + native ingester's per-outcome tally (ingest_core) — build through this so the + key set can't drift between backends. Phase 3 + the Logs UI read these keys. + """ + return { + "exit_code": exit_code, + "downloaded_count": downloaded_count, + "skipped_count": skipped_count, + "per_item_failures": per_item_failures, + "warning_count": warning_count, + "tier_gated_count": tier_gated_count, + "quarantined_count": quarantined_count, + "dead_lettered_count": dead_lettered_count, + } + + class GalleryDLService: """Service for executing gallery-dl downloads.""" @@ -521,10 +583,6 @@ class GalleryDLService: if not written_paths: return quarantined_relpaths, failures - quarantine_root = ( - self.images_root / "_quarantine" / artist_slug / platform - ) - for path in written_paths: if not is_validatable(path): continue @@ -535,34 +593,14 @@ class GalleryDLService: continue if result.ok: continue - try: - quarantine_root.mkdir(parents=True, exist_ok=True) - dest = quarantine_root / path.name - counter = 1 - while dest.exists(): - dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}" - counter += 1 - path.rename(dest) - sidecar = dest.with_suffix(dest.suffix + ".quarantine.json") - sidecar.write_text( - json.dumps( - { - "original_path": str(path), - "source_url": url, - "artist_slug": artist_slug, - "platform": platform, - "format": result.format, - "reason": result.reason, - "size": result.size, - "quarantined_at": datetime.now(UTC).isoformat(), - }, - indent=2, - ) - ) - except OSError as exc: - log.error("Failed to quarantine %s: %s. File left in place.", path, exc) - continue - + # Shared move+sidecar (file_validator.quarantine_file) — same impl the + # native ingester uses, so the layout + provenance sidecar can't drift. + dest = quarantine_file( + self.images_root, path, artist_slug, platform, + url=url, result=result, + ) + if dest is None: + continue # move failed → left in place, not counted log.warning( "Quarantined corrupt file: %s → %s (%s: %s)", path, dest, result.format, result.reason, @@ -600,41 +638,24 @@ class GalleryDLService: if "][warning]" in line.lower() and "not allowed to view post" in line.lower() ) - return { - "exit_code": return_code, - "downloaded_count": self._count_downloaded_files(stdout), - "skipped_count": skipped_stdout + skipped_stderr, - "per_item_failures": per_item_failures, - "warning_count": warning_count, - "tier_gated_count": tier_gated_count, - } + return make_run_stats( + exit_code=return_code, + downloaded_count=self._count_downloaded_files(stdout), + skipped_count=skipped_stdout + skipped_stderr, + per_item_failures=per_item_failures, + warning_count=warning_count, + tier_gated_count=tier_gated_count, + ) @staticmethod def _extract_errors_warnings(stderr: str) -> str: - if not stderr: - return "" - kept = [ - line for line in stderr.splitlines() - if "][error]" in line.lower() or "][warning]" in line.lower() - ] - return "\n".join(kept) + # Thin delegator to the module-level helper (kept for existing callers). + return extract_errors_warnings(stderr) @staticmethod def _truncate_log(text: str, max_bytes: int = 500_000) -> str: - if not text: - return text - encoded = text.encode("utf-8") - if len(encoded) <= max_bytes: - return text - half = max_bytes // 2 - head = encoded[:half].decode("utf-8", errors="ignore") - tail = encoded[-half:].decode("utf-8", errors="ignore") - head_lines = head.count("\n") - tail_lines = tail.count("\n") - total_lines = text.count("\n") - elided = max(0, total_lines - head_lines - tail_lines) - marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n" - return head + marker + tail + # Thin delegator to the module-level helper (kept for existing callers). + return truncate_log(text, max_bytes) async def download( self, diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index 213b86e..e1acc6e 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -31,7 +31,12 @@ from ..models import ( Source, ) from ..utils import safe_probe -from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name +from ..utils.paths import ( + derive_subdir, + derive_top_level_artist, + hash_suffixed_name, + safe_ext, +) from ..utils.phash import compute_phash, find_similar from ..utils.sidecar import find_sidecar, parse_sidecar from ..utils.slug import slugify @@ -82,27 +87,9 @@ def is_video(path: Path) -> bool: def _safe_ext(path: Path) -> str: """Conservatively extract a file extension for PostAttachment.ext - (varchar(32)). - - gallery-dl produces some filenames with URL-encoded query-string - artifacts embedded into the basename (e.g. - `79507046_media_..._https___www.patreon.com_media-u_Z0FBQUFBQm5q...`). - `Path.suffix` finds the LAST dot and returns everything after, which - in those cases yields a 50+ char "extension" of mostly base64-ish - junk. That blows the column. Operator-flagged 2026-05-25. - - Real extensions are short and alphanumeric. We accept anything ≤ 16 - chars where every post-dot character is alphanumeric; anything else - means the input wasn't a real extension and we return the empty - string. ext is nullable-ish (empty string still satisfies NOT NULL) - and consumers should treat "" as "no known extension". - """ - suffix = path.suffix.lower() - if not suffix or len(suffix) > 16: - return "" - if not all(c.isalnum() for c in suffix[1:]): - return "" - return suffix + (varchar(32)). Thin wrapper over the shared `utils.paths.safe_ext` (kept for + the Path-typed call sites + the [[path_suffix_sanitize]] memory pointer).""" + return safe_ext(path) def _mime_for(path: Path) -> str: diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 0e6fb5d..be418b0 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -32,7 +32,7 @@ from collections.abc import Callable from sqlalchemy import delete, func, select, text from sqlalchemy.dialects.postgresql import insert as pg_insert -from .gallery_dl import DownloadResult, ErrorType +from .gallery_dl import DownloadResult, ErrorType, make_run_stats log = logging.getLogger(__name__) @@ -153,16 +153,14 @@ class Ingester: duration_seconds=time.monotonic() - start, cursor=emitted_cursor, posts_processed=posts_processed, - run_stats={ - "exit_code": return_code, - "downloaded_count": downloaded, - "skipped_count": skipped_count, - "per_item_failures": errors, - "warning_count": 0, - "tier_gated_count": 0, - "quarantined_count": quarantined, - "dead_lettered_count": dead_lettered, - }, + run_stats=make_run_stats( + exit_code=return_code, + downloaded_count=downloaded, + skipped_count=skipped_count, + per_item_failures=errors, + quarantined_count=quarantined, + dead_lettered_count=dead_lettered, + ), ) try: diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index 5d1ef2b..f2af42f 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -39,6 +39,8 @@ from urllib.parse import parse_qs, urlsplit import requests +from ..utils.paths import safe_ext + log = logging.getLogger(__name__) _POSTS_URL = "https://www.patreon.com/api/posts" @@ -99,11 +101,6 @@ _FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})") # by filehash. Tolerant of attribute ordering and single/double quotes. _CONTENT_IMG_RE = re.compile(r"]*?\bsrc=[\"']([^\"']+)[\"']", re.IGNORECASE) -# A bounded, sane extension parse (see importer._safe_ext for the FC gotcha: -# URL-encoded basenames make Path.suffix return base64-ish junk). We only ever -# use this for a fallback filename, never a network call. -_MAX_EXT_LEN = 16 - class PatreonAPIError(Exception): """Base for native Patreon client failures. @@ -187,15 +184,6 @@ def _filehash(url: str) -> str | None: return match.group(1).lower() if match else None -def _safe_ext(name: str) -> str: - suffix = Path(name).suffix.lower() - if not suffix or len(suffix) > _MAX_EXT_LEN: - return "" - if not all(c.isalnum() for c in suffix[1:]): - return "" - return suffix - - def _basename_from_url(url: str) -> str: """Derive a sane filename from a URL when the media has no file_name. @@ -206,7 +194,7 @@ def _basename_from_url(url: str) -> str: path = urlsplit(url).path base = os.path.basename(path) if base: - ext = _safe_ext(base) + ext = safe_ext(base) stem = base[: -len(Path(base).suffix)] if Path(base).suffix else base # Keep the stem bounded; URL-encoded stems can be enormous. stem = stem[:120] or "file" diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index ad001c8..06be462 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -31,7 +31,6 @@ import contextlib import json import logging import os -import shutil import subprocess import time from collections.abc import Callable @@ -42,7 +41,7 @@ from urllib.parse import urlsplit import requests -from .file_validator import is_validatable, validate_file +from .file_validator import is_validatable, quarantine_file, validate_file from .patreon_client import ( _BACKOFF_CAP_SECONDS, _load_session, @@ -268,7 +267,9 @@ class PatreonDownloader: out_path = Path(out_path) else: out_path = self._fetch_get(media.url, media_path) - reason, quarantine_dest = self._validate_path(out_path, artist_slug) + reason, quarantine_dest = self._validate_path( + out_path, artist_slug, media.url + ) if reason is not None: # Quarantined (corrupt/invalid) — distinct from a download error # so the run can report a real files_quarantined count + paths. @@ -426,16 +427,16 @@ class PatreonDownloader: # -- validation -------------------------------------------------------- def _validate_path( - self, path: Path, artist_slug: str + self, path: Path, artist_slug: str, source_url: str | None = None ) -> tuple[str | None, Path | None]: """Validate a freshly-written file; quarantine if bad. - Mirrors gallery_dl._validate_and_quarantine: fail-open for unknown - formats, move the corrupt file to _quarantine//patreon. Returns - `(reason, quarantine_dest)` when quarantined (dest is the original path if - the move itself failed), else `(None, None)` (ok / not validatable / - disabled). plan #704: the dest is surfaced so the run reports a real - quarantined-paths list instead of a blank count. + Uses the shared `file_validator.quarantine_file` — same move + provenance + sidecar gallery-dl writes (the native path used to skip the sidecar; that + parity gap is closed here). Returns `(reason, quarantine_dest)` when + quarantined (dest is the original path if the move itself failed), else + `(None, None)` (ok / not validatable / disabled). plan #704: the dest is + surfaced so the run reports a real quarantined-paths list. """ if not self._validate or not is_validatable(path): return None, None @@ -446,20 +447,11 @@ class PatreonDownloader: return None, None if result.ok: return None, None - quarantine_root = self.images_root / "_quarantine" / artist_slug / "patreon" - dest = path - try: - quarantine_root.mkdir(parents=True, exist_ok=True) - dest = quarantine_root / path.name - counter = 1 - while dest.exists(): - dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}" - counter += 1 - shutil.move(str(path), str(dest)) - except OSError as exc: - log.error("Failed to quarantine %s: %s. File left in place.", path, exc) - dest = path - return (result.reason or "validation failed"), dest + dest = quarantine_file( + self.images_root, path, artist_slug, "patreon", + url=source_url, result=result, + ) + return (result.reason or "validation failed"), (dest or path) # -- sidecar ----------------------------------------------------------- diff --git a/backend/app/utils/paths.py b/backend/app/utils/paths.py index c19b7f8..d9bc48e 100644 --- a/backend/app/utils/paths.py +++ b/backend/app/utils/paths.py @@ -2,6 +2,26 @@ from pathlib import Path +_MAX_EXT_LEN = 16 + + +def safe_ext(name: str | Path) -> str: + """Conservatively extract a short, alphanumeric file extension. + + gallery-dl and Patreon CDN URLs produce basenames with URL-encoded + query-string artifacts, so `Path.suffix` can return 50+ chars of base64-ish + junk that blows bounded VARCHAR columns (e.g. PostAttachment.ext varchar(32)). + Accept only a suffix ≤16 chars whose post-dot characters are all alphanumeric; + otherwise return "" (no known extension). Operator-flagged 2026-05-25 — ONE + impl for the importer and the native Patreon client. + """ + suffix = Path(name).suffix.lower() + if not suffix or len(suffix) > _MAX_EXT_LEN: + return "" + if not all(c.isalnum() for c in suffix[1:]): + return "" + return suffix + def derive_subdir(source_path: Path, import_root: Path) -> str: """Returns the relative subdirectory of source_path under import_root. diff --git a/tests/test_patreon_downloader.py b/tests/test_patreon_downloader.py index e606489..774f6fe 100644 --- a/tests/test_patreon_downloader.py +++ b/tests/test_patreon_downloader.py @@ -305,6 +305,28 @@ def test_validation_quarantines_corrupt(tmp_path): assert any(quarantine.iterdir()) +def test_quarantine_writes_provenance_sidecar(tmp_path): + """A1 parity fix: the native path now writes the same `.quarantine.json` + provenance sidecar gallery-dl writes (it skipped it before the shared + file_validator.quarantine_file helper).""" + import json + + bad = b"not a real png" + url = "https://cdn.patreon.com/corrupt.png" + session = _FakeSession({url: bad}) + dl = _downloader(tmp_path, session=session, validate=True) + outcomes = dl.download_post(_post(), [_img("corrupt.png", url=url)], "artist-x") + + dest = outcomes[0].path + sidecar = dest.with_suffix(dest.suffix + ".quarantine.json") + assert sidecar.is_file() + meta = json.loads(sidecar.read_text()) + assert meta["source_url"] == url + assert meta["platform"] == "patreon" + assert meta["artist_slug"] == "artist-x" + assert meta["reason"] + + def test_validation_off_keeps_bytes(tmp_path): bad = b"not a real png" session = _FakeSession({"https://cdn.patreon.com/x.png": bad}) -- 2.52.0 From e47fa0cf4b9b3236a7984eb500ef030fe8a0e687 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 11:22:48 -0400 Subject: [PATCH 21/26] =?UTF-8?q?refactor(downloads):=20unify=20phase-2=20?= =?UTF-8?q?dispatch=20in=20download=5Fbackends=20=E2=80=94=20A5=20(plan=20?= =?UTF-8?q?#707)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror verify_source_credential: download_backends.run_download is now the single download entry, so that module is the ONE registry of how each platform both downloads AND verifies — the seam that makes adding a platform a bounded job (write its adapter construction next to its verify). The native-ingester construction + campaign-id resolution moves out of download_service into download_backends._run_native_ingester. download_service.download_source drops its `if uses_native_ingester ... else gdl.download` branch and calls one `self._run_download(...)` (a thin delegate to run_download passing the service's gdl + sync sessionmaker). mode (tick/backfill/ recovery) is still chosen there from the backfill state machine and threaded through. Removed the now-unused PatreonIngester / resolve_campaign_id_for_source imports from download_service. Tests: the phase-2 stub seam moves from svc._run_patreon_ingester to svc._run_download (helper + the db-release test); the two native-construction tests repoint to download_backends.run_download (patching download_backends.resolve_campaign_id_for_source / PatreonIngester). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/download_backends.py | 110 +++++++++++++++++++++ backend/app/services/download_service.py | 115 ++++------------------ tests/test_download_service.py | 76 +++++++------- 3 files changed, 167 insertions(+), 134 deletions(-) diff --git a/backend/app/services/download_backends.py b/backend/app/services/download_backends.py index 737da4d..89b295a 100644 --- a/backend/app/services/download_backends.py +++ b/backend/app/services/download_backends.py @@ -20,8 +20,13 @@ function regardless of platform: from __future__ import annotations +import asyncio from pathlib import Path +from .gallery_dl import DownloadResult, ErrorType +from .patreon_ingester import PatreonIngester +from .patreon_resolver import resolve_campaign_id_for_source + # Platforms whose download + verify go through the native ingester rather than # gallery-dl. gallery-dl still serves every other platform (subscribestar, # hentaifoundry, discord, pixiv, deviantart) unchanged. @@ -34,6 +39,111 @@ def uses_native_ingester(platform: str) -> bool: return platform in NATIVE_INGESTER_PLATFORMS +async def run_download( + *, + ctx: dict, + source_config, + skip_value: bool | str, + mode: str | None, + gdl, + sync_session_factory, +) -> tuple[DownloadResult, str | None]: + """Uniform download across backends — the download counterpart to + `verify_source_credential`, so this module is the ONE place that knows how + each platform both downloads AND verifies (the seam that makes adding a + platform a bounded job). + + Returns `(DownloadResult, resolved_campaign_id)`; `resolved_campaign_id` is + non-None only when a native vanity lookup ran this call (so phase 3 caches + it). Native platforms route through their ingester in `mode` + (tick/backfill/recovery); gallery-dl platforms run the subprocess. The caller + (download_service) prepares `source_config`/`skip_value`/`mode` from the + backfill state machine and owns phase 3. + """ + platform = ctx["platform"] + if uses_native_ingester(platform): + return await _run_native_ingester( + ctx, source_config, mode, gdl, sync_session_factory + ) + result = await gdl.download( + url=ctx["url"], + artist_slug=ctx["artist_slug"], + platform=platform, + source_config=source_config, + cookies_path=ctx["cookies_path"], + auth_token=ctx["auth_token"], + skip_value=skip_value, + ) + return result, None + + +async def _run_native_ingester( + ctx: dict, source_config, mode: str | None, gdl, sync_session_factory, +) -> tuple[DownloadResult, str | None]: + """Patreon (today the only native platform): resolve the campaign id, then run + the native ingester in a worker thread (it is sync requests/subprocess). + + `resolved_campaign_id` is non-None only when we had to look it up from the + vanity URL this run, so phase 3 caches it the way the old gallery-dl retry + did. A campaign id we cannot resolve is a loud NOT_FOUND — never a silent + empty success. + """ + overrides = ctx["config_overrides"] or {} + campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source( + ctx["url"], ctx["cookies_path"], overrides + ) + if not campaign_id: + return ( + DownloadResult( + success=False, + url=ctx["url"], + artist_slug=ctx["artist_slug"], + platform="patreon", + error_type=ErrorType.NOT_FOUND, + error_message=( + "Could not resolve Patreon campaign id from the source URL " + "(vanity lookup failed — cookies expired or creator moved?)" + ), + ), + None, + ) + + # Honor the operator's existing rate-limit knobs on the native path (plan + # #703): the global download_rate_limit_seconds (gallery-dl's `rate_limit`, + # here on the gdl service) paces media downloads; page fetches use the + # per-source sleep_request override, else `max(0.5, rate_limit/4)` — the same + # API-pacing default gallery-dl applied as its `sleep-request`. + rate_limit = gdl._rate_limit + request_sleep = ( + source_config.sleep_request + if source_config.sleep_request is not None + else max(0.5, rate_limit / 4) + ) + ingester = PatreonIngester( + images_root=gdl.images_root, + cookies_path=ctx["cookies_path"], + session_factory=sync_session_factory, + validate=gdl._validate_files, + rate_limit=rate_limit, + request_sleep=request_sleep, + ) + loop = asyncio.get_running_loop() + dl_result = await loop.run_in_executor( + None, + lambda: ingester.run( + source_id=ctx["source_id"], + campaign_id=campaign_id, + artist_slug=ctx["artist_slug"], + url=ctx["url"], + mode=mode, + resume_cursor=source_config.resume_cursor, + time_budget_seconds=source_config.timeout, + posts_base=int(overrides.get("_backfill_posts", 0)), + ), + ) + return dl_result, resolved_campaign_id + + async def verify_source_credential( *, platform: str, diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 86d8324..5295dcd 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -24,7 +24,7 @@ from sqlalchemy.orm import joinedload from ..models import Artist, DownloadEvent, Source from .credential_service import CredentialService -from .download_backends import uses_native_ingester +from .download_backends import run_download, uses_native_ingester from .gallery_dl import ( BACKFILL_CHUNK_SECONDS, BACKFILL_SKIP_VALUE, @@ -37,8 +37,6 @@ from .gallery_dl import ( truncate_log, ) from .importer import Importer -from .patreon_ingester import PatreonIngester -from .patreon_resolver import resolve_campaign_id_for_source from .platforms import auth_type_for from .scheduler_service import set_platform_cooldown @@ -122,32 +120,23 @@ class DownloadService: else: skip_value = TICK_SKIP_VALUE - resolved_campaign_id: str | None = None + # Phase 2 dispatch is uniform across backends (download_backends. + # run_download — the download counterpart to verify_source_credential): + # native platforms run their ingester in `mode` (zero per-file HEADs, + # native cursor/resume, loud drift detection); gallery-dl platforms run + # the subprocess. Either returns a DownloadResult-shaped object so phase 3 + # is untouched. `mode` is None for gallery-dl (run_download ignores it). + mode: str | None = None if uses_native_ingester(ctx["platform"]): - # Native ingester (plan #697) fully replaces gallery-dl for Patreon - # in phase 2 — zero per-file HEADs, native cursor/resume, loud drift - # detection. Returns a DownloadResult-shaped object so phase 3 is - # untouched. The gallery-dl Patreon path + its campaign-id retry are - # removed at the step-5 cutover. if in_backfill and bypass_seen: mode = "recovery" elif in_backfill: mode = "backfill" else: mode = "tick" - dl_result, resolved_campaign_id = await self._run_patreon_ingester( - ctx, source_config, mode, - ) - else: - dl_result = await self.gdl.download( - url=ctx["url"], - artist_slug=ctx["artist_slug"], - platform=ctx["platform"], - source_config=source_config, - cookies_path=ctx["cookies_path"], - auth_token=ctx["auth_token"], - skip_value=skip_value, - ) + dl_result, resolved_campaign_id = await self._run_download( + ctx=ctx, source_config=source_config, skip_value=skip_value, mode=mode, + ) # A backfill chunk that hit its time-box but made forward progress is # NORMAL, not a failure — reclassify TIMEOUT → PARTIAL so it reads as @@ -171,82 +160,18 @@ class DownloadService: ctx["event_id"], ctx, dl_result, resolved_campaign_id, ) - async def _run_patreon_ingester( - self, ctx: dict, source_config, mode: str, + async def _run_download( + self, *, ctx: dict, source_config, skip_value, mode: str | None, ) -> tuple[DownloadResult, str | None]: - """Phase-2 Patreon branch: resolve the campaign id, then run the native - ingester in a worker thread (it is sync requests/subprocess). - - Returns (DownloadResult, resolved_campaign_id). `resolved_campaign_id` is - non-None only when we had to look it up from the vanity URL this run, so - phase 3 caches it on the source the same way the old gallery-dl retry did. - A campaign id we cannot resolve is a loud failure (NOT_FOUND) — never a - silent empty success. - """ - overrides = ctx["config_overrides"] or {} - # Shared resolution path (override / id: URL / vanity lookup) — the same - # helper the credential-verify probe uses. resolved_campaign_id is - # non-None only when a vanity lookup ran, so phase 3 caches it. - campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source( - ctx["url"], ctx["cookies_path"], overrides + """Phase-2 dispatch → `download_backends.run_download`, passing this + service's gdl + sync sessionmaker. Kept as a thin instance method so tests + can stub the whole phase-2 dispatch on the service, and so the per-backend + construction lives in download_backends (the backend registry).""" + return await run_download( + ctx=ctx, source_config=source_config, skip_value=skip_value, mode=mode, + gdl=self.gdl, sync_session_factory=self.sync_session_factory, ) - if not campaign_id: - return ( - DownloadResult( - success=False, - url=ctx["url"], - artist_slug=ctx["artist_slug"], - platform="patreon", - error_type=ErrorType.NOT_FOUND, - error_message=( - "Could not resolve Patreon campaign id from the source " - "URL (vanity lookup failed — cookies expired or creator " - "moved?)" - ), - ), - None, - ) - - # Honor the operator's existing rate-limit knobs on the native path - # (plan #703): the global download_rate_limit_seconds (gallery-dl's - # `rate_limit`, here on self.gdl) paces media downloads. Page fetches use - # the per-source sleep_request override, else `max(0.5, rate_limit/4)` — - # the SAME API-pacing default gallery-dl applied as its `sleep-request` - # (gallery_dl._get_default_config), so the rate-limited /api/posts - # endpoint stays politely paced without over-throttling. - rate_limit = self.gdl._rate_limit - request_sleep = ( - source_config.sleep_request - if source_config.sleep_request is not None - else max(0.5, rate_limit / 4) - ) - ingester = PatreonIngester( - images_root=self.gdl.images_root, - cookies_path=ctx["cookies_path"], - session_factory=self.sync_session_factory, - validate=self.gdl._validate_files, - rate_limit=rate_limit, - request_sleep=request_sleep, - ) - loop = asyncio.get_running_loop() - dl_result = await loop.run_in_executor( - None, - lambda: ingester.run( - source_id=ctx["source_id"], - campaign_id=campaign_id, - artist_slug=ctx["artist_slug"], - url=ctx["url"], - mode=mode, - resume_cursor=source_config.resume_cursor, - time_budget_seconds=source_config.timeout, - # plan #704 #5: seed the live posts badge from prior chunks so the - # ingester writes a monotonic absolute mid-walk (it owns the key). - posts_base=int(overrides.get("_backfill_posts", 0)), - ), - ) - return dl_result, resolved_campaign_id - async def _phase1_setup(self, source_id: int) -> dict[str, Any]: source = (await self.async_session.execute( select(Source).options(joinedload(Source.artist)).where(Source.id == source_id) diff --git a/tests/test_download_service.py b/tests/test_download_service.py index d08eb12..354e04e 100644 --- a/tests/test_download_service.py +++ b/tests/test_download_service.py @@ -89,7 +89,7 @@ def _make_fake_dl_result( def _fake_gdl_with_result(result): fake = MagicMock() fake.download = AsyncMock(return_value=result) - # Real values for the attrs _run_patreon_ingester reads off the gdl service + # Real values for the attrs run_download's native branch reads off the gdl # (the native pacing config, plan #703) — a MagicMock would break the # arithmetic (max(0.5, rate_limit/4)). fake._rate_limit = 3.0 @@ -105,19 +105,24 @@ def _fake_gdl_with_result(result): def _stub_patreon_ingester(svc, result, resolved_campaign_id=None): - """Route the Patreon phase-2 branch (plan #697 native ingester) to a canned - DownloadResult so these tests exercise download_service's - phase-2-result→phase-3 handling (status mapping, backfill lifecycle, health) - without the real ingester's network/DB. The ingester itself is covered by - test_patreon_ingester.py. Returns a list capturing (ctx, source_config, mode) - per call so a test can assert mode / resume_cursor threading.""" + """Route the phase-2 dispatch (download_backends.run_download via + DownloadService._run_download, plan #707 A5) to a canned DownloadResult so + these tests exercise download_service's phase-2-result→phase-3 handling + (status mapping, backfill lifecycle, health) without the real ingester's + network/DB. The ingester itself is covered by test_patreon_ingester.py; the + native construction/resolution by the run_download tests below. Returns a list + capturing (ctx, source_config, skip_value, mode) per call so a test can assert + mode / resume_cursor threading.""" calls = [] - async def fake(ctx, source_config, mode): - calls.append({"ctx": ctx, "source_config": source_config, "mode": mode}) + async def fake(*, ctx, source_config, skip_value, mode): + calls.append({ + "ctx": ctx, "source_config": source_config, + "skip_value": skip_value, "mode": mode, + }) return result, resolved_campaign_id - svc._run_patreon_ingester = fake + svc._run_download = fake return calls @@ -285,20 +290,19 @@ async def test_patreon_resolved_campaign_id_is_cached( @pytest.mark.asyncio -async def test_run_patreon_ingester_resolves_vanity_and_runs( +async def test_run_download_native_resolves_vanity_and_runs( db, db_sync, tmp_path, seed_artist_and_source, monkeypatch, ): - """_run_patreon_ingester: with no cached campaign id, resolve the vanity and - pass the resolved id straight to the ingester; report it back so phase 3 can - cache it.""" - from backend.app.services import download_service as dl_mod - from backend.app.services.download_service import DownloadService + """run_download (native branch, plan #707 A5): with no cached campaign id, + resolve the vanity and pass the resolved id straight to the ingester; report + it back so phase 3 can cache it.""" + from backend.app.services import download_backends as db_mod from backend.app.services.gallery_dl import SourceConfig _artist, source = seed_artist_and_source monkeypatch.setattr( - dl_mod, "resolve_campaign_id_for_source", + db_mod, "resolve_campaign_id_for_source", AsyncMock(return_value=("4242", "4242")), ) run_kwargs = {} @@ -311,21 +315,19 @@ async def test_run_patreon_ingester_resolves_vanity_and_runs( run_kwargs.update(kw) return _make_fake_dl_result(success=True, written_paths=[]) - monkeypatch.setattr(dl_mod, "PatreonIngester", _FakeIngester) + monkeypatch.setattr(db_mod, "PatreonIngester", _FakeIngester) - svc = DownloadService( - async_session=db, sync_session=db_sync, - gdl=_fake_gdl_with_result(_make_fake_dl_result(success=True)), - importer=MagicMock(), cred_service=MagicMock(), - sync_session_factory=MagicMock(), - ) ctx = { + "platform": "patreon", "source_id": source.id, "url": "https://patreon.com/alice", "artist_slug": "alice", "cookies_path": None, "config_overrides": {}, } - result, resolved = await svc._run_patreon_ingester( - ctx, SourceConfig.from_dict({}), "tick", + result, resolved = await db_mod.run_download( + ctx=ctx, source_config=SourceConfig.from_dict({}), skip_value=True, + mode="tick", + gdl=_fake_gdl_with_result(_make_fake_dl_result(success=True)), + sync_session_factory=MagicMock(), ) assert result.success is True assert resolved == "4242" @@ -334,33 +336,29 @@ async def test_run_patreon_ingester_resolves_vanity_and_runs( @pytest.mark.asyncio -async def test_run_patreon_ingester_unresolvable_fails_loud( +async def test_run_download_native_unresolvable_fails_loud( db, db_sync, tmp_path, seed_artist_and_source, monkeypatch, ): """A campaign id we can't resolve is a loud NOT_FOUND failure, never a silent empty success.""" - from backend.app.services import download_service as dl_mod - from backend.app.services.download_service import DownloadService + from backend.app.services import download_backends as db_mod from backend.app.services.gallery_dl import ErrorType, SourceConfig _artist, source = seed_artist_and_source monkeypatch.setattr( - dl_mod, "resolve_campaign_id_for_source", + db_mod, "resolve_campaign_id_for_source", AsyncMock(return_value=(None, None)), ) - svc = DownloadService( - async_session=db, sync_session=db_sync, - gdl=MagicMock(), importer=MagicMock(), cred_service=MagicMock(), - sync_session_factory=MagicMock(), - ) ctx = { + "platform": "patreon", "source_id": source.id, "url": "https://patreon.com/alice", "artist_slug": "alice", "cookies_path": None, "config_overrides": {}, } - result, resolved = await svc._run_patreon_ingester( - ctx, SourceConfig.from_dict({}), "tick", + result, resolved = await db_mod.run_download( + ctx=ctx, source_config=SourceConfig.from_dict({}), skip_value=True, + mode="tick", gdl=MagicMock(), sync_session_factory=MagicMock(), ) assert result.success is False assert result.error_type == ErrorType.NOT_FOUND @@ -926,11 +924,11 @@ async def test_releases_db_connections_before_subprocess( gdl=fake_gdl, importer=importer, cred_service=cred_service, ) - async def fake_ingest(ctx, source_config, mode): + async def fake_ingest(*, ctx, source_config, skip_value, mode): seen["closed_before_phase2"] = closed["async"] return _make_fake_dl_result(success=True, written_paths=[], stdout=""), None - svc._run_patreon_ingester = fake_ingest + svc._run_download = fake_ingest event_id = await svc.download_source(source.id) assert seen["closed_before_phase2"] is True -- 2.52.0 From fb7383eea73384486c6b783c2462a7f0bbb5fbc2 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 11:34:37 -0400 Subject: [PATCH 22/26] =?UTF-8?q?feat(downloads):=20platform=20cooldown=20?= =?UTF-8?q?honors=20server=20Retry-After=20=E2=80=94=20B1=20(plan=20#708)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owning the native client means we see the 429 Retry-After header — previously discarded. PatreonAPIError now carries `retry_after`; on a PERSISTENT page-fetch 429 the client attaches the server's raw Retry-After seconds. New DownloadResult.retry_after_seconds; patreon_ingester._failure_result sets it on RATE_LIMITED. download_service._update_source_health passes it to set_platform_cooldown as `seconds=`, clamped to [60, 3600] (a tiny hint can't leave the platform effectively un-cooled; a huge one can't strand it for hours); no hint → the flat 900s default. So a rate-limited platform cools for as long as the server actually asks, not a fixed guess. Tests: terminal 429 surfaces retry_after (test_patreon_client); cooldown honors + clamps the hint, falls back to default when absent (test_download_service). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/download_service.py | 15 +++++++++-- backend/app/services/gallery_dl.py | 4 +++ backend/app/services/patreon_client.py | 24 ++++++++++++++++- backend/app/services/patreon_ingester.py | 6 ++++- tests/test_download_service.py | 34 ++++++++++++++++++++++++ tests/test_patreon_client.py | 12 +++++++++ 6 files changed, 91 insertions(+), 4 deletions(-) diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 5295dcd..66e960b 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -407,6 +407,7 @@ class DownloadService: await self._update_source_health( source_id=ctx["source_id"], status=status, error_message=ev.error, error_type=dl_result.error_type.value if dl_result.error_type else None, + retry_after_seconds=getattr(dl_result, "retry_after_seconds", None), ) await self._apply_backfill_lifecycle(ctx, dl_result) await self.async_session.commit() @@ -501,7 +502,7 @@ class DownloadService: async def _update_source_health( self, *, source_id: int, status: str, error_message: str | None, - error_type: str | None = None, + error_type: str | None = None, retry_after_seconds: float | None = None, ) -> None: """FC-3d: update Source.{consecutive_failures, last_error, last_checked_at}. @@ -533,7 +534,17 @@ class DownloadService: # by error class without opening Logs per row. source.error_type = error_type if error_type == "rate_limited": - await set_platform_cooldown(self.async_session, source.platform) + # plan #708 B1: honor the server's Retry-After when the native + # client surfaced one (clamped to a sane [60, 3600] window so a + # tiny hint can't leave the platform effectively un-cooled and a + # huge one can't strand it for hours); else the flat default. + if retry_after_seconds is not None: + seconds = int(min(max(retry_after_seconds, 60), 3600)) + await set_platform_cooldown( + self.async_session, source.platform, seconds=seconds, + ) + else: + await set_platform_cooldown(self.async_session, source.platform) elif status == "skipped": source.last_error = None source.last_checked_at = now diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index d9e9541..3add0e1 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -163,6 +163,10 @@ class DownloadResult: run_stats: dict | None = None cursor: str | None = None posts_processed: int = 0 + # Plan #708 B1 — the server's Retry-After seconds on a RATE_LIMITED result, so + # the platform cooldown matches the hint instead of a flat default. None when + # unknown (no header, or not a rate-limit failure). + retry_after_seconds: float | None = None def extract_errors_warnings(stderr: str) -> str: diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index f2af42f..69846db 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -108,11 +108,21 @@ class PatreonAPIError(Exception): `status_code` carries the HTTP status when the failure was an HTTP response (None for transport-level / parse failures), so the ingester can map it to a DownloadResult.error_type (429 → rate_limited, 404 → not_found, …). + `retry_after` carries the server's `Retry-After` seconds on a terminal 429, so + the platform cooldown can match the server's hint instead of a flat default + (plan #708 B1). """ - def __init__(self, message: str, *, status_code: int | None = None): + def __init__( + self, + message: str, + *, + status_code: int | None = None, + retry_after: float | None = None, + ): super().__init__(message) self.status_code = status_code + self.retry_after = retry_after class PatreonAuthError(PatreonAPIError): @@ -294,10 +304,22 @@ class PatreonClient: status_code=resp.status_code, ) if resp.status_code != 200: + # A persistent 429 (retries exhausted) is terminal RATE_LIMITED — carry + # the server's raw Retry-After seconds so the cooldown matches its hint + # (plan #708 B1). Header is uncapped here; the cooldown clamps it. + retry_after = None + if resp.status_code == 429: + hdr = resp.headers.get("Retry-After") + if hdr: + try: + retry_after = float(hdr) + except (TypeError, ValueError): + retry_after = None raise PatreonAPIError( f"Patreon posts API returned HTTP {resp.status_code} " f"(campaign_id={campaign_id})", status_code=resp.status_code, + retry_after=retry_after, ) try: payload = resp.json() diff --git a/backend/app/services/patreon_ingester.py b/backend/app/services/patreon_ingester.py index 3862088..5915da7 100644 --- a/backend/app/services/patreon_ingester.py +++ b/backend/app/services/patreon_ingester.py @@ -162,10 +162,14 @@ class PatreonIngester(Ingester): else: error_type = ErrorType.NETWORK_ERROR log.warning("Patreon ingest failed (%s): %s", error_type.value, message) - return _result( + result = _result( success=False, return_code=1, error_type=error_type, error_message=message, ) + # plan #708 B1: carry the server's Retry-After up to the cooldown. + if error_type == ErrorType.RATE_LIMITED: + result.retry_after_seconds = getattr(exc, "retry_after", None) + return result async def verify_patreon_credential( diff --git a/tests/test_download_service.py b/tests/test_download_service.py index 354e04e..a4b4779 100644 --- a/tests/test_download_service.py +++ b/tests/test_download_service.py @@ -437,6 +437,40 @@ async def test_finalize_error_increments_failures_and_sets_error(db): assert row.last_checked_at is not None +@pytest.mark.asyncio +async def test_rate_limited_cooldown_honors_retry_after(db, monkeypatch): + """plan #708 B1: a RATE_LIMITED result with a Retry-After stamps the platform + cooldown at that duration, clamped to [60, 3600]; no hint → the flat default.""" + from unittest.mock import AsyncMock, MagicMock + + from backend.app.services import download_service as dl_mod + from backend.app.services.download_service import DownloadService + + source_id, _event_id = await _seed_source_with_health(db, suffix="rl") + cooldown = AsyncMock() + monkeypatch.setattr(dl_mod, "set_platform_cooldown", cooldown) + svc = DownloadService( + async_session=db, sync_session=None, + gdl=MagicMock(), importer=MagicMock(), cred_service=MagicMock(), + ) + + async def _health(retry): + cooldown.reset_mock() + await svc._update_source_health( + source_id=source_id, status="error", error_message="rate limited", + error_type="rate_limited", retry_after_seconds=retry, + ) + + await _health(120.0) + assert cooldown.await_args.kwargs["seconds"] == 120 # honored as-is + await _health(5.0) + assert cooldown.await_args.kwargs["seconds"] == 60 # floored + await _health(99999.0) + assert cooldown.await_args.kwargs["seconds"] == 3600 # capped + await _health(None) + assert "seconds" not in cooldown.await_args.kwargs # default cooldown + + @pytest.mark.asyncio async def test_finalize_skipped_preserves_failures_clears_error(db): from backend.app.services.download_service import DownloadService diff --git a/tests/test_patreon_client.py b/tests/test_patreon_client.py index f2939e2..0426147 100644 --- a/tests/test_patreon_client.py +++ b/tests/test_patreon_client.py @@ -327,6 +327,18 @@ def test_fetch_persistent_429_raises_after_retries(monkeypatch): assert not isinstance(ei.value, PatreonAuthError) +def test_fetch_persistent_429_surfaces_retry_after(monkeypatch): + """plan #708 B1: a terminal 429 carries the server's raw Retry-After seconds + (uncapped here — the cooldown clamps) so the platform cooldown matches it.""" + client, _slept = _client_with_sequence( + monkeypatch, [_FakeResp(429, headers={"Retry-After": "120"})] * 10, + ) + with pytest.raises(PatreonAPIError) as ei: + client._fetch("5555", None) + assert ei.value.status_code == 429 + assert ei.value.retry_after == 120.0 + + def test_request_sleep_paces_before_fetch(monkeypatch): client = PatreonClient(cookies_path=None, request_sleep=1.5) monkeypatch.setattr( -- 2.52.0 From 402086c34c450f688c2335ac894b61e7757061a9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 11:38:11 -0400 Subject: [PATCH 23/26] =?UTF-8?q?feat(patreon):=20resume=20partial=20media?= =?UTF-8?q?=20downloads=20via=20HTTP=20Range=20=E2=80=94=20B5=20(plan=20#7?= =?UTF-8?q?08)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owning the media fetch means a mid-download transport cut no longer refetches from zero. _fetch_to_file now resumes: on a transient retry, if bytes already landed in the .part, it requests Range: bytes=- and appends on a 206; falls back to a clean truncate-and-restart if the server ignores Range (200) or the range is past EOF (416). The .part staging means a non-range server never corrupts the output — worst case is the old behavior (refetch from zero). Tests: mid-stream cut resumes from the offset (asserts the Range header); a Range-ignoring server refetches clean (no double-write). Test session fakes updated to accept the new headers kwarg. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/patreon_downloader.py | 25 ++++++- tests/test_patreon_downloader.py | 79 +++++++++++++++++++++- 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index 06be462..b5a9dea 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -301,18 +301,30 @@ class PatreonDownloader: def _fetch_to_file(self, url: str, dest: Path) -> None: """Stream a non-video URL to `dest` via the (stubbable) session, retrying - TRANSIENT failures within the same pass (plan #705 #8). + TRANSIENT failures within the same pass (plan #705 #8) and RESUMING from + the bytes already on disk via a Range request when a retry follows a + mid-download cut (plan #708 B5). Retried (backoff): transport blips (connection reset / timeout / truncated stream — incl. mid-download), HTTP 429 (honoring Retry-After), and 5xx. Failed fast (no retry → HTTPError → per-item error → dead-letter path): 4xx other than 429 (404 gone, 403 forbidden) — re-fetching a permanent failure is pointless. + + Resume: on a retry, if bytes already landed in `dest`, ask for the rest + with `Range: bytes=-`. A 206 means the server honored it → append; a + 200 means it ignored it (served the whole file) → start clean. The caller + (_fetch_get) stages into a `.part`, so a non-range server never corrupts + the output — the worst case is re-downloading from zero, as before. """ attempt = 0 while True: + have = dest.stat().st_size if dest.exists() else 0 + headers = {"Range": f"bytes={have}-"} if have > 0 else None try: - resp = self.session.get(url, stream=True, timeout=_TIMEOUT_SECONDS) + resp = self.session.get( + url, stream=True, timeout=_TIMEOUT_SECONDS, headers=headers, + ) if (resp.status_code == 429 or resp.status_code >= 500) \ and attempt < _MAX_MEDIA_RETRIES: attempt += 1 @@ -324,10 +336,17 @@ class PatreonDownloader: ) time.sleep(delay) continue + # A Range that starts at/past EOF (we already have the whole file) + # comes back 416 — the bytes we kept ARE the file. + if have > 0 and resp.status_code == 416: + return # 2xx → ok; 4xx-non-429 (or an exhausted 429/5xx) → HTTPError # (permanent for this pass) → not caught below → per-item error. resp.raise_for_status() - with open(dest, "wb") as fh: + # 206 → server honored the Range; append after the kept bytes. + # Anything else (200) → it served the whole file → start clean. + mode = "ab" if (have > 0 and resp.status_code == 206) else "wb" + with open(dest, mode) as fh: for chunk in resp.iter_content(chunk_size=_CHUNK): if chunk: fh.write(chunk) diff --git a/tests/test_patreon_downloader.py b/tests/test_patreon_downloader.py index 774f6fe..8533088 100644 --- a/tests/test_patreon_downloader.py +++ b/tests/test_patreon_downloader.py @@ -50,7 +50,7 @@ class _FakeSession: self.payloads = payloads or {} self.calls: list[str] = [] - def get(self, url, stream=False, timeout=None): + def get(self, url, stream=False, timeout=None, headers=None): self.calls.append(url) return _FakeResponse(self.payloads.get(url, _PNG_BYTES)) @@ -63,7 +63,7 @@ class _SeqSession: self._responses = list(responses) self.calls: list[str] = [] - def get(self, url, stream=False, timeout=None): + def get(self, url, stream=False, timeout=None, headers=None): self.calls.append(url) item = self._responses.pop(0) if isinstance(item, Exception): @@ -71,6 +71,26 @@ class _SeqSession: return item +class _FlakyResp: + """A streaming response that yields its payload then optionally raises a + mid-stream transport error (for the Range-resume tests, plan #708 B5).""" + + headers: dict = {} + + def __init__(self, payload, status_code=200, die=False): + self._payload = payload + self.status_code = status_code + self._die = die + + def raise_for_status(self): + return None + + def iter_content(self, chunk_size=65536): + yield self._payload + if self._die: + raise requests.exceptions.ChunkedEncodingError("cut mid-stream") + + def _post(post_id="1001", title="My Post Title", published="2026-05-01T12:00:00Z"): return { "id": post_id, @@ -442,3 +462,58 @@ def test_exhausted_transient_becomes_error(tmp_path, monkeypatch): dl = _dl(tmp_path, session, monkeypatch) outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") assert outcomes[0].status == "error" + + +def test_partial_download_resumes_via_range(tmp_path, monkeypatch): + """plan #708 B5: a mid-download transport cut resumes from the bytes already + on disk via a Range request, not a refetch from zero.""" + full = bytes(i % 256 for i in range(1000)) + split = 400 + + class _ResumeSession: + def __init__(self): + self.headers_seen = [] + + def get(self, url, stream=False, timeout=None, headers=None): + self.headers_seen.append(headers) + if len(self.headers_seen) == 1: + return _FlakyResp(full[:split], die=True) # partial then cut + start = int(headers["Range"].split("=")[1].rstrip("-")) + return _FlakyResp(full[start:], status_code=206) + + monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a: None) + session = _ResumeSession() + dl = PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, session=session, + ) + outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") + assert outcomes[0].status == "downloaded" + assert outcomes[0].path.read_bytes() == full + assert session.headers_seen[0] is None # first GET: no Range + assert session.headers_seen[1] == {"Range": "bytes=400-"} # resume from offset + + +def test_range_ignored_by_server_refetches_clean(tmp_path, monkeypatch): + """plan #708 B5: if the server ignores Range (200, not 206) on the resume, we + truncate and start clean — never append into a corrupt double-write.""" + full = b"ABCD" * 250 # 1000 bytes + split = 400 + + class _NoRangeSession: + def __init__(self): + self.n = 0 + + def get(self, url, stream=False, timeout=None, headers=None): + self.n += 1 + if self.n == 1: + return _FlakyResp(full[:split], die=True) + return _FlakyResp(full, status_code=200) # ignores Range, whole file + + monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a: None) + dl = PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, + session=_NoRangeSession(), + ) + outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") + assert outcomes[0].status == "downloaded" + assert outcomes[0].path.read_bytes() == full # clean, not full[:400] + full -- 2.52.0 From bde19944db58cbc769db45f145029dae45044aed Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 11:41:48 -0400 Subject: [PATCH 24/26] test(patreon): fix _BoomSession stub for the B5 headers kwarg test_one_failure_isolated's _BoomSession overrode get() without the headers kwarg _fetch_to_file now passes (B5 Range resume), so the call TypeError'd and both items errored. Add headers=None to match the base fake. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_patreon_downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_patreon_downloader.py b/tests/test_patreon_downloader.py index 8533088..0522807 100644 --- a/tests/test_patreon_downloader.py +++ b/tests/test_patreon_downloader.py @@ -289,7 +289,7 @@ def test_video_permanent_failure_fails_fast(tmp_path, monkeypatch): def test_one_failure_isolated(tmp_path): class _BoomSession(_FakeSession): - def get(self, url, stream=False, timeout=None): + def get(self, url, stream=False, timeout=None, headers=None): self.calls.append(url) if "bad" in url: raise RuntimeError("network boom") -- 2.52.0 From cd43439401cad2086e4aa95e7b845d7157f8eb91 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 12:10:24 -0400 Subject: [PATCH 25/26] =?UTF-8?q?feat(ingester):=20graceful=20mid-walk=20c?= =?UTF-8?q?ancel=20on=20Stop=20=E2=80=94=20B4=20cancel=20(plan=20#708)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owning the walk lets Stop interrupt a live backfill chunk instead of letting it run to its ~14.5-min time-box. ingest_core.run now polls _backfill_state at each page boundary (a short SELECT, never held across the walk) and bails with PARTIAL when an operator Stop has popped it. Latched on the first observed "running" state so a run invoked without one (unit test / stale call) never self-cancels. Progress is already checkpointed per-page, so a restart resumes from the cursor; Stop clears it for a clean reset. No UI change — the existing Stop button now just takes effect immediately. Tests: _still_running reads the state; a latched run bails PARTIAL at the next boundary when the state disappears (only the pre-cancel post ran). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/ingest_core.py | 41 +++++++++++++++++++++++++++ tests/test_patreon_ingester.py | 44 +++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index be418b0..7f7dcab 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -128,6 +128,8 @@ class Ingester: reached_bottom = False budget_hit = False early_out = False + stopped = False # plan #708 B4: operator hit Stop mid-walk + cancel_armed = False # latched once we observe a live "running" state def _result( *, success: bool, return_code: int, @@ -179,6 +181,18 @@ class Ingester: # the live posts count alongside it so the badge climbs DURING # the chunk, not only when it ends. if checkpoint: + # plan #708 B4: an operator Stop pops `_backfill_state` — + # bail at the page boundary (progress already checkpointed) + # before more network work, so the live chunk halts + # promptly instead of running to its time-box. LATCH on the + # first observed "running" state, so a run invoked WITHOUT a + # running state (a unit test, or a stale call) never + # spuriously self-cancels. A short SELECT, never held. + if self._still_running(source_id): + cancel_armed = True + elif cancel_armed: + stopped = True + break self._checkpoint_cursor(source_id, emitted_cursor) self._checkpoint_posts(source_id, posts_base + chunk_new_posts) @@ -283,6 +297,16 @@ class Ingester: # maps it to a typed error. return self._failure_result(exc, _result) + # plan #708 B4: a Stop already popped the backfill state (incl. cursor + + # posts), so don't re-write them — return PARTIAL (reads as "ok/progress", + # the lifecycle no-ops since state is gone) instead of a false "complete". + if stopped: + return _result( + success=False, return_code=-1, + error_type=ErrorType.PARTIAL, + error_message=f"Stopped by operator: {downloaded} file(s) this chunk", + ) + # Final authoritative posts count for the badge — captures the last page # after the last boundary write and the time-box break (plan #704 #5). if checkpoint: @@ -389,6 +413,23 @@ class Ingester: ) session.commit() + def _still_running(self, source_id: int) -> bool: + """True while the source is armed for a deep walk (plan #708 B4). + + An operator Stop (`source_service.stop_backfill`) pops `_backfill_state`, + so a False here means "cancel this chunk now". One short SELECT on its own + session — never held across the walk + ([[db-connection-held-across-subprocess]]).""" + with self.session_factory() as session: + state = session.execute( + text( + "SELECT config_overrides::jsonb ->> '_backfill_state' " + "FROM source WHERE id = :sid" + ), + {"sid": source_id}, + ).scalar_one_or_none() + return state == "running" + def _checkpoint_posts(self, source_id: int, posts: int) -> None: """Persist the live backfill posts-processed count mid-walk (plan #704 #5). diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index 3ca19a5..57a491b 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -382,6 +382,50 @@ async def test_backfill_budget_cut_returns_partial_with_progress( assert result.cursor == "CUR2" +@pytest.mark.asyncio +async def test_still_running_reads_backfill_state(source_id, sync_engine, tmp_path, db): + """plan #708 B4: _still_running reflects the source's `_backfill_state` — the + flag an operator Stop pops to cancel a live chunk.""" + ing = _ingester(sync_engine, tmp_path, _FakeClient([]), _FakeDownloader(tmp_path)) + assert ing._still_running(source_id) is False # absent → not running + src = (await db.execute(select(Source).where(Source.id == source_id))).scalar_one() + src.config_overrides = {"_backfill_state": "running"} + await db.commit() + assert ing._still_running(source_id) is True + + +@pytest.mark.asyncio +async def test_backfill_stops_when_operator_cancels_mid_walk( + source_id, sync_engine, tmp_path, +): + """plan #708 B4: with the running state latched, a later page boundary that + finds it gone (Stop) bails the chunk with PARTIAL instead of finishing.""" + pages = [ + ("CUR2", [("p1", [_media("p1", 1)])]), + ("CUR3", [("p2", [_media("p2", 1)])]), + ("CUR4", [("p3", [_media("p3", 1)])]), + ] + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), downloader) + + # Latch on the page-2 boundary (running), then "Stop" before page 3. + calls = {"n": 0} + + def fake_still_running(_source_id): + calls["n"] += 1 + return calls["n"] == 1 + + ing._still_running = fake_still_running + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="backfill", + ) + assert result.error_type == ErrorType.PARTIAL + assert "Stopped by operator" in result.error_message + assert downloader.download_calls == 1 # only p1 ran; cancelled before p2 + + # --- recovery ------------------------------------------------------------- -- 2.52.0 From e82c2ee57ba272d809381934e92f9c1aa08fbb80 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 12:18:08 -0400 Subject: [PATCH 26/26] =?UTF-8?q?feat(subscriptions):=20dry-run=20backfill?= =?UTF-8?q?=20preview=20=E2=80=94=20B4=20preview=20(plan=20#708)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owning the walk lets an operator gauge "is this source worth a backfill?" before arming one. ingest_core.Ingester.preview walks the first few feed pages and counts media NOT already in the seen/dead ledgers, downloading nothing (read-only). download_backends.preview_source resolves the campaign id + runs it (native-only, mirrors verify_source_credential / run_download); POST /api/sources/{id}/preview returns {total_new, posts_scanned, has_more, sample[]} (409 on auth/drift/unresolvable, 400 for gallery-dl platforms). PatreonClient gains post_meta(post) for the sample's title/date. UI: a Patreon-only Preview button (mdi-eye-outline) on SourceRow + SourceCard opens PreviewDialog — self-fetches with loading / error / empty / result states and a "Start backfill" shortcut. Store action previewSource. Tests: preview counts new media without downloading + samples only posts with new items; page_limit caps the walk + flags has_more. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/api/sources.py | 44 +++++++ backend/app/services/download_backends.py | 48 ++++++++ backend/app/services/ingest_core.py | 61 ++++++++++ backend/app/services/patreon_client.py | 12 ++ .../subscriptions/PreviewDialog.vue | 113 ++++++++++++++++++ .../components/subscriptions/SourceCard.vue | 12 +- .../components/subscriptions/SourceRow.vue | 12 +- .../subscriptions/SubscriptionsTab.vue | 22 ++++ frontend/src/stores/sources.js | 7 ++ tests/test_patreon_ingester.py | 39 ++++++ 10 files changed, 368 insertions(+), 2 deletions(-) create mode 100644 frontend/src/components/subscriptions/PreviewDialog.vue diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index 19d47ce..d5871e1 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -191,6 +191,50 @@ async def set_backfill(source_id: int): return jsonify(record.to_dict()) +@sources_bp.route("//preview", methods=["POST"]) +async def preview_source_endpoint(source_id: int): + """Plan #708 B4: dry-run — count what a backfill WOULD download for a native + platform (Patreon today), without downloading. Walks the first few feed pages + and counts media not already in the seen/dead ledgers. Returns + {total_new, posts_scanned, pages_scanned, has_more, sample[]} or 409 + reason + (unresolvable campaign id / auth / drift). 400 for gallery-dl platforms (no + cheap dry-run — their verify is a slow --simulate).""" + from pathlib import Path + + from ..services.credential_service import CredentialService + from ..services.download_backends import preview_source, uses_native_ingester + from ..tasks._sync_engine import sync_session_factory + from .credentials import _get_crypto + + async with get_session() as session: + rec = await SourceService(session).get(source_id) + if rec is None: + return _bad("not_found", status=404) + if not uses_native_ingester(rec.platform): + return _bad( + "unsupported", + detail="Preview is only available for native-ingester platforms.", + status=400, + ) + cred = CredentialService(session, _get_crypto()) + cookies_path = await cred.get_cookies_path(rec.platform) + + # The walk + ledger reads are sync (run off the request loop); the process + # sync engine is the same one the download task uses. + result = await preview_source( + platform=rec.platform, + url=rec.url, + source_id=source_id, + config_overrides=rec.config_overrides or {}, + cookies_path=str(cookies_path) if cookies_path else None, + images_root=Path("/images"), + sync_session_factory=sync_session_factory(), + ) + if "error" in result: + return _bad("preview_failed", detail=result["error"], status=409) + return jsonify(result) + + @sources_bp.route("//check", methods=["POST"]) async def check_source(source_id: int): """FC-3c: enqueue a download for this source. diff --git a/backend/app/services/download_backends.py b/backend/app/services/download_backends.py index 89b295a..02095f8 100644 --- a/backend/app/services/download_backends.py +++ b/backend/app/services/download_backends.py @@ -144,6 +144,54 @@ async def _run_native_ingester( return dl_result, resolved_campaign_id +async def preview_source( + *, + platform: str, + url: str, + source_id: int, + config_overrides: dict | None, + cookies_path: str | None, + images_root: Path, + sync_session_factory, + page_limit: int = 3, +) -> dict: + """Dry-run preview for a native platform (plan #708 B4): resolve the campaign + id, then walk a few pages counting media not already seen/dead — no download. + + Returns the preview dict (total_new / posts_scanned / pages_scanned / + has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure. + Native-only — the caller gates on `uses_native_ingester`. + """ + import asyncio + + from .patreon_client import PatreonAPIError + + campaign_id, _ = await resolve_campaign_id_for_source( + url, cookies_path, config_overrides or {} + ) + if not campaign_id: + return { + "error": ( + "Couldn't resolve the campaign id from the source URL " + "(cookies expired, or the creator moved/renamed?)." + ) + } + ingester = PatreonIngester( + images_root=images_root, + cookies_path=cookies_path, + session_factory=sync_session_factory, + ) + loop = asyncio.get_running_loop() + try: + result = await loop.run_in_executor( + None, + lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit), + ) + except PatreonAPIError as exc: + return {"error": f"Couldn't preview: {exc}"} + return result + + async def verify_source_credential( *, platform: str, diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 7f7dcab..2b5286c 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -361,6 +361,67 @@ class Ingester: error_type=None, error_message=None, ) + # -- preview (dry-run) ------------------------------------------------- + + def preview( + self, + source_id: int, + campaign_id: str, + *, + page_limit: int = 3, + sample_size: int = 10, + ) -> dict: + """Dry-run (plan #708 B4): walk up to `page_limit` pages and count media + NOT already in the seen/dead ledgers, WITHOUT downloading anything. + + Read-only — only the seen/dead SELECTs touch the DB (short sessions). Lets + an operator gauge "is this source worth a backfill?" cheaply. Returns: + {total_new, posts_scanned, pages_scanned, has_more, + sample: [{title, date, new}, ...]} # sample = posts with new media + A client-level failure (auth/drift) propagates to the caller. + """ + total_new = 0 + posts_scanned = 0 + pages_scanned = 0 + has_more = False + sample: list[dict] = [] + unset = object() + last_page: object = unset + for post, included, page_cursor in self.client.iter_posts( + campaign_id, cursor=None + ): + if page_cursor != last_page: + last_page = page_cursor + pages_scanned += 1 + if pages_scanned > page_limit: + has_more = True + pages_scanned = page_limit + break + posts_scanned += 1 + media = self.client.extract_media(post, included) + if not media: + continue + keys = [self._ledger_key(m) for m in media] + skip = self._seen_keys(source_id, keys) | self._dead_keys(source_id, keys) + new_count = sum(1 for m in media if self._ledger_key(m) not in skip) + total_new += new_count + if new_count > 0 and len(sample) < sample_size: + meta = self.client.post_meta(post) + sample.append( + { + "title": meta.get("title") or "(untitled)", + "date": meta.get("date"), + "new": new_count, + } + ) + return { + "total_new": total_new, + "posts_scanned": posts_scanned, + "pages_scanned": pages_scanned, + "has_more": has_more, + "sample": sample, + } + # -- failure mapping (adapter overrides) ------------------------------- def _failure_result(self, exc: Exception, _result) -> DownloadResult: diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index 69846db..98d8134 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -520,6 +520,18 @@ class PatreonClient: return _dedup_by_filehash(items) + @staticmethod + def post_meta(post: dict) -> dict: + """Title + published date for a post — for the preview sample (plan #708 + B4). Part of the client contract `ingest_core.Ingester.preview` calls.""" + attrs = post.get("attributes") or {} + title = attrs.get("title") + published = attrs.get("published_at") + return { + "title": title if isinstance(title, str) else None, + "date": published if isinstance(published, str) else None, + } + # -- iteration --------------------------------------------------------- def iter_posts( diff --git a/frontend/src/components/subscriptions/PreviewDialog.vue b/frontend/src/components/subscriptions/PreviewDialog.vue new file mode 100644 index 0000000..7639526 --- /dev/null +++ b/frontend/src/components/subscriptions/PreviewDialog.vue @@ -0,0 +1,113 @@ + + + + + diff --git a/frontend/src/components/subscriptions/SourceCard.vue b/frontend/src/components/subscriptions/SourceCard.vue index c0c092f..0e0617b 100644 --- a/frontend/src/components/subscriptions/SourceCard.vue +++ b/frontend/src/components/subscriptions/SourceCard.vue @@ -61,6 +61,16 @@ : 'Backfill full history' }} + + mdi-eye-outline + + Preview — count what a backfill would download (no download) + + + + mdi-eye-outline + + Preview — count what a backfill would download (no download) + + @@ -253,6 +254,7 @@ @check="onCheck" @backfill="onBackfill" @recover="onRecover" + @preview="onPreview" />
No sources yet. Tap + to add one. @@ -268,6 +270,11 @@ @saved="onSourceSaved" /> +
@@ -284,6 +291,7 @@ import SourceCard from './SourceCard.vue' import SourceHealthDot from './SourceHealthDot.vue' import SourceFormDialog from './SourceFormDialog.vue' import ArtistCreateDialog from './ArtistCreateDialog.vue' +import PreviewDialog from './PreviewDialog.vue' import PlatformChip from './PlatformChip.vue' import SchedulerStatusBar from './SchedulerStatusBar.vue' import { formatRelative } from '../../utils/date.js' @@ -334,6 +342,8 @@ const showSourceDialog = ref(false) const editingSource = ref(null) const editingArtist = ref(null) const showArtistDialog = ref(false) +const showPreviewDialog = ref(false) +const previewTarget = ref(null) const artistFilter = computed(() => { const raw = route.query.artist_id @@ -572,6 +582,18 @@ async function onRecover(source) { } } +// Plan #708 B4: open the dry-run preview dialog (it self-fetches on open). +function onPreview(source) { + previewTarget.value = source + showPreviewDialog.value = true +} + +// "Start backfill" from inside the preview dialog → arm it + close. +async function onPreviewBackfill(source) { + showPreviewDialog.value = false + await onBackfill(source) +} + async function checkAll(group) { let ok = 0 let conflict = 0 diff --git a/frontend/src/stores/sources.js b/frontend/src/stores/sources.js index e7916bc..f16ad8e 100644 --- a/frontend/src/stores/sources.js +++ b/frontend/src/stores/sources.js @@ -107,6 +107,12 @@ export const useSourcesStore = defineStore('sources', () => { return body } + // Plan #708 B4: dry-run preview — count what a backfill WOULD download + // (native platforms), without downloading. Read-only, so no cache invalidate. + async function previewSource(id) { + return await api.post(`/api/sources/${id}/preview`) + } + function sourcesByArtistGrouped() { // returns [{artist: {id,name,slug}, sources: [...]}, ...] const arr = byArtist.value.get(null) ?? [] @@ -136,6 +142,7 @@ export const useSourcesStore = defineStore('sources', () => { startBackfill, stopBackfill, recoverSource, + previewSource, findOrCreateArtist, autocompleteArtist, loadScheduleStatus, sourcesByArtistGrouped, diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index 57a491b..95da4bd 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -65,6 +65,9 @@ class _FakeClient: def extract_media(self, post, included_index): return post["_media"] + def post_meta(self, post): + return {"title": post.get("id"), "date": None} + class _FakeDownloader: """Stub PatreonDownloader. Honors the injected is_seen (tier-1); media in @@ -382,6 +385,42 @@ async def test_backfill_budget_cut_returns_partial_with_progress( assert result.cursor == "CUR2" +@pytest.mark.asyncio +async def test_preview_counts_new_media_without_downloading( + source_id, sync_engine, tmp_path, +): + """plan #708 B4: preview walks + counts media not already seen/dead, downloads + nothing, and samples only posts that have new media.""" + m1, m2, m3 = _media("p1", 1), _media("p2", 1), _media("p2", 2) + # Seed m1 as already-seen → only p2's two media are "new". + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1")) + s.commit() + client = _FakeClient([(None, [("p1", [m1]), ("p2", [m2, m3])])]) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.preview(source_id, "c1") + assert result["total_new"] == 2 # p2's m2 + m3 (m1 already seen) + assert result["posts_scanned"] == 2 + assert result["has_more"] is False + assert downloader.download_calls == 0 # dry-run — nothing downloaded + # The sample lists only posts WITH new media (p2), not the all-seen p1. + assert [row["new"] for row in result["sample"]] == [2] + + +@pytest.mark.asyncio +async def test_preview_page_limit_caps_walk(source_id, sync_engine, tmp_path): + """plan #708 B4: preview stops after page_limit pages and flags has_more.""" + pages = [(f"CUR{i}", [(f"p{i}", [_media(f"p{i}", 1)])]) for i in range(6)] + ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path)) + result = ing.preview(source_id, "c1", page_limit=2) + assert result["pages_scanned"] == 2 + assert result["posts_scanned"] == 2 # one post per page, 2 pages + assert result["has_more"] is True + + @pytest.mark.asyncio async def test_still_running_reads_backfill_state(source_id, sync_engine, tmp_path, db): """plan #708 B4: _still_running reflects the source's `_backfill_state` — the -- 2.52.0