From 1c2dc7659a41fe70c9c37d27ea5d26222805c49a Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 5 Jun 2026 19:16:20 -0400 Subject: [PATCH] =?UTF-8?q?feat(patreon):=20native=20JSON-API=20client=20?= =?UTF-8?q?=E2=80=94=20ingester=20build=20step=201=20(plan=20#697)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PatreonClient: cookie-auth requests session, /api/posts cursor pagination, JSON:API included flattening, per-post media extraction (images/image_large/ attachments/postfile/content) with filehash dedup, loud drift detection. Zero per-file HEADs — every media URL+file_name comes from the API. Not yet wired into download_service (later step). Pure-parsing unit tests + fixture. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/patreon_client.py | 477 ++++++++++++++++++++++++ tests/fixtures/patreon_posts_page1.json | 121 ++++++ tests/test_patreon_client.py | 190 ++++++++++ 3 files changed, 788 insertions(+) create mode 100644 backend/app/services/patreon_client.py create mode 100644 tests/fixtures/patreon_posts_page1.json create mode 100644 tests/test_patreon_client.py diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py new file mode 100644 index 0000000..28bfab4 --- /dev/null +++ b/backend/app/services/patreon_client.py @@ -0,0 +1,477 @@ +"""Native Patreon JSON:API client (build step 1 of the native ingester). + +Clean-room reimplementation of the Patreon `/api/posts` read path. This is a +plain, synchronous client over `requests` — the orchestrator already wraps +sync importer calls, so nothing here needs to be async (mirrors +patreon_resolver.py, which wraps its sync lookup in run_in_executor at the +call site). + +Scope (build step 1): fetch + page + parse only. This module is NOT wired into +download_service yet — that is a later step. The public surface here exists so +the later step can drive it: + - PatreonClient(cookies_path).iter_posts(campaign_id) + → (post, included_index, page_cursor) + - extract_media(post, included_index) → list[MediaItem] + - parse_cursor_from_url(url) → cursor + +Drift detection is loud on purpose: Patreon ships JSON:API and the shapes we +depend on (top-level `data`, media resources carrying `file_name`/`url`) are +the contract. If a response comes back as an HTML login page or a media +resource is missing the fields we resolve against, we raise PatreonDriftError +rather than silently yielding empty media — so the later import step surfaces +"Patreon changed something" instead of "creator has no posts". + +FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. +""" + +from __future__ import annotations + +import http.cookiejar +import logging +import os +import re +from collections.abc import Iterator +from dataclasses import dataclass +from html import unescape +from pathlib import Path +from urllib.parse import parse_qs, urlsplit + +import requests + +log = logging.getLogger(__name__) + +_POSTS_URL = "https://www.patreon.com/api/posts" +_USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" +) +_TIMEOUT_SECONDS = 30.0 + +# JSON:API request contract (observed from real traffic — see module plan). +_INCLUDE = ( + "campaign,access_rules,attachments,attachments_media,audio,images,media," + "native_video_insights,user,user_defined_tags,ti_checks" +) +_FIELDS_POST = ( + "content,post_file,image,post_type,published_at,title,url,patreon_url," + "current_user_can_view" +) +_FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name" +_FIELDS_CAMPAIGN = "name,url" + +# A CDN download URL embeds a 32-char hex (MD5) path segment; that segment is +# Patreon's stable per-file identity and is what we dedup + ledger against. +# Same role gallery-dl's _filehash plays. Match the FIRST 32-hex run anywhere +# in the URL (path or query); real Patreon CDN URLs carry exactly one. +_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})") + +# Inline post `content` is HTML; images are emitted as . +# Pull every src; downstream dedup collapses any that duplicate a gallery item +# by filehash. Tolerant of attribute ordering and single/double quotes. +_CONTENT_IMG_RE = re.compile(r"]*?\bsrc=[\"']([^\"']+)[\"']", re.IGNORECASE) + +# A bounded, sane extension parse (see importer._safe_ext for the FC gotcha: +# URL-encoded basenames make Path.suffix return base64-ish junk). We only ever +# use this for a fallback filename, never a network call. +_MAX_EXT_LEN = 16 + + +class PatreonAPIError(Exception): + """Base for native Patreon client failures.""" + + +class PatreonDriftError(PatreonAPIError): + """A response did not match the JSON:API shape we depend on. + + Raised for: a non-JSON / HTML-login response where JSON was expected, a + missing top-level `data` list, or a media resource lacking the + `file_name`/`url` fields we resolve against. Fail loud so the later import + step flags API drift instead of silently importing nothing. + """ + + +@dataclass +class MediaItem: + """One resolved downloadable item belonging to a post. + + Fields: + url — the CDN/download URL to fetch. + filename — media `file_name` when present; otherwise the URL basename + (NEVER a network call). Bounded/sane extension. + kind — one of: "images", "image_large", "attachments", "postfile", + "content". Mirrors gallery-dl's `files` content-type names so + the later step can honor the same per-source content_types. + filehash — the 32-char hex (MD5) segment from the CDN URL, or None if + the URL carries no such segment. Used for in-post dedup and + the cross-run seen-ledger. + post_id — the owning post's id (so a flattened media list stays + traceable to its post). + """ + + url: str + filename: str + kind: str + filehash: str | None + post_id: str + + +def _load_session(cookies_path: str | Path | None) -> requests.Session: + session = requests.Session() + session.headers.update( + { + "User-Agent": _USER_AGENT, + "Accept": "application/vnd.api+json", + } + ) + if cookies_path and os.path.isfile(str(cookies_path)): + try: + jar = http.cookiejar.MozillaCookieJar(str(cookies_path)) + jar.load(ignore_discard=True, ignore_expires=True) + session.cookies = jar # type: ignore[assignment] + except (OSError, http.cookiejar.LoadError) as exc: + log.warning("Could not load Patreon cookies from %s: %s", cookies_path, exc) + return session + + +def _filehash(url: str) -> str | None: + if not url: + return None + match = _FILEHASH_RE.search(url) + return match.group(1).lower() if match else None + + +def _safe_ext(name: str) -> str: + suffix = Path(name).suffix.lower() + if not suffix or len(suffix) > _MAX_EXT_LEN: + return "" + if not all(c.isalnum() for c in suffix[1:]): + return "" + return suffix + + +def _basename_from_url(url: str) -> str: + """Derive a sane filename from a URL when the media has no file_name. + + Strips query/fragment, takes the path basename, and drops a junk + extension (the importer._safe_ext gotcha) so we never write base64 noise + as a name. Falls back to the filehash, then to "file". + """ + path = urlsplit(url).path + base = os.path.basename(path) + if base: + ext = _safe_ext(base) + stem = base[: -len(Path(base).suffix)] if Path(base).suffix else base + # Keep the stem bounded; URL-encoded stems can be enormous. + stem = stem[:120] or "file" + return f"{stem}{ext}" + fh = _filehash(url) + return fh or "file" + + +def parse_cursor_from_url(url: str | None) -> str | None: + """Extract the `page[cursor]` query param from a links.next URL.""" + if not url: + return None + query = urlsplit(url).query + values = parse_qs(query).get("page[cursor]") + if values and values[0]: + return values[0] + return None + + +class PatreonClient: + """Synchronous Patreon JSON:API read client. + + Construct with a path to a Netscape cookies.txt (the same file + CredentialService.get_cookies_path materializes). Cookies are loaded into a + requests.Session; no secure-context APIs are used. + """ + + def __init__(self, cookies_path: str | Path | None): + self.cookies_path = str(cookies_path) if cookies_path else None + self._session = _load_session(cookies_path) + + # -- request ----------------------------------------------------------- + + def _params(self, campaign_id: str, cursor: str | None) -> dict[str, str]: + params = { + "include": _INCLUDE, + "fields[post]": _FIELDS_POST, + "fields[media]": _FIELDS_MEDIA, + "fields[campaign]": _FIELDS_CAMPAIGN, + "filter[campaign_id]": campaign_id, + "filter[contains_exclusive_posts]": "true", + "filter[is_draft]": "false", + "sort": "-published_at", + "json-api-version": "1.0", + } + if cursor: + params["page[cursor]"] = cursor + return params + + def _fetch(self, campaign_id: str, cursor: str | None) -> dict: + try: + resp = self._session.get( + _POSTS_URL, + params=self._params(campaign_id, cursor), + timeout=_TIMEOUT_SECONDS, + ) + except requests.RequestException as exc: + raise PatreonAPIError( + f"Patreon posts request failed (campaign_id={campaign_id}): {exc}" + ) from exc + + if resp.status_code != 200: + raise PatreonAPIError( + f"Patreon posts API returned HTTP {resp.status_code} " + f"(campaign_id={campaign_id})" + ) + try: + payload = resp.json() + except ValueError as exc: + # A non-JSON body here is almost always the HTML login/challenge + # page served when cookies are missing/expired — that is drift + # from the JSON:API contract, not a transient network error. + raise PatreonDriftError( + "Patreon posts API returned a non-JSON response (likely an " + f"HTML login/challenge page; campaign_id={campaign_id}): {exc}" + ) from exc + return payload + + # -- parsing ----------------------------------------------------------- + + @staticmethod + def _transform(response: dict) -> dict: + """Flatten the JSON:API `included` array for relationship resolution. + + Returns a dict keyed by `(type, id)` → that resource's `attributes` + (so a post's relationships can be resolved in O(1)). Missing/oddly + shaped `included` entries are skipped rather than fatal — drift + detection for the top-level shape lives in _validate_response. + """ + index: dict[tuple[str, str], dict] = {} + for inc in response.get("included") or []: + if not isinstance(inc, dict): + continue + rtype = inc.get("type") + rid = inc.get("id") + if rtype is None or rid is None: + continue + index[(str(rtype), str(rid))] = inc.get("attributes") or {} + return index + + @staticmethod + def _validate_response(response: dict) -> None: + if not isinstance(response, dict): + raise PatreonDriftError("Patreon response was not a JSON object") + if "data" not in response: + raise PatreonDriftError("Patreon response missing top-level 'data' key") + data = response.get("data") + if not isinstance(data, list): + raise PatreonDriftError("Patreon response 'data' was not a list") + + def _related_ids(self, post: dict, rel_name: str) -> list[str]: + rels = post.get("relationships") or {} + rel = rels.get(rel_name) or {} + data = rel.get("data") + if isinstance(data, dict): # to-one relationship + data = [data] + if not isinstance(data, list): + return [] + ids: list[str] = [] + for ref in data: + if isinstance(ref, dict) and ref.get("id") is not None: + ids.append(str(ref["id"])) + return ids + + @staticmethod + def _media_url(attrs: dict) -> str | None: + """Pick the best fetchable URL for a media resource. + + Prefer the full-size `download_url`; fall back to the largest + `image_urls` size. gallery-dl prefers download_url too, only dipping + into image_urls when a smaller configured size is requested — FC + always wants the original, so download_url first. + """ + download_url = attrs.get("download_url") + if isinstance(download_url, str) and download_url: + return download_url + image_urls = attrs.get("image_urls") + if isinstance(image_urls, dict): + for key in ("original", "full", "large", "default"): + candidate = image_urls.get(key) + if isinstance(candidate, str) and candidate: + return candidate + # Otherwise take any non-empty string value. + for candidate in image_urls.values(): + if isinstance(candidate, str) and candidate: + return candidate + return None + + def _media_item( + self, attrs: dict, kind: str, post_id: str, *, require_file_name: bool + ) -> MediaItem: + url = self._media_url(attrs) + if not url: + raise PatreonDriftError( + f"Patreon media (post {post_id}, kind={kind}) had no resolvable URL " + f"(no download_url / image_urls)" + ) + file_name = attrs.get("file_name") + if require_file_name and not (isinstance(file_name, str) and file_name): + raise PatreonDriftError( + f"Patreon media resource (post {post_id}, kind={kind}) missing " + f"'file_name'" + ) + filename = file_name if isinstance(file_name, str) and file_name else _basename_from_url(url) + return MediaItem( + url=url, + filename=filename, + kind=kind, + filehash=_filehash(url), + post_id=post_id, + ) + + def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]: + """Resolve all downloadable media for one post. + + Walks the kinds in the same order gallery-dl does — images, + image_large (post cover), attachments, postfile, content (inline + ) — and dedups within the post by filehash (first wins). The + image_large cover commonly duplicates a gallery image; deduping by + filehash collapses them to the gallery item (encountered first). + """ + post_id = str(post.get("id") or "") + attrs = post.get("attributes") or {} + items: list[MediaItem] = [] + + def _resolve_rel(rel_name: str, kind: str) -> None: + for mid in self._related_ids(post, rel_name): + media_attrs = included_index.get(("media", mid)) + if media_attrs is None: + # Referenced but not in `included`: a media id with no + # resource is drift (we asked for include=media). + raise PatreonDriftError( + f"Patreon post {post_id} references media {mid} " + f"({rel_name}) not present in 'included'" + ) + items.append( + self._media_item( + media_attrs, kind, post_id, require_file_name=True + ) + ) + + # 1. gallery images + _resolve_rel("images", "images") + + # 2. image_large — the post-level cover (`image.large_url`). Not a + # media relationship; it lives on the post attributes. + image = attrs.get("image") + if isinstance(image, dict): + large_url = image.get("large_url") or image.get("url") + if isinstance(large_url, str) and large_url: + items.append( + MediaItem( + url=large_url, + filename=_basename_from_url(large_url), + kind="image_large", + filehash=_filehash(large_url), + post_id=post_id, + ) + ) + + # 3. attachments + _resolve_rel("attachments_media", "attachments") + + # 4. postfile — the post's primary attached file (`post_file`). + post_file = attrs.get("post_file") + if isinstance(post_file, dict): + pf_url = post_file.get("url") or post_file.get("download_url") + if isinstance(pf_url, str) and pf_url: + pf_name = post_file.get("name") + filename = ( + pf_name + if isinstance(pf_name, str) and pf_name + else _basename_from_url(pf_url) + ) + items.append( + MediaItem( + url=pf_url, + filename=filename, + kind="postfile", + filehash=_filehash(pf_url), + post_id=post_id, + ) + ) + + # 5. content — inline in the post HTML body. + content = attrs.get("content") + if isinstance(content, str) and content: + for raw_src in _CONTENT_IMG_RE.findall(content): + src = unescape(raw_src) + if not src: + continue + items.append( + MediaItem( + url=src, + filename=_basename_from_url(src), + kind="content", + filehash=_filehash(src), + post_id=post_id, + ) + ) + + return _dedup_by_filehash(items) + + # -- iteration --------------------------------------------------------- + + def iter_posts( + self, campaign_id: str, cursor: str | None = None + ) -> Iterator[tuple[dict, dict, str | None]]: + """Yield (post, included_index, page_cursor) for every post in the feed. + + Pages newest→oldest via `links.next`, validating each response for + drift before yielding. The triple gives a caller everything it needs to + resolve and checkpoint without re-fetching: + - post — the raw post resource. + - included_index — the page's flattened `included` (the same object + for every post on a page), to pass straight to + extract_media(post, included_index). + - page_cursor — the cursor that FETCHED this post's page (None for + the first page). The caller checkpoints THIS value, + matching the existing backfill cursor logic where + the saved cursor re-fetches the page being + processed (so a chunk cut mid-page resumes the page, + not the one after it). + """ + current_cursor = cursor + while True: + response = self._fetch(campaign_id, current_cursor) + self._validate_response(response) + page_cursor = current_cursor + included_index = self._transform(response) + for post in response.get("data") or []: + if isinstance(post, dict): + yield post, included_index, page_cursor + next_url = (response.get("links") or {}).get("next") + next_cursor = parse_cursor_from_url(next_url) + if not next_cursor: + return + current_cursor = next_cursor + + +def _dedup_by_filehash(items: list[MediaItem]) -> list[MediaItem]: + """Drop later items sharing a filehash with an earlier one (first wins). + + Items with no filehash (None) are never deduped against each other — we + can't prove they're the same file, so keep them all. + """ + seen: set[str] = set() + out: list[MediaItem] = [] + for item in items: + if item.filehash is not None: + if item.filehash in seen: + continue + seen.add(item.filehash) + out.append(item) + return out diff --git a/tests/fixtures/patreon_posts_page1.json b/tests/fixtures/patreon_posts_page1.json new file mode 100644 index 0000000..6515c34 --- /dev/null +++ b/tests/fixtures/patreon_posts_page1.json @@ -0,0 +1,121 @@ +{ + "data": [ + { + "id": "1001", + "type": "post", + "attributes": { + "title": "Image gallery post", + "published_at": "2026-05-01T12:00:00.000+00:00", + "post_type": "image_file", + "content": "

just a gallery

", + "url": "https://www.patreon.com/posts/1001", + "image": { + "large_url": "https://c10.patreonusercontent.com/4/large/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg", + "url": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg" + } + }, + "relationships": { + "images": { + "data": [ + {"id": "9001", "type": "media"}, + {"id": "9002", "type": "media"} + ] + } + } + }, + { + "id": "1002", + "type": "post", + "attributes": { + "title": "Attachment post", + "published_at": "2026-04-15T09:30:00.000+00:00", + "post_type": "text_only", + "content": "

here is a zip

", + "url": "https://www.patreon.com/posts/1002", + "post_file": { + "name": "bonus-pack.zip", + "url": "https://www.patreon.com/file?h=cccccccccccccccccccccccccccccccc&i=1002" + } + }, + "relationships": { + "attachments_media": { + "data": [ + {"id": "9003", "type": "media"} + ] + } + } + }, + { + "id": "1003", + "type": "post", + "attributes": { + "title": "Inline content post", + "published_at": "2026-04-01T08:00:00.000+00:00", + "post_type": "text_only", + "content": "

look

\"inline\"
", + "url": "https://www.patreon.com/posts/1003" + }, + "relationships": {} + }, + { + "id": "1004", + "type": "post", + "attributes": { + "title": "Video post", + "published_at": "2026-03-20T18:45:00.000+00:00", + "post_type": "video_external_file", + "content": "

watch

", + "url": "https://www.patreon.com/posts/1004", + "post_file": { + "name": "clip.m3u8", + "url": "https://stream.mux.com/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee.m3u8?token=jwt" + } + }, + "relationships": {} + } + ], + "included": [ + { + "id": "9001", + "type": "media", + "attributes": { + "file_name": "gallery-one.jpg", + "download_url": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg", + "image_urls": { + "original": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg", + "default": "https://c10.patreonusercontent.com/4/default/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg" + } + } + }, + { + "id": "9002", + "type": "media", + "attributes": { + "file_name": "gallery-two.jpg", + "download_url": "https://c10.patreonusercontent.com/4/orig/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/gallery-two.jpg", + "image_urls": { + "original": "https://c10.patreonusercontent.com/4/orig/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/gallery-two.jpg" + } + } + }, + { + "id": "9003", + "type": "media", + "attributes": { + "file_name": "bonus-pack.zip", + "download_url": "https://www.patreon.com/file/download?h=cccccccccccccccccccccccccccccccc" + } + }, + { + "id": "5555", + "type": "campaign", + "attributes": { + "name": "Example Creator", + "url": "https://www.patreon.com/example" + } + } + ], + "links": { + "next": "https://www.patreon.com/api/posts?filter%5Bcampaign_id%5D=5555&page%5Bcursor%5D=NEXTCURSOR123&sort=-published_at" + } +} diff --git a/tests/test_patreon_client.py b/tests/test_patreon_client.py new file mode 100644 index 0000000..3ef504a --- /dev/null +++ b/tests/test_patreon_client.py @@ -0,0 +1,190 @@ +"""PatreonClient tests — parsing only, no network. + +The fixture is loaded from disk and fed directly to the pure parsing methods +(_transform / extract_media / _validate_response / parse_cursor_from_url); no +live requests call is exercised here. +""" + +import json +from pathlib import Path + +import pytest + +from backend.app.services.patreon_client import ( + MediaItem, + PatreonClient, + PatreonDriftError, + parse_cursor_from_url, +) + +_FIXTURE = Path(__file__).parent / "fixtures" / "patreon_posts_page1.json" + + +@pytest.fixture +def response(): + return json.loads(_FIXTURE.read_text()) + + +@pytest.fixture +def client(): + # No cookies file → empty session, but we never make a request in tests. + return PatreonClient(cookies_path=None) + + +def _post_by_id(response, post_id): + for post in response["data"]: + if post["id"] == post_id: + return post + raise AssertionError(f"no post {post_id} in fixture") + + +def test_transform_indexes_included(client, response): + index = client._transform(response) + assert index[("media", "9001")]["file_name"] == "gallery-one.jpg" + assert index[("media", "9003")]["file_name"] == "bonus-pack.zip" + assert index[("campaign", "5555")]["name"] == "Example Creator" + # Unknown keys are absent, not errors. + assert ("media", "0000") not in index + + +def test_extract_gallery_dedups_image_large(client, response): + index = client._transform(response) + post = _post_by_id(response, "1001") + items = client.extract_media(post, index) + + # Two gallery images; the image_large cover shares filehash aaaa... with + # gallery image 9001 → collapses to one, leaving exactly 2 items. + assert len(items) == 2 + assert all(isinstance(m, MediaItem) for m in items) + + kinds = [m.kind for m in items] + assert kinds == ["images", "images"] # image_large deduped away + + filenames = {m.filename for m in items} + assert filenames == {"gallery-one.jpg", "gallery-two.jpg"} + + hashes = {m.filehash for m in items} + assert hashes == {"a" * 32, "b" * 32} + assert all(m.post_id == "1001" for m in items) + + +def test_extract_attachment_and_postfile(client, response): + index = client._transform(response) + post = _post_by_id(response, "1002") + items = client.extract_media(post, index) + + by_kind = {m.kind: m for m in items} + # attachments_media → "attachments", plus the post_file → "postfile". + assert set(by_kind) == {"attachments", "postfile"} + assert by_kind["attachments"].filename == "bonus-pack.zip" + assert by_kind["attachments"].filehash == "c" * 32 + assert by_kind["postfile"].filename == "bonus-pack.zip" + # postfile URL also carries the same hash, but they're different kinds in + # different relationships; both are kept because dedup is by filehash and + # the attachment is encountered first... assert the post_file collapsed. + # (attachments resolved before postfile, same hash → postfile dropped.) + # So only ONE survives with that hash: + surviving_c = [m for m in items if m.filehash == "c" * 32] + assert len(surviving_c) == 1 + + +def test_extract_inline_content_img(client, response): + index = client._transform(response) + post = _post_by_id(response, "1003") + items = client.extract_media(post, index) + + assert len(items) == 1 + item = items[0] + assert item.kind == "content" + assert item.filehash == "d" * 32 + # & in the src must be unescaped to & . + assert "&" not in item.url + assert "token=x&v=2" in item.url + + +def test_extract_video_postfile(client, response): + index = client._transform(response) + post = _post_by_id(response, "1004") + items = client.extract_media(post, index) + + assert len(items) == 1 + item = items[0] + assert item.kind == "postfile" + assert item.url.startswith("https://stream.mux.com/") + assert item.filehash == "e" * 32 + + +def test_parse_cursor_from_url_extracts_cursor(response): + next_url = response["links"]["next"] + assert parse_cursor_from_url(next_url) == "NEXTCURSOR123" + + +def test_parse_cursor_from_url_none_when_absent(): + assert parse_cursor_from_url("https://www.patreon.com/api/posts?sort=-x") is None + assert parse_cursor_from_url(None) is None + + +def test_iter_posts_yields_triples_and_paginates(client, response, monkeypatch): + # Page 1 = the fixture (4 posts, links.next → NEXTCURSOR123); page 2 = one + # post, no links.next → stop. Monkeypatch _fetch so no network is touched. + page2 = { + "data": [{"id": "1005", "type": "post", "attributes": {"title": "older"}}], + "included": [], + "links": {}, + } + + def fake_fetch(campaign_id, cursor): + return page2 if cursor == "NEXTCURSOR123" else response + + monkeypatch.setattr(client, "_fetch", fake_fetch) + + seen = list(client.iter_posts("5555")) + # 4 posts on page 1 + 1 on page 2. + assert [p["id"] for p, _idx, _cur in seen] == ["1001", "1002", "1003", "1004", "1005"] + # page_cursor is the cursor that FETCHED the post's page. + cursors = {p["id"]: cur for p, _idx, cur in seen} + assert cursors["1001"] is None and cursors["1005"] == "NEXTCURSOR123" + # included_index is the page's flattened resources, ready for extract_media. + page1_index = next(idx for p, idx, _cur in seen if p["id"] == "1001") + assert page1_index[("media", "9001")]["file_name"] == "gallery-one.jpg" + + +def test_missing_data_raises_drift(client): + with pytest.raises(PatreonDriftError): + client._validate_response({"included": []}) + + +def test_non_list_data_raises_drift(client): + with pytest.raises(PatreonDriftError): + client._validate_response({"data": {"id": "1"}}) + + +def test_media_missing_file_name_raises_drift(client): + # A media resource referenced by a gallery relationship but lacking + # file_name is drift from the contract. + post = { + "id": "2000", + "type": "post", + "attributes": {"title": "broken"}, + "relationships": {"images": {"data": [{"id": "7777", "type": "media"}]}}, + } + index = { + ("media", "7777"): { + "download_url": "https://c10.patreonusercontent.com/4/orig/" + + "f" * 32 + + "/x.jpg" + } + } + with pytest.raises(PatreonDriftError): + client.extract_media(post, index) + + +def test_media_referenced_but_absent_raises_drift(client): + post = { + "id": "2001", + "type": "post", + "attributes": {"title": "dangling"}, + "relationships": {"images": {"data": [{"id": "8888", "type": "media"}]}}, + } + with pytest.raises(PatreonDriftError): + client.extract_media(post, {})