feat(patreon): native JSON-API client — ingester build step 1 (plan #697)
PatreonClient: cookie-auth requests session, /api/posts cursor pagination, JSON:API included flattening, per-post media extraction (images/image_large/ attachments/postfile/content) with filehash dedup, loud drift detection. Zero per-file HEADs — every media URL+file_name comes from the API. Not yet wired into download_service (later step). Pure-parsing unit tests + fixture. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,477 @@
|
||||
"""Native Patreon JSON:API client (build step 1 of the native ingester).
|
||||
|
||||
Clean-room reimplementation of the Patreon `/api/posts` read path. This is a
|
||||
plain, synchronous client over `requests` — the orchestrator already wraps
|
||||
sync importer calls, so nothing here needs to be async (mirrors
|
||||
patreon_resolver.py, which wraps its sync lookup in run_in_executor at the
|
||||
call site).
|
||||
|
||||
Scope (build step 1): fetch + page + parse only. This module is NOT wired into
|
||||
download_service yet — that is a later step. The public surface here exists so
|
||||
the later step can drive it:
|
||||
- PatreonClient(cookies_path).iter_posts(campaign_id)
|
||||
→ (post, included_index, page_cursor)
|
||||
- extract_media(post, included_index) → list[MediaItem]
|
||||
- parse_cursor_from_url(url) → cursor
|
||||
|
||||
Drift detection is loud on purpose: Patreon ships JSON:API and the shapes we
|
||||
depend on (top-level `data`, media resources carrying `file_name`/`url`) are
|
||||
the contract. If a response comes back as an HTML login page or a media
|
||||
resource is missing the fields we resolve against, we raise PatreonDriftError
|
||||
rather than silently yielding empty media — so the later import step surfaces
|
||||
"Patreon changed something" instead of "creator has no posts".
|
||||
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.cookiejar
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from html import unescape
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_POSTS_URL = "https://www.patreon.com/api/posts"
|
||||
_USER_AGENT = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||||
)
|
||||
_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
# JSON:API request contract (observed from real traffic — see module plan).
|
||||
_INCLUDE = (
|
||||
"campaign,access_rules,attachments,attachments_media,audio,images,media,"
|
||||
"native_video_insights,user,user_defined_tags,ti_checks"
|
||||
)
|
||||
_FIELDS_POST = (
|
||||
"content,post_file,image,post_type,published_at,title,url,patreon_url,"
|
||||
"current_user_can_view"
|
||||
)
|
||||
_FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name"
|
||||
_FIELDS_CAMPAIGN = "name,url"
|
||||
|
||||
# A CDN download URL embeds a 32-char hex (MD5) path segment; that segment is
|
||||
# Patreon's stable per-file identity and is what we dedup + ledger against.
|
||||
# Same role gallery-dl's _filehash plays. Match the FIRST 32-hex run anywhere
|
||||
# in the URL (path or query); real Patreon CDN URLs carry exactly one.
|
||||
_FILEHASH_RE = re.compile(r"([0-9a-fA-F]{32})")
|
||||
|
||||
# Inline post `content` is HTML; images are emitted as <img ... src="...">.
|
||||
# Pull every src; downstream dedup collapses any that duplicate a gallery item
|
||||
# by filehash. Tolerant of attribute ordering and single/double quotes.
|
||||
_CONTENT_IMG_RE = re.compile(r"<img\b[^>]*?\bsrc=[\"']([^\"']+)[\"']", re.IGNORECASE)
|
||||
|
||||
# A bounded, sane extension parse (see importer._safe_ext for the FC gotcha:
|
||||
# URL-encoded basenames make Path.suffix return base64-ish junk). We only ever
|
||||
# use this for a fallback filename, never a network call.
|
||||
_MAX_EXT_LEN = 16
|
||||
|
||||
|
||||
class PatreonAPIError(Exception):
|
||||
"""Base for native Patreon client failures."""
|
||||
|
||||
|
||||
class PatreonDriftError(PatreonAPIError):
|
||||
"""A response did not match the JSON:API shape we depend on.
|
||||
|
||||
Raised for: a non-JSON / HTML-login response where JSON was expected, a
|
||||
missing top-level `data` list, or a media resource lacking the
|
||||
`file_name`/`url` fields we resolve against. Fail loud so the later import
|
||||
step flags API drift instead of silently importing nothing.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class MediaItem:
|
||||
"""One resolved downloadable item belonging to a post.
|
||||
|
||||
Fields:
|
||||
url — the CDN/download URL to fetch.
|
||||
filename — media `file_name` when present; otherwise the URL basename
|
||||
(NEVER a network call). Bounded/sane extension.
|
||||
kind — one of: "images", "image_large", "attachments", "postfile",
|
||||
"content". Mirrors gallery-dl's `files` content-type names so
|
||||
the later step can honor the same per-source content_types.
|
||||
filehash — the 32-char hex (MD5) segment from the CDN URL, or None if
|
||||
the URL carries no such segment. Used for in-post dedup and
|
||||
the cross-run seen-ledger.
|
||||
post_id — the owning post's id (so a flattened media list stays
|
||||
traceable to its post).
|
||||
"""
|
||||
|
||||
url: str
|
||||
filename: str
|
||||
kind: str
|
||||
filehash: str | None
|
||||
post_id: str
|
||||
|
||||
|
||||
def _load_session(cookies_path: str | Path | None) -> requests.Session:
|
||||
session = requests.Session()
|
||||
session.headers.update(
|
||||
{
|
||||
"User-Agent": _USER_AGENT,
|
||||
"Accept": "application/vnd.api+json",
|
||||
}
|
||||
)
|
||||
if cookies_path and os.path.isfile(str(cookies_path)):
|
||||
try:
|
||||
jar = http.cookiejar.MozillaCookieJar(str(cookies_path))
|
||||
jar.load(ignore_discard=True, ignore_expires=True)
|
||||
session.cookies = jar # type: ignore[assignment]
|
||||
except (OSError, http.cookiejar.LoadError) as exc:
|
||||
log.warning("Could not load Patreon cookies from %s: %s", cookies_path, exc)
|
||||
return session
|
||||
|
||||
|
||||
def _filehash(url: str) -> str | None:
|
||||
if not url:
|
||||
return None
|
||||
match = _FILEHASH_RE.search(url)
|
||||
return match.group(1).lower() if match else None
|
||||
|
||||
|
||||
def _safe_ext(name: str) -> str:
|
||||
suffix = Path(name).suffix.lower()
|
||||
if not suffix or len(suffix) > _MAX_EXT_LEN:
|
||||
return ""
|
||||
if not all(c.isalnum() for c in suffix[1:]):
|
||||
return ""
|
||||
return suffix
|
||||
|
||||
|
||||
def _basename_from_url(url: str) -> str:
|
||||
"""Derive a sane filename from a URL when the media has no file_name.
|
||||
|
||||
Strips query/fragment, takes the path basename, and drops a junk
|
||||
extension (the importer._safe_ext gotcha) so we never write base64 noise
|
||||
as a name. Falls back to the filehash, then to "file".
|
||||
"""
|
||||
path = urlsplit(url).path
|
||||
base = os.path.basename(path)
|
||||
if base:
|
||||
ext = _safe_ext(base)
|
||||
stem = base[: -len(Path(base).suffix)] if Path(base).suffix else base
|
||||
# Keep the stem bounded; URL-encoded stems can be enormous.
|
||||
stem = stem[:120] or "file"
|
||||
return f"{stem}{ext}"
|
||||
fh = _filehash(url)
|
||||
return fh or "file"
|
||||
|
||||
|
||||
def parse_cursor_from_url(url: str | None) -> str | None:
|
||||
"""Extract the `page[cursor]` query param from a links.next URL."""
|
||||
if not url:
|
||||
return None
|
||||
query = urlsplit(url).query
|
||||
values = parse_qs(query).get("page[cursor]")
|
||||
if values and values[0]:
|
||||
return values[0]
|
||||
return None
|
||||
|
||||
|
||||
class PatreonClient:
|
||||
"""Synchronous Patreon JSON:API read client.
|
||||
|
||||
Construct with a path to a Netscape cookies.txt (the same file
|
||||
CredentialService.get_cookies_path materializes). Cookies are loaded into a
|
||||
requests.Session; no secure-context APIs are used.
|
||||
"""
|
||||
|
||||
def __init__(self, cookies_path: str | Path | None):
|
||||
self.cookies_path = str(cookies_path) if cookies_path else None
|
||||
self._session = _load_session(cookies_path)
|
||||
|
||||
# -- request -----------------------------------------------------------
|
||||
|
||||
def _params(self, campaign_id: str, cursor: str | None) -> dict[str, str]:
|
||||
params = {
|
||||
"include": _INCLUDE,
|
||||
"fields[post]": _FIELDS_POST,
|
||||
"fields[media]": _FIELDS_MEDIA,
|
||||
"fields[campaign]": _FIELDS_CAMPAIGN,
|
||||
"filter[campaign_id]": campaign_id,
|
||||
"filter[contains_exclusive_posts]": "true",
|
||||
"filter[is_draft]": "false",
|
||||
"sort": "-published_at",
|
||||
"json-api-version": "1.0",
|
||||
}
|
||||
if cursor:
|
||||
params["page[cursor]"] = cursor
|
||||
return params
|
||||
|
||||
def _fetch(self, campaign_id: str, cursor: str | None) -> dict:
|
||||
try:
|
||||
resp = self._session.get(
|
||||
_POSTS_URL,
|
||||
params=self._params(campaign_id, cursor),
|
||||
timeout=_TIMEOUT_SECONDS,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
raise PatreonAPIError(
|
||||
f"Patreon posts request failed (campaign_id={campaign_id}): {exc}"
|
||||
) from exc
|
||||
|
||||
if resp.status_code != 200:
|
||||
raise PatreonAPIError(
|
||||
f"Patreon posts API returned HTTP {resp.status_code} "
|
||||
f"(campaign_id={campaign_id})"
|
||||
)
|
||||
try:
|
||||
payload = resp.json()
|
||||
except ValueError as exc:
|
||||
# A non-JSON body here is almost always the HTML login/challenge
|
||||
# page served when cookies are missing/expired — that is drift
|
||||
# from the JSON:API contract, not a transient network error.
|
||||
raise PatreonDriftError(
|
||||
"Patreon posts API returned a non-JSON response (likely an "
|
||||
f"HTML login/challenge page; campaign_id={campaign_id}): {exc}"
|
||||
) from exc
|
||||
return payload
|
||||
|
||||
# -- parsing -----------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _transform(response: dict) -> dict:
|
||||
"""Flatten the JSON:API `included` array for relationship resolution.
|
||||
|
||||
Returns a dict keyed by `(type, id)` → that resource's `attributes`
|
||||
(so a post's relationships can be resolved in O(1)). Missing/oddly
|
||||
shaped `included` entries are skipped rather than fatal — drift
|
||||
detection for the top-level shape lives in _validate_response.
|
||||
"""
|
||||
index: dict[tuple[str, str], dict] = {}
|
||||
for inc in response.get("included") or []:
|
||||
if not isinstance(inc, dict):
|
||||
continue
|
||||
rtype = inc.get("type")
|
||||
rid = inc.get("id")
|
||||
if rtype is None or rid is None:
|
||||
continue
|
||||
index[(str(rtype), str(rid))] = inc.get("attributes") or {}
|
||||
return index
|
||||
|
||||
@staticmethod
|
||||
def _validate_response(response: dict) -> None:
|
||||
if not isinstance(response, dict):
|
||||
raise PatreonDriftError("Patreon response was not a JSON object")
|
||||
if "data" not in response:
|
||||
raise PatreonDriftError("Patreon response missing top-level 'data' key")
|
||||
data = response.get("data")
|
||||
if not isinstance(data, list):
|
||||
raise PatreonDriftError("Patreon response 'data' was not a list")
|
||||
|
||||
def _related_ids(self, post: dict, rel_name: str) -> list[str]:
|
||||
rels = post.get("relationships") or {}
|
||||
rel = rels.get(rel_name) or {}
|
||||
data = rel.get("data")
|
||||
if isinstance(data, dict): # to-one relationship
|
||||
data = [data]
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
ids: list[str] = []
|
||||
for ref in data:
|
||||
if isinstance(ref, dict) and ref.get("id") is not None:
|
||||
ids.append(str(ref["id"]))
|
||||
return ids
|
||||
|
||||
@staticmethod
|
||||
def _media_url(attrs: dict) -> str | None:
|
||||
"""Pick the best fetchable URL for a media resource.
|
||||
|
||||
Prefer the full-size `download_url`; fall back to the largest
|
||||
`image_urls` size. gallery-dl prefers download_url too, only dipping
|
||||
into image_urls when a smaller configured size is requested — FC
|
||||
always wants the original, so download_url first.
|
||||
"""
|
||||
download_url = attrs.get("download_url")
|
||||
if isinstance(download_url, str) and download_url:
|
||||
return download_url
|
||||
image_urls = attrs.get("image_urls")
|
||||
if isinstance(image_urls, dict):
|
||||
for key in ("original", "full", "large", "default"):
|
||||
candidate = image_urls.get(key)
|
||||
if isinstance(candidate, str) and candidate:
|
||||
return candidate
|
||||
# Otherwise take any non-empty string value.
|
||||
for candidate in image_urls.values():
|
||||
if isinstance(candidate, str) and candidate:
|
||||
return candidate
|
||||
return None
|
||||
|
||||
def _media_item(
|
||||
self, attrs: dict, kind: str, post_id: str, *, require_file_name: bool
|
||||
) -> MediaItem:
|
||||
url = self._media_url(attrs)
|
||||
if not url:
|
||||
raise PatreonDriftError(
|
||||
f"Patreon media (post {post_id}, kind={kind}) had no resolvable URL "
|
||||
f"(no download_url / image_urls)"
|
||||
)
|
||||
file_name = attrs.get("file_name")
|
||||
if require_file_name and not (isinstance(file_name, str) and file_name):
|
||||
raise PatreonDriftError(
|
||||
f"Patreon media resource (post {post_id}, kind={kind}) missing "
|
||||
f"'file_name'"
|
||||
)
|
||||
filename = file_name if isinstance(file_name, str) and file_name else _basename_from_url(url)
|
||||
return MediaItem(
|
||||
url=url,
|
||||
filename=filename,
|
||||
kind=kind,
|
||||
filehash=_filehash(url),
|
||||
post_id=post_id,
|
||||
)
|
||||
|
||||
def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]:
|
||||
"""Resolve all downloadable media for one post.
|
||||
|
||||
Walks the kinds in the same order gallery-dl does — images,
|
||||
image_large (post cover), attachments, postfile, content (inline
|
||||
<img>) — and dedups within the post by filehash (first wins). The
|
||||
image_large cover commonly duplicates a gallery image; deduping by
|
||||
filehash collapses them to the gallery item (encountered first).
|
||||
"""
|
||||
post_id = str(post.get("id") or "")
|
||||
attrs = post.get("attributes") or {}
|
||||
items: list[MediaItem] = []
|
||||
|
||||
def _resolve_rel(rel_name: str, kind: str) -> None:
|
||||
for mid in self._related_ids(post, rel_name):
|
||||
media_attrs = included_index.get(("media", mid))
|
||||
if media_attrs is None:
|
||||
# Referenced but not in `included`: a media id with no
|
||||
# resource is drift (we asked for include=media).
|
||||
raise PatreonDriftError(
|
||||
f"Patreon post {post_id} references media {mid} "
|
||||
f"({rel_name}) not present in 'included'"
|
||||
)
|
||||
items.append(
|
||||
self._media_item(
|
||||
media_attrs, kind, post_id, require_file_name=True
|
||||
)
|
||||
)
|
||||
|
||||
# 1. gallery images
|
||||
_resolve_rel("images", "images")
|
||||
|
||||
# 2. image_large — the post-level cover (`image.large_url`). Not a
|
||||
# media relationship; it lives on the post attributes.
|
||||
image = attrs.get("image")
|
||||
if isinstance(image, dict):
|
||||
large_url = image.get("large_url") or image.get("url")
|
||||
if isinstance(large_url, str) and large_url:
|
||||
items.append(
|
||||
MediaItem(
|
||||
url=large_url,
|
||||
filename=_basename_from_url(large_url),
|
||||
kind="image_large",
|
||||
filehash=_filehash(large_url),
|
||||
post_id=post_id,
|
||||
)
|
||||
)
|
||||
|
||||
# 3. attachments
|
||||
_resolve_rel("attachments_media", "attachments")
|
||||
|
||||
# 4. postfile — the post's primary attached file (`post_file`).
|
||||
post_file = attrs.get("post_file")
|
||||
if isinstance(post_file, dict):
|
||||
pf_url = post_file.get("url") or post_file.get("download_url")
|
||||
if isinstance(pf_url, str) and pf_url:
|
||||
pf_name = post_file.get("name")
|
||||
filename = (
|
||||
pf_name
|
||||
if isinstance(pf_name, str) and pf_name
|
||||
else _basename_from_url(pf_url)
|
||||
)
|
||||
items.append(
|
||||
MediaItem(
|
||||
url=pf_url,
|
||||
filename=filename,
|
||||
kind="postfile",
|
||||
filehash=_filehash(pf_url),
|
||||
post_id=post_id,
|
||||
)
|
||||
)
|
||||
|
||||
# 5. content — inline <img> in the post HTML body.
|
||||
content = attrs.get("content")
|
||||
if isinstance(content, str) and content:
|
||||
for raw_src in _CONTENT_IMG_RE.findall(content):
|
||||
src = unescape(raw_src)
|
||||
if not src:
|
||||
continue
|
||||
items.append(
|
||||
MediaItem(
|
||||
url=src,
|
||||
filename=_basename_from_url(src),
|
||||
kind="content",
|
||||
filehash=_filehash(src),
|
||||
post_id=post_id,
|
||||
)
|
||||
)
|
||||
|
||||
return _dedup_by_filehash(items)
|
||||
|
||||
# -- iteration ---------------------------------------------------------
|
||||
|
||||
def iter_posts(
|
||||
self, campaign_id: str, cursor: str | None = None
|
||||
) -> Iterator[tuple[dict, dict, str | None]]:
|
||||
"""Yield (post, included_index, page_cursor) for every post in the feed.
|
||||
|
||||
Pages newest→oldest via `links.next`, validating each response for
|
||||
drift before yielding. The triple gives a caller everything it needs to
|
||||
resolve and checkpoint without re-fetching:
|
||||
- post — the raw post resource.
|
||||
- included_index — the page's flattened `included` (the same object
|
||||
for every post on a page), to pass straight to
|
||||
extract_media(post, included_index).
|
||||
- page_cursor — the cursor that FETCHED this post's page (None for
|
||||
the first page). The caller checkpoints THIS value,
|
||||
matching the existing backfill cursor logic where
|
||||
the saved cursor re-fetches the page being
|
||||
processed (so a chunk cut mid-page resumes the page,
|
||||
not the one after it).
|
||||
"""
|
||||
current_cursor = cursor
|
||||
while True:
|
||||
response = self._fetch(campaign_id, current_cursor)
|
||||
self._validate_response(response)
|
||||
page_cursor = current_cursor
|
||||
included_index = self._transform(response)
|
||||
for post in response.get("data") or []:
|
||||
if isinstance(post, dict):
|
||||
yield post, included_index, page_cursor
|
||||
next_url = (response.get("links") or {}).get("next")
|
||||
next_cursor = parse_cursor_from_url(next_url)
|
||||
if not next_cursor:
|
||||
return
|
||||
current_cursor = next_cursor
|
||||
|
||||
|
||||
def _dedup_by_filehash(items: list[MediaItem]) -> list[MediaItem]:
|
||||
"""Drop later items sharing a filehash with an earlier one (first wins).
|
||||
|
||||
Items with no filehash (None) are never deduped against each other — we
|
||||
can't prove they're the same file, so keep them all.
|
||||
"""
|
||||
seen: set[str] = set()
|
||||
out: list[MediaItem] = []
|
||||
for item in items:
|
||||
if item.filehash is not None:
|
||||
if item.filehash in seen:
|
||||
continue
|
||||
seen.add(item.filehash)
|
||||
out.append(item)
|
||||
return out
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "1001",
|
||||
"type": "post",
|
||||
"attributes": {
|
||||
"title": "Image gallery post",
|
||||
"published_at": "2026-05-01T12:00:00.000+00:00",
|
||||
"post_type": "image_file",
|
||||
"content": "<p>just a gallery</p>",
|
||||
"url": "https://www.patreon.com/posts/1001",
|
||||
"image": {
|
||||
"large_url": "https://c10.patreonusercontent.com/4/large/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg",
|
||||
"url": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg"
|
||||
}
|
||||
},
|
||||
"relationships": {
|
||||
"images": {
|
||||
"data": [
|
||||
{"id": "9001", "type": "media"},
|
||||
{"id": "9002", "type": "media"}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "1002",
|
||||
"type": "post",
|
||||
"attributes": {
|
||||
"title": "Attachment post",
|
||||
"published_at": "2026-04-15T09:30:00.000+00:00",
|
||||
"post_type": "text_only",
|
||||
"content": "<p>here is a zip</p>",
|
||||
"url": "https://www.patreon.com/posts/1002",
|
||||
"post_file": {
|
||||
"name": "bonus-pack.zip",
|
||||
"url": "https://www.patreon.com/file?h=cccccccccccccccccccccccccccccccc&i=1002"
|
||||
}
|
||||
},
|
||||
"relationships": {
|
||||
"attachments_media": {
|
||||
"data": [
|
||||
{"id": "9003", "type": "media"}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "1003",
|
||||
"type": "post",
|
||||
"attributes": {
|
||||
"title": "Inline content post",
|
||||
"published_at": "2026-04-01T08:00:00.000+00:00",
|
||||
"post_type": "text_only",
|
||||
"content": "<p>look</p><figure><img src=\"https://c10.patreonusercontent.com/4/orig/dddddddddddddddddddddddddddddddd/inline.png?token=x&v=2\" alt=\"inline\"></figure>",
|
||||
"url": "https://www.patreon.com/posts/1003"
|
||||
},
|
||||
"relationships": {}
|
||||
},
|
||||
{
|
||||
"id": "1004",
|
||||
"type": "post",
|
||||
"attributes": {
|
||||
"title": "Video post",
|
||||
"published_at": "2026-03-20T18:45:00.000+00:00",
|
||||
"post_type": "video_external_file",
|
||||
"content": "<p>watch</p>",
|
||||
"url": "https://www.patreon.com/posts/1004",
|
||||
"post_file": {
|
||||
"name": "clip.m3u8",
|
||||
"url": "https://stream.mux.com/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee.m3u8?token=jwt"
|
||||
}
|
||||
},
|
||||
"relationships": {}
|
||||
}
|
||||
],
|
||||
"included": [
|
||||
{
|
||||
"id": "9001",
|
||||
"type": "media",
|
||||
"attributes": {
|
||||
"file_name": "gallery-one.jpg",
|
||||
"download_url": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg",
|
||||
"image_urls": {
|
||||
"original": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg",
|
||||
"default": "https://c10.patreonusercontent.com/4/default/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "9002",
|
||||
"type": "media",
|
||||
"attributes": {
|
||||
"file_name": "gallery-two.jpg",
|
||||
"download_url": "https://c10.patreonusercontent.com/4/orig/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/gallery-two.jpg",
|
||||
"image_urls": {
|
||||
"original": "https://c10.patreonusercontent.com/4/orig/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/gallery-two.jpg"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "9003",
|
||||
"type": "media",
|
||||
"attributes": {
|
||||
"file_name": "bonus-pack.zip",
|
||||
"download_url": "https://www.patreon.com/file/download?h=cccccccccccccccccccccccccccccccc"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "5555",
|
||||
"type": "campaign",
|
||||
"attributes": {
|
||||
"name": "Example Creator",
|
||||
"url": "https://www.patreon.com/example"
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": {
|
||||
"next": "https://www.patreon.com/api/posts?filter%5Bcampaign_id%5D=5555&page%5Bcursor%5D=NEXTCURSOR123&sort=-published_at"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
"""PatreonClient tests — parsing only, no network.
|
||||
|
||||
The fixture is loaded from disk and fed directly to the pure parsing methods
|
||||
(_transform / extract_media / _validate_response / parse_cursor_from_url); no
|
||||
live requests call is exercised here.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.app.services.patreon_client import (
|
||||
MediaItem,
|
||||
PatreonClient,
|
||||
PatreonDriftError,
|
||||
parse_cursor_from_url,
|
||||
)
|
||||
|
||||
_FIXTURE = Path(__file__).parent / "fixtures" / "patreon_posts_page1.json"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def response():
|
||||
return json.loads(_FIXTURE.read_text())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
# No cookies file → empty session, but we never make a request in tests.
|
||||
return PatreonClient(cookies_path=None)
|
||||
|
||||
|
||||
def _post_by_id(response, post_id):
|
||||
for post in response["data"]:
|
||||
if post["id"] == post_id:
|
||||
return post
|
||||
raise AssertionError(f"no post {post_id} in fixture")
|
||||
|
||||
|
||||
def test_transform_indexes_included(client, response):
|
||||
index = client._transform(response)
|
||||
assert index[("media", "9001")]["file_name"] == "gallery-one.jpg"
|
||||
assert index[("media", "9003")]["file_name"] == "bonus-pack.zip"
|
||||
assert index[("campaign", "5555")]["name"] == "Example Creator"
|
||||
# Unknown keys are absent, not errors.
|
||||
assert ("media", "0000") not in index
|
||||
|
||||
|
||||
def test_extract_gallery_dedups_image_large(client, response):
|
||||
index = client._transform(response)
|
||||
post = _post_by_id(response, "1001")
|
||||
items = client.extract_media(post, index)
|
||||
|
||||
# Two gallery images; the image_large cover shares filehash aaaa... with
|
||||
# gallery image 9001 → collapses to one, leaving exactly 2 items.
|
||||
assert len(items) == 2
|
||||
assert all(isinstance(m, MediaItem) for m in items)
|
||||
|
||||
kinds = [m.kind for m in items]
|
||||
assert kinds == ["images", "images"] # image_large deduped away
|
||||
|
||||
filenames = {m.filename for m in items}
|
||||
assert filenames == {"gallery-one.jpg", "gallery-two.jpg"}
|
||||
|
||||
hashes = {m.filehash for m in items}
|
||||
assert hashes == {"a" * 32, "b" * 32}
|
||||
assert all(m.post_id == "1001" for m in items)
|
||||
|
||||
|
||||
def test_extract_attachment_and_postfile(client, response):
|
||||
index = client._transform(response)
|
||||
post = _post_by_id(response, "1002")
|
||||
items = client.extract_media(post, index)
|
||||
|
||||
by_kind = {m.kind: m for m in items}
|
||||
# attachments_media → "attachments", plus the post_file → "postfile".
|
||||
assert set(by_kind) == {"attachments", "postfile"}
|
||||
assert by_kind["attachments"].filename == "bonus-pack.zip"
|
||||
assert by_kind["attachments"].filehash == "c" * 32
|
||||
assert by_kind["postfile"].filename == "bonus-pack.zip"
|
||||
# postfile URL also carries the same hash, but they're different kinds in
|
||||
# different relationships; both are kept because dedup is by filehash and
|
||||
# the attachment is encountered first... assert the post_file collapsed.
|
||||
# (attachments resolved before postfile, same hash → postfile dropped.)
|
||||
# So only ONE survives with that hash:
|
||||
surviving_c = [m for m in items if m.filehash == "c" * 32]
|
||||
assert len(surviving_c) == 1
|
||||
|
||||
|
||||
def test_extract_inline_content_img(client, response):
|
||||
index = client._transform(response)
|
||||
post = _post_by_id(response, "1003")
|
||||
items = client.extract_media(post, index)
|
||||
|
||||
assert len(items) == 1
|
||||
item = items[0]
|
||||
assert item.kind == "content"
|
||||
assert item.filehash == "d" * 32
|
||||
# & in the src must be unescaped to & .
|
||||
assert "&" not in item.url
|
||||
assert "token=x&v=2" in item.url
|
||||
|
||||
|
||||
def test_extract_video_postfile(client, response):
|
||||
index = client._transform(response)
|
||||
post = _post_by_id(response, "1004")
|
||||
items = client.extract_media(post, index)
|
||||
|
||||
assert len(items) == 1
|
||||
item = items[0]
|
||||
assert item.kind == "postfile"
|
||||
assert item.url.startswith("https://stream.mux.com/")
|
||||
assert item.filehash == "e" * 32
|
||||
|
||||
|
||||
def test_parse_cursor_from_url_extracts_cursor(response):
|
||||
next_url = response["links"]["next"]
|
||||
assert parse_cursor_from_url(next_url) == "NEXTCURSOR123"
|
||||
|
||||
|
||||
def test_parse_cursor_from_url_none_when_absent():
|
||||
assert parse_cursor_from_url("https://www.patreon.com/api/posts?sort=-x") is None
|
||||
assert parse_cursor_from_url(None) is None
|
||||
|
||||
|
||||
def test_iter_posts_yields_triples_and_paginates(client, response, monkeypatch):
|
||||
# Page 1 = the fixture (4 posts, links.next → NEXTCURSOR123); page 2 = one
|
||||
# post, no links.next → stop. Monkeypatch _fetch so no network is touched.
|
||||
page2 = {
|
||||
"data": [{"id": "1005", "type": "post", "attributes": {"title": "older"}}],
|
||||
"included": [],
|
||||
"links": {},
|
||||
}
|
||||
|
||||
def fake_fetch(campaign_id, cursor):
|
||||
return page2 if cursor == "NEXTCURSOR123" else response
|
||||
|
||||
monkeypatch.setattr(client, "_fetch", fake_fetch)
|
||||
|
||||
seen = list(client.iter_posts("5555"))
|
||||
# 4 posts on page 1 + 1 on page 2.
|
||||
assert [p["id"] for p, _idx, _cur in seen] == ["1001", "1002", "1003", "1004", "1005"]
|
||||
# page_cursor is the cursor that FETCHED the post's page.
|
||||
cursors = {p["id"]: cur for p, _idx, cur in seen}
|
||||
assert cursors["1001"] is None and cursors["1005"] == "NEXTCURSOR123"
|
||||
# included_index is the page's flattened resources, ready for extract_media.
|
||||
page1_index = next(idx for p, idx, _cur in seen if p["id"] == "1001")
|
||||
assert page1_index[("media", "9001")]["file_name"] == "gallery-one.jpg"
|
||||
|
||||
|
||||
def test_missing_data_raises_drift(client):
|
||||
with pytest.raises(PatreonDriftError):
|
||||
client._validate_response({"included": []})
|
||||
|
||||
|
||||
def test_non_list_data_raises_drift(client):
|
||||
with pytest.raises(PatreonDriftError):
|
||||
client._validate_response({"data": {"id": "1"}})
|
||||
|
||||
|
||||
def test_media_missing_file_name_raises_drift(client):
|
||||
# A media resource referenced by a gallery relationship but lacking
|
||||
# file_name is drift from the contract.
|
||||
post = {
|
||||
"id": "2000",
|
||||
"type": "post",
|
||||
"attributes": {"title": "broken"},
|
||||
"relationships": {"images": {"data": [{"id": "7777", "type": "media"}]}},
|
||||
}
|
||||
index = {
|
||||
("media", "7777"): {
|
||||
"download_url": "https://c10.patreonusercontent.com/4/orig/"
|
||||
+ "f" * 32
|
||||
+ "/x.jpg"
|
||||
}
|
||||
}
|
||||
with pytest.raises(PatreonDriftError):
|
||||
client.extract_media(post, index)
|
||||
|
||||
|
||||
def test_media_referenced_but_absent_raises_drift(client):
|
||||
post = {
|
||||
"id": "2001",
|
||||
"type": "post",
|
||||
"attributes": {"title": "dangling"},
|
||||
"relationships": {"images": {"data": [{"id": "8888", "type": "media"}]}},
|
||||
}
|
||||
with pytest.raises(PatreonDriftError):
|
||||
client.extract_media(post, {})
|
||||
Reference in New Issue
Block a user