bvandeusen
3df191e255
Operator-flagged: 9 StickySpoodge posts had empty bodies in FC despite the body plainly existing + being accessible (creds refresh didn't help). All 9 are body-only / poll / embed / announcement posts with no downloadable gallery media — Patreon's detail endpoint returns content:null for these under the sparse fields[post]=content request even though the body exists. fetch_post_detail_content now re-fetches the FULL post resource once when the sparse request comes back empty: recovers the body when the sparse fieldset was the cause, and logs post_type when even the full resource is empty (body lives elsewhere). Only the empty cases pay the extra GET; the 126 already-working posts keep the fast sparse path. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
691 lines
29 KiB
Python
691 lines
29 KiB
Python
"""Native Patreon JSON:API client (build step 1 of the native ingester).
|
|
|
|
Clean-room reimplementation of the Patreon `/api/posts` read path. This is a
|
|
plain, synchronous client over `requests` — the orchestrator already wraps
|
|
sync importer calls, so nothing here needs to be async (mirrors
|
|
patreon_resolver.py, which wraps its sync lookup in run_in_executor at the
|
|
call site).
|
|
|
|
Scope (build step 1): fetch + page + parse only. This module is NOT wired into
|
|
download_service yet — that is a later step. The public surface here exists so
|
|
the later step can drive it:
|
|
- PatreonClient(cookies_path).iter_posts(campaign_id)
|
|
→ (post, included_index, page_cursor)
|
|
- extract_media(post, included_index) → list[MediaItem]
|
|
- parse_cursor_from_url(url) → cursor
|
|
|
|
Drift detection is loud on purpose: Patreon ships JSON:API and the shapes we
|
|
depend on (top-level `data`, media resources carrying `file_name`/`url`) are
|
|
the contract. If a response comes back as an HTML login page or a media
|
|
resource is missing the fields we resolve against, we raise PatreonDriftError
|
|
rather than silently yielding empty media — so the later import step surfaces
|
|
"Patreon changed something" instead of "creator has no posts".
|
|
|
|
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import http.cookiejar
|
|
import logging
|
|
import os
|
|
import re
|
|
import time
|
|
from collections.abc import Iterator
|
|
from dataclasses import dataclass
|
|
from html import unescape
|
|
from pathlib import Path
|
|
from urllib.parse import parse_qs, urlsplit
|
|
|
|
import requests
|
|
|
|
from ..utils.paths import filehash_from_url, safe_ext
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
_POSTS_URL = "https://www.patreon.com/api/posts"
|
|
_USER_AGENT = (
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
|
"(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
|
)
|
|
_TIMEOUT_SECONDS = 30.0
|
|
|
|
# 429 backoff (plan #703): ride out a transient API rate-limit instead of
|
|
# failing the whole walk (which would stamp RATE_LIMITED → platform-wide
|
|
# cooldown → every Patreon source dark). Honor the server's `Retry-After`;
|
|
# otherwise exponential base·2^(n-1), capped. Only after the retries are
|
|
# exhausted does the 429 propagate as terminal RATE_LIMITED.
|
|
_MAX_429_RETRIES = 3
|
|
_BACKOFF_BASE_SECONDS = 2.0
|
|
_BACKOFF_CAP_SECONDS = 30.0
|
|
|
|
|
|
def _retry_after_seconds(
|
|
resp: requests.Response,
|
|
attempt: int,
|
|
*,
|
|
base: float = _BACKOFF_BASE_SECONDS,
|
|
cap: float = _BACKOFF_CAP_SECONDS,
|
|
) -> float:
|
|
"""Backoff delay for a 429: the `Retry-After` seconds header if present and
|
|
numeric, else exponential `base·2^(attempt-1)`, both capped. (HTTP-date form
|
|
of Retry-After is rare here and falls through to exponential.)"""
|
|
header = resp.headers.get("Retry-After")
|
|
if header:
|
|
try:
|
|
return min(float(header), cap)
|
|
except (TypeError, ValueError):
|
|
pass
|
|
return min(base * (2 ** max(0, attempt - 1)), cap)
|
|
|
|
# JSON:API request contract (observed from real traffic — see module plan).
|
|
_INCLUDE = (
|
|
"campaign,access_rules,attachments,attachments_media,audio,images,media,"
|
|
"native_video_insights,user,user_defined_tags,ti_checks"
|
|
)
|
|
_FIELDS_POST = (
|
|
"content,post_file,image,post_type,published_at,title,url,patreon_url,"
|
|
"current_user_can_view"
|
|
)
|
|
_FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name"
|
|
_FIELDS_CAMPAIGN = "name,url"
|
|
|
|
# Inline post `content` is HTML; images are emitted as <img ... src="...">.
|
|
# Pull every src; downstream dedup collapses any that duplicate a gallery item
|
|
# by filehash. Tolerant of attribute ordering and single/double quotes.
|
|
_CONTENT_IMG_RE = re.compile(r"<img\b[^>]*?\bsrc=[\"']([^\"']+)[\"']", re.IGNORECASE)
|
|
|
|
|
|
class PatreonAPIError(Exception):
|
|
"""Base for native Patreon client failures.
|
|
|
|
`status_code` carries the HTTP status when the failure was an HTTP response
|
|
(None for transport-level / parse failures), so the ingester can map it to a
|
|
DownloadResult.error_type (429 → rate_limited, 404 → not_found, …).
|
|
`retry_after` carries the server's `Retry-After` seconds on a terminal 429, so
|
|
the platform cooldown can match the server's hint instead of a flat default
|
|
(plan #708 B1).
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
message: str,
|
|
*,
|
|
status_code: int | None = None,
|
|
retry_after: float | None = None,
|
|
):
|
|
super().__init__(message)
|
|
self.status_code = status_code
|
|
self.retry_after = retry_after
|
|
|
|
|
|
class PatreonAuthError(PatreonAPIError):
|
|
"""Authentication / authorization failure — missing or expired session
|
|
cookies, an insufficient pledge tier, or an HTML login/challenge page served
|
|
where JSON was expected. DISTINCT from drift: the fix is rotating the
|
|
credential, not updating the ingester. Maps to error_type 'auth_error'.
|
|
"""
|
|
|
|
|
|
class PatreonDriftError(PatreonAPIError):
|
|
"""A JSON response did not match the JSON:API shape we depend on.
|
|
|
|
Raised for: a missing top-level `data` list, `data` not a list, or a media
|
|
resource lacking the `file_name`/`url` fields we resolve against. Fail loud
|
|
so the import step flags API drift (ingester needs update) instead of
|
|
silently importing nothing. An HTML-login / non-JSON body is auth, not
|
|
drift — that raises PatreonAuthError.
|
|
"""
|
|
|
|
|
|
@dataclass
|
|
class MediaItem:
|
|
"""One resolved downloadable item belonging to a post.
|
|
|
|
Fields:
|
|
url — the CDN/download URL to fetch.
|
|
filename — media `file_name` when present; otherwise the URL basename
|
|
(NEVER a network call). Bounded/sane extension.
|
|
kind — one of: "images", "image_large", "attachments", "postfile",
|
|
"content". Mirrors gallery-dl's `files` content-type names so
|
|
the later step can honor the same per-source content_types.
|
|
filehash — the 32-char hex (MD5) segment from the CDN URL, or None if
|
|
the URL carries no such segment. Used for in-post dedup and
|
|
the cross-run seen-ledger.
|
|
post_id — the owning post's id (so a flattened media list stays
|
|
traceable to its post).
|
|
"""
|
|
|
|
url: str
|
|
filename: str
|
|
kind: str
|
|
filehash: str | None
|
|
post_id: str
|
|
|
|
|
|
def _load_session(cookies_path: str | Path | None) -> requests.Session:
|
|
session = requests.Session()
|
|
session.headers.update(
|
|
{
|
|
"User-Agent": _USER_AGENT,
|
|
"Accept": "application/vnd.api+json",
|
|
}
|
|
)
|
|
if cookies_path and os.path.isfile(str(cookies_path)):
|
|
try:
|
|
jar = http.cookiejar.MozillaCookieJar(str(cookies_path))
|
|
jar.load(ignore_discard=True, ignore_expires=True)
|
|
session.cookies = jar # type: ignore[assignment]
|
|
except (OSError, http.cookiejar.LoadError) as exc:
|
|
log.warning("Could not load Patreon cookies from %s: %s", cookies_path, exc)
|
|
return session
|
|
|
|
|
|
def _filehash(url: str) -> str | None:
|
|
# Delegate to the shared extractor (utils.paths) so capture-time persistence
|
|
# and render-time inline-image matching use the EXACT same identity.
|
|
return filehash_from_url(url)
|
|
|
|
|
|
def _basename_from_url(url: str) -> str:
|
|
"""Derive a sane filename from a URL when the media has no file_name.
|
|
|
|
Strips query/fragment, takes the path basename, and drops a junk
|
|
extension (the importer._safe_ext gotcha) so we never write base64 noise
|
|
as a name. Falls back to the filehash, then to "file".
|
|
"""
|
|
path = urlsplit(url).path
|
|
base = os.path.basename(path)
|
|
if base:
|
|
ext = safe_ext(base)
|
|
stem = base[: -len(Path(base).suffix)] if Path(base).suffix else base
|
|
# Keep the stem bounded; URL-encoded stems can be enormous.
|
|
stem = stem[:120] or "file"
|
|
return f"{stem}{ext}"
|
|
fh = _filehash(url)
|
|
return fh or "file"
|
|
|
|
|
|
def parse_cursor_from_url(url: str | None) -> str | None:
|
|
"""Extract the `page[cursor]` query param from a links.next URL."""
|
|
if not url:
|
|
return None
|
|
query = urlsplit(url).query
|
|
values = parse_qs(query).get("page[cursor]")
|
|
if values and values[0]:
|
|
return values[0]
|
|
return None
|
|
|
|
|
|
class PatreonClient:
|
|
"""Synchronous Patreon JSON:API read client.
|
|
|
|
Construct with a path to a Netscape cookies.txt (the same file
|
|
CredentialService.get_cookies_path materializes). Cookies are loaded into a
|
|
requests.Session; no secure-context APIs are used.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
cookies_path: str | Path | None,
|
|
*,
|
|
request_sleep: float = 0.0,
|
|
max_retries: int = _MAX_429_RETRIES,
|
|
):
|
|
self.cookies_path = str(cookies_path) if cookies_path else None
|
|
self._session = _load_session(cookies_path)
|
|
# Politeness: seconds to sleep before each /api/posts page fetch (paces
|
|
# the rate-limited API endpoint). 0 = no pacing. plan #703.
|
|
self._request_sleep = request_sleep or 0.0
|
|
self._max_retries = max_retries
|
|
|
|
# -- request -----------------------------------------------------------
|
|
|
|
def _params(self, campaign_id: str, cursor: str | None) -> dict[str, str]:
|
|
params = {
|
|
"include": _INCLUDE,
|
|
"fields[post]": _FIELDS_POST,
|
|
"fields[media]": _FIELDS_MEDIA,
|
|
"fields[campaign]": _FIELDS_CAMPAIGN,
|
|
"filter[campaign_id]": campaign_id,
|
|
"filter[contains_exclusive_posts]": "true",
|
|
"filter[is_draft]": "false",
|
|
"sort": "-published_at",
|
|
"json-api-version": "1.0",
|
|
}
|
|
if cursor:
|
|
params["page[cursor]"] = cursor
|
|
return params
|
|
|
|
def _fetch(self, campaign_id: str, cursor: str | None) -> dict:
|
|
if self._request_sleep > 0:
|
|
time.sleep(self._request_sleep) # pace the API endpoint
|
|
attempt = 0
|
|
while True:
|
|
try:
|
|
resp = self._session.get(
|
|
_POSTS_URL,
|
|
params=self._params(campaign_id, cursor),
|
|
timeout=_TIMEOUT_SECONDS,
|
|
)
|
|
except requests.RequestException as exc:
|
|
raise PatreonAPIError(
|
|
f"Patreon posts request failed (campaign_id={campaign_id}): {exc}"
|
|
) from exc
|
|
|
|
# Transient rate-limit: back off and retry rather than failing the
|
|
# whole walk. Only a PERSISTENT 429 (retries exhausted) falls
|
|
# through to the terminal RATE_LIMITED raise below.
|
|
if resp.status_code == 429 and attempt < self._max_retries:
|
|
attempt += 1
|
|
delay = _retry_after_seconds(resp, attempt)
|
|
log.warning(
|
|
"Patreon 429 (campaign_id=%s) — backing off %.1fs (retry %d/%d)",
|
|
campaign_id, delay, attempt, self._max_retries,
|
|
)
|
|
time.sleep(delay)
|
|
continue
|
|
break
|
|
|
|
if resp.status_code in (401, 403):
|
|
# Auth rejected — expired/missing cookies or an insufficient tier.
|
|
# Actionable as "rotate credentials", so it's auth, not drift/http.
|
|
raise PatreonAuthError(
|
|
f"Patreon posts API returned HTTP {resp.status_code} — auth "
|
|
f"rejected (cookies expired or tier insufficient; "
|
|
f"campaign_id={campaign_id})",
|
|
status_code=resp.status_code,
|
|
)
|
|
if resp.status_code != 200:
|
|
# A persistent 429 (retries exhausted) is terminal RATE_LIMITED — carry
|
|
# the server's raw Retry-After seconds so the cooldown matches its hint
|
|
# (plan #708 B1). Header is uncapped here; the cooldown clamps it.
|
|
retry_after = None
|
|
if resp.status_code == 429:
|
|
hdr = resp.headers.get("Retry-After")
|
|
if hdr:
|
|
try:
|
|
retry_after = float(hdr)
|
|
except (TypeError, ValueError):
|
|
retry_after = None
|
|
raise PatreonAPIError(
|
|
f"Patreon posts API returned HTTP {resp.status_code} "
|
|
f"(campaign_id={campaign_id})",
|
|
status_code=resp.status_code,
|
|
retry_after=retry_after,
|
|
)
|
|
try:
|
|
payload = resp.json()
|
|
except ValueError as exc:
|
|
# A non-JSON body here is almost always the HTML login/challenge
|
|
# page served when cookies are missing/expired — that is an AUTH
|
|
# failure (rotate cookies), not API drift (update the ingester) and
|
|
# not a transient network error.
|
|
raise PatreonAuthError(
|
|
"Patreon posts API returned a non-JSON response (likely an "
|
|
f"HTML login/challenge page — session expired; "
|
|
f"campaign_id={campaign_id}): {exc}"
|
|
) from exc
|
|
return payload
|
|
|
|
# -- parsing -----------------------------------------------------------
|
|
|
|
@staticmethod
|
|
def _transform(response: dict) -> dict:
|
|
"""Flatten the JSON:API `included` array for relationship resolution.
|
|
|
|
Returns a dict keyed by `(type, id)` → that resource's `attributes`
|
|
(so a post's relationships can be resolved in O(1)). Missing/oddly
|
|
shaped `included` entries are skipped rather than fatal — drift
|
|
detection for the top-level shape lives in _validate_response.
|
|
"""
|
|
index: dict[tuple[str, str], dict] = {}
|
|
for inc in response.get("included") or []:
|
|
if not isinstance(inc, dict):
|
|
continue
|
|
rtype = inc.get("type")
|
|
rid = inc.get("id")
|
|
if rtype is None or rid is None:
|
|
continue
|
|
index[(str(rtype), str(rid))] = inc.get("attributes") or {}
|
|
return index
|
|
|
|
@staticmethod
|
|
def _validate_response(response: dict) -> None:
|
|
if not isinstance(response, dict):
|
|
raise PatreonDriftError("Patreon response was not a JSON object")
|
|
if "data" not in response:
|
|
raise PatreonDriftError("Patreon response missing top-level 'data' key")
|
|
data = response.get("data")
|
|
if not isinstance(data, list):
|
|
raise PatreonDriftError("Patreon response 'data' was not a list")
|
|
|
|
def _related_ids(self, post: dict, rel_name: str) -> list[str]:
|
|
rels = post.get("relationships") or {}
|
|
rel = rels.get(rel_name) or {}
|
|
data = rel.get("data")
|
|
if isinstance(data, dict): # to-one relationship
|
|
data = [data]
|
|
if not isinstance(data, list):
|
|
return []
|
|
ids: list[str] = []
|
|
for ref in data:
|
|
if isinstance(ref, dict) and ref.get("id") is not None:
|
|
ids.append(str(ref["id"]))
|
|
return ids
|
|
|
|
@staticmethod
|
|
def _media_url(attrs: dict) -> str | None:
|
|
"""Pick the best fetchable URL for a media resource.
|
|
|
|
Prefer the full-size `download_url`; fall back to the largest
|
|
`image_urls` size. gallery-dl prefers download_url too, only dipping
|
|
into image_urls when a smaller configured size is requested — FC
|
|
always wants the original, so download_url first.
|
|
"""
|
|
download_url = attrs.get("download_url")
|
|
if isinstance(download_url, str) and download_url:
|
|
return download_url
|
|
image_urls = attrs.get("image_urls")
|
|
if isinstance(image_urls, dict):
|
|
for key in ("original", "full", "large", "default"):
|
|
candidate = image_urls.get(key)
|
|
if isinstance(candidate, str) and candidate:
|
|
return candidate
|
|
# Otherwise take any non-empty string value.
|
|
for candidate in image_urls.values():
|
|
if isinstance(candidate, str) and candidate:
|
|
return candidate
|
|
return None
|
|
|
|
def _media_item(self, attrs: dict, kind: str, post_id: str) -> MediaItem:
|
|
url = self._media_url(attrs)
|
|
if not url:
|
|
raise PatreonDriftError(
|
|
f"Patreon media (post {post_id}, kind={kind}) had no resolvable URL "
|
|
f"(no download_url / image_urls)"
|
|
)
|
|
# file_name is OPTIONAL: Patreon legitimately serves some gallery images
|
|
# without it (operator-flagged 2026-06-07, BlenderKnight post 73665615),
|
|
# and the URL basename is a fine fallback — the same thing gallery-dl
|
|
# uses. A genuine schema change shows up as no URL (above) or a media id
|
|
# absent from `included` (caller), not a missing name.
|
|
file_name = attrs.get("file_name")
|
|
filename = file_name if isinstance(file_name, str) and file_name else _basename_from_url(url)
|
|
return MediaItem(
|
|
url=url,
|
|
filename=filename,
|
|
kind=kind,
|
|
filehash=_filehash(url),
|
|
post_id=post_id,
|
|
)
|
|
|
|
def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]:
|
|
"""Resolve all downloadable media for one post.
|
|
|
|
Walks the kinds in the same order gallery-dl does — images,
|
|
image_large (post cover), attachments, postfile, content (inline
|
|
<img>) — and dedups within the post by filehash (first wins). The
|
|
image_large cover commonly duplicates a gallery image; deduping by
|
|
filehash collapses them to the gallery item (encountered first).
|
|
"""
|
|
post_id = str(post.get("id") or "")
|
|
attrs = post.get("attributes") or {}
|
|
items: list[MediaItem] = []
|
|
|
|
def _resolve_rel(rel_name: str, kind: str) -> None:
|
|
for mid in self._related_ids(post, rel_name):
|
|
media_attrs = included_index.get(("media", mid))
|
|
if media_attrs is None:
|
|
# Referenced but not in `included`: a media id with no
|
|
# resource is drift (we asked for include=media).
|
|
raise PatreonDriftError(
|
|
f"Patreon post {post_id} references media {mid} "
|
|
f"({rel_name}) not present in 'included'"
|
|
)
|
|
items.append(self._media_item(media_attrs, kind, post_id))
|
|
|
|
# 1. gallery images
|
|
_resolve_rel("images", "images")
|
|
|
|
# 2. image_large — the post-level cover (`image.large_url`). Not a
|
|
# media relationship; it lives on the post attributes.
|
|
image = attrs.get("image")
|
|
if isinstance(image, dict):
|
|
large_url = image.get("large_url") or image.get("url")
|
|
if isinstance(large_url, str) and large_url:
|
|
items.append(
|
|
MediaItem(
|
|
url=large_url,
|
|
filename=_basename_from_url(large_url),
|
|
kind="image_large",
|
|
filehash=_filehash(large_url),
|
|
post_id=post_id,
|
|
)
|
|
)
|
|
|
|
# 3. attachments
|
|
_resolve_rel("attachments_media", "attachments")
|
|
|
|
# 4. postfile — the post's primary attached file (`post_file`).
|
|
post_file = attrs.get("post_file")
|
|
if isinstance(post_file, dict):
|
|
pf_url = post_file.get("url") or post_file.get("download_url")
|
|
if isinstance(pf_url, str) and pf_url:
|
|
pf_name = post_file.get("name")
|
|
filename = (
|
|
pf_name
|
|
if isinstance(pf_name, str) and pf_name
|
|
else _basename_from_url(pf_url)
|
|
)
|
|
items.append(
|
|
MediaItem(
|
|
url=pf_url,
|
|
filename=filename,
|
|
kind="postfile",
|
|
filehash=_filehash(pf_url),
|
|
post_id=post_id,
|
|
)
|
|
)
|
|
|
|
# 5. content — inline <img> in the post HTML body.
|
|
content = attrs.get("content")
|
|
if isinstance(content, str) and content:
|
|
for raw_src in _CONTENT_IMG_RE.findall(content):
|
|
src = unescape(raw_src)
|
|
if not src:
|
|
continue
|
|
items.append(
|
|
MediaItem(
|
|
url=src,
|
|
filename=_basename_from_url(src),
|
|
kind="content",
|
|
filehash=_filehash(src),
|
|
post_id=post_id,
|
|
)
|
|
)
|
|
|
|
return _dedup_by_filehash(items)
|
|
|
|
@staticmethod
|
|
def post_meta(post: dict) -> dict:
|
|
"""Title + published date for a post — for the preview sample (plan #708
|
|
B4). Part of the client contract `ingest_core.Ingester.preview` calls."""
|
|
attrs = post.get("attributes") or {}
|
|
title = attrs.get("title")
|
|
published = attrs.get("published_at")
|
|
return {
|
|
"title": title if isinstance(title, str) else None,
|
|
"date": published if isinstance(published, str) else None,
|
|
}
|
|
|
|
@staticmethod
|
|
def post_record_key(post: dict) -> tuple[str, str] | None:
|
|
"""`(ledger_key, post_id)` for a media-less post's seen-ledger entry, or
|
|
None when the post has no id. The synthetic `post:<id>` key lets the
|
|
generic core gate text-post capture through the SAME seen-ledger as media
|
|
— so a text post's body is detail-fetched + recorded ONCE, not re-fetched
|
|
every tick. Part of the client contract the core uses via getattr."""
|
|
pid = post.get("id")
|
|
pid = str(pid) if pid is not None else ""
|
|
if not pid:
|
|
return None
|
|
return (f"post:{pid}", pid)
|
|
|
|
# -- iteration ---------------------------------------------------------
|
|
|
|
def iter_posts(
|
|
self, campaign_id: str, cursor: str | None = None
|
|
) -> Iterator[tuple[dict, dict, str | None]]:
|
|
"""Yield (post, included_index, page_cursor) for every post in the feed.
|
|
|
|
Pages newest→oldest via `links.next`, validating each response for
|
|
drift before yielding. The triple gives a caller everything it needs to
|
|
resolve and checkpoint without re-fetching:
|
|
- post — the raw post resource.
|
|
- included_index — the page's flattened `included` (the same object
|
|
for every post on a page), to pass straight to
|
|
extract_media(post, included_index).
|
|
- page_cursor — the cursor that FETCHED this post's page (None for
|
|
the first page). The caller checkpoints THIS value,
|
|
matching the existing backfill cursor logic where
|
|
the saved cursor re-fetches the page being
|
|
processed (so a chunk cut mid-page resumes the page,
|
|
not the one after it).
|
|
"""
|
|
current_cursor = cursor
|
|
while True:
|
|
response = self._fetch(campaign_id, current_cursor)
|
|
self._validate_response(response)
|
|
page_cursor = current_cursor
|
|
included_index = self._transform(response)
|
|
for post in response.get("data") or []:
|
|
if isinstance(post, dict):
|
|
yield post, included_index, page_cursor
|
|
next_url = (response.get("links") or {}).get("next")
|
|
next_cursor = parse_cursor_from_url(next_url)
|
|
if not next_cursor:
|
|
return
|
|
current_cursor = next_cursor
|
|
|
|
# -- detail (full body enrichment) -------------------------------------
|
|
|
|
def fetch_post_detail_content(self, post_id: str) -> str | None:
|
|
"""Best-effort fetch of a post's full HTML `content` from the per-post
|
|
DETAIL endpoint (`/api/posts/{id}`).
|
|
|
|
The feed/list endpoint (`/api/posts`) frequently returns `content` as
|
|
null even though we request it — the full body (its formatting, inline
|
|
`<img>`, and external `<a href>` links) only comes back from the
|
|
single-post detail resource. The downloader calls this to enrich a post
|
|
whose feed body was empty before writing the importer sidecar.
|
|
|
|
Best-effort BY DESIGN: the media download is the primary job, so a body
|
|
we can't fetch must never fail the walk. Every failure path (no id,
|
|
transport error, non-200, non-JSON, missing/empty content) returns None
|
|
rather than raising — distinct from the loud drift/auth raises on the
|
|
feed path, which gate real downloads.
|
|
"""
|
|
if not post_id:
|
|
return None
|
|
if self._request_sleep > 0:
|
|
time.sleep(self._request_sleep) # pace the API endpoint (plan #703)
|
|
url = f"{_POSTS_URL}/{post_id}"
|
|
params = {"fields[post]": "content", "json-api-version": "1.0"}
|
|
try:
|
|
resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS)
|
|
except requests.RequestException as exc:
|
|
log.warning("Patreon post-detail fetch failed (post %s): %s", post_id, exc)
|
|
return None
|
|
if resp.status_code != 200:
|
|
log.warning(
|
|
"Patreon post-detail fetch HTTP %s (post %s)", resp.status_code, post_id
|
|
)
|
|
return None
|
|
try:
|
|
payload = resp.json()
|
|
except ValueError:
|
|
return None
|
|
data = payload.get("data") if isinstance(payload, dict) else None
|
|
attrs = data.get("attributes") if isinstance(data, dict) else None
|
|
content = attrs.get("content") if isinstance(attrs, dict) else None
|
|
if isinstance(content, str) and content.strip():
|
|
log.info("post-detail: fetched %d chars (post %s)", len(content), post_id)
|
|
return content
|
|
# Sparse `fields[post]=content` came back null/empty. Patreon serves null
|
|
# under the sparse fieldset for some post types (polls, embeds/films,
|
|
# body-only announcements) even when the body plainly exists. Re-fetch the
|
|
# FULL post resource ONCE — only the empty cases pay this extra GET — which
|
|
# both DIAGNOSES (logs post_type) and RECOVERS the body when the sparse
|
|
# fieldset was the cause (#842).
|
|
try:
|
|
full = self._session.get(
|
|
url, params={"json-api-version": "1.0"}, timeout=_TIMEOUT_SECONDS,
|
|
)
|
|
except requests.RequestException as exc:
|
|
log.warning("post-detail full re-fetch failed (post %s): %s", post_id, exc)
|
|
return None
|
|
fa = None
|
|
if full.status_code == 200:
|
|
try:
|
|
fp = full.json()
|
|
except ValueError:
|
|
fp = None
|
|
fdata = fp.get("data") if isinstance(fp, dict) else None
|
|
fa = fdata.get("attributes") if isinstance(fdata, dict) else None
|
|
fcontent = fa.get("content") if isinstance(fa, dict) else None
|
|
ptype = fa.get("post_type") if isinstance(fa, dict) else None
|
|
if isinstance(fcontent, str) and fcontent.strip():
|
|
log.warning(
|
|
"post-detail: sparse fieldset gave null but FULL fetch has %d chars "
|
|
"(post %s, post_type=%s) — using full body",
|
|
len(fcontent), post_id, ptype,
|
|
)
|
|
return fcontent
|
|
log.info(
|
|
"post-detail: empty/null content (post %s, post_type=%s) even on full "
|
|
"fetch — body not in the post resource", post_id, ptype,
|
|
)
|
|
return None
|
|
|
|
# -- verify ------------------------------------------------------------
|
|
|
|
def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
|
|
"""Cheap auth probe: fetch the first `/api/posts` page and report whether
|
|
the credential authenticated, WITHOUT downloading anything.
|
|
|
|
Returns `(ok, message)` matching the credential-verify contract:
|
|
- True — authenticated (the feed returned a valid JSON:API page).
|
|
- False — the credential was rejected (PatreonAuthError: 401/403, or an
|
|
HTML login page → cookies expired / tier insufficient).
|
|
- None — inconclusive: API drift (our parser is stale, not a cred
|
|
problem) or a transient network/HTTP error.
|
|
"""
|
|
try:
|
|
response = self._fetch(campaign_id, None)
|
|
self._validate_response(response)
|
|
except PatreonAuthError as exc:
|
|
return False, f"Patreon rejected the credential — {exc}"
|
|
except PatreonDriftError as exc:
|
|
return None, f"Couldn't verify — Patreon's API shape changed: {exc}"
|
|
except PatreonAPIError as exc:
|
|
return None, f"Couldn't verify (network/HTTP issue): {exc}"
|
|
return True, "Credentials valid — the Patreon feed authenticated."
|
|
|
|
|
|
def _dedup_by_filehash(items: list[MediaItem]) -> list[MediaItem]:
|
|
"""Drop later items sharing a filehash with an earlier one (first wins).
|
|
|
|
Items with no filehash (None) are never deduped against each other — we
|
|
can't prove they're the same file, so keep them all.
|
|
"""
|
|
seen: set[str] = set()
|
|
out: list[MediaItem] = []
|
|
for item in items:
|
|
if item.filehash is not None:
|
|
if item.filehash in seen:
|
|
continue
|
|
seen.add(item.filehash)
|
|
out.append(item)
|
|
return out
|