feat(patreon): capture full post body via adaptive detail-fetch
The feed endpoint (/api/posts) returns `content` empty for many posts, so post
bodies — their formatting, inline <img>, and external <a href> links — were
never captured (the post showed "(no description)"). Enrich an empty feed body
from the per-post detail endpoint (/api/posts/{id}) before writing the importer
sidecar, memoized by mutating the shared post dict so a multi-image post fetches
detail exactly once and fully-seen posts (no fresh download) pay nothing.
Best-effort by design: a body we can't fetch returns None and never fails the
walk. No-doubling and no-clobber-of-populated-body already hold (post upsert is
keyed on external_post_id; an empty body parses to None and isn't applied).
First slice of milestone #64 (rich post capture + faithful rendering +
external-host downloads). Refs FC #830.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -562,6 +562,49 @@ class PatreonClient:
|
||||
return
|
||||
current_cursor = next_cursor
|
||||
|
||||
# -- detail (full body enrichment) -------------------------------------
|
||||
|
||||
def fetch_post_detail_content(self, post_id: str) -> str | None:
|
||||
"""Best-effort fetch of a post's full HTML `content` from the per-post
|
||||
DETAIL endpoint (`/api/posts/{id}`).
|
||||
|
||||
The feed/list endpoint (`/api/posts`) frequently returns `content` as
|
||||
null even though we request it — the full body (its formatting, inline
|
||||
`<img>`, and external `<a href>` links) only comes back from the
|
||||
single-post detail resource. The downloader calls this to enrich a post
|
||||
whose feed body was empty before writing the importer sidecar.
|
||||
|
||||
Best-effort BY DESIGN: the media download is the primary job, so a body
|
||||
we can't fetch must never fail the walk. Every failure path (no id,
|
||||
transport error, non-200, non-JSON, missing/empty content) returns None
|
||||
rather than raising — distinct from the loud drift/auth raises on the
|
||||
feed path, which gate real downloads.
|
||||
"""
|
||||
if not post_id:
|
||||
return None
|
||||
if self._request_sleep > 0:
|
||||
time.sleep(self._request_sleep) # pace the API endpoint (plan #703)
|
||||
url = f"{_POSTS_URL}/{post_id}"
|
||||
params = {"fields[post]": "content", "json-api-version": "1.0"}
|
||||
try:
|
||||
resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS)
|
||||
except requests.RequestException as exc:
|
||||
log.warning("Patreon post-detail fetch failed (post %s): %s", post_id, exc)
|
||||
return None
|
||||
if resp.status_code != 200:
|
||||
log.warning(
|
||||
"Patreon post-detail fetch HTTP %s (post %s)", resp.status_code, post_id
|
||||
)
|
||||
return None
|
||||
try:
|
||||
payload = resp.json()
|
||||
except ValueError:
|
||||
return None
|
||||
data = payload.get("data") if isinstance(payload, dict) else None
|
||||
attrs = data.get("attributes") if isinstance(data, dict) else None
|
||||
content = attrs.get("content") if isinstance(attrs, dict) else None
|
||||
return content if isinstance(content, str) and content.strip() else None
|
||||
|
||||
# -- verify ------------------------------------------------------------
|
||||
|
||||
def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
|
||||
|
||||
@@ -168,10 +168,17 @@ class PatreonDownloader:
|
||||
validate: bool = True,
|
||||
rate_limit: float = 0.0,
|
||||
session: requests.Session | None = None,
|
||||
content_fetcher: Callable[[str], str | None] | None = None,
|
||||
):
|
||||
self.images_root = Path(images_root)
|
||||
self.cookies_path = str(cookies_path) if cookies_path else None
|
||||
self._validate = validate
|
||||
# Best-effort enrichment seam: (post_id) -> full HTML body, or None. The
|
||||
# feed endpoint often omits `content`; the adapter wires this to
|
||||
# PatreonClient.fetch_post_detail_content so the sidecar captures the
|
||||
# real body (formatting + inline <img> + external <a href> links).
|
||||
# None in unit tests / when enrichment isn't wanted.
|
||||
self._content_fetcher = content_fetcher
|
||||
# Politeness: seconds to sleep before each actual media download (paces
|
||||
# the CDN; honors ImportSettings.download_rate_limit_seconds, the same
|
||||
# value gallery-dl used as its between-downloads `sleep`). 0 = no pacing.
|
||||
@@ -495,6 +502,18 @@ class PatreonDownloader:
|
||||
attrs = post.get("attributes") or {}
|
||||
title = attrs.get("title")
|
||||
content = attrs.get("content")
|
||||
# The feed/list endpoint frequently returns an empty `content`; the full
|
||||
# HTML body (formatting + inline <img> + external <a href> links) only
|
||||
# comes from the per-post detail endpoint. Enrich on first write for this
|
||||
# post and MEMOIZE by mutating the shared `post` dict — so a multi-image
|
||||
# post fetches detail exactly once, and a fully-seen post (no fresh
|
||||
# download → no sidecar write) never pays the extra GET.
|
||||
if (not isinstance(content, str) or not content.strip()) and self._content_fetcher:
|
||||
fetched = self._content_fetcher(str(post.get("id") or ""))
|
||||
if fetched:
|
||||
attrs["content"] = fetched
|
||||
post["attributes"] = attrs
|
||||
content = fetched
|
||||
published = attrs.get("published_at")
|
||||
url = attrs.get("url")
|
||||
data = {
|
||||
|
||||
@@ -112,7 +112,10 @@ class PatreonIngester(Ingester):
|
||||
downloader
|
||||
if downloader is not None
|
||||
else PatreonDownloader(
|
||||
self.images_root, cookies_path, validate=validate, rate_limit=rate_limit
|
||||
self.images_root, cookies_path, validate=validate, rate_limit=rate_limit,
|
||||
# Enrich empty feed bodies from the per-post detail endpoint, via
|
||||
# the SAME client (shares its cookie session + request pacing).
|
||||
content_fetcher=resolved_client.fetch_post_detail_content,
|
||||
)
|
||||
)
|
||||
super().__init__(
|
||||
|
||||
Reference in New Issue
Block a user