From 2c67c27044fcb619e8f80f02327697f2f31c21c0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 14 Jun 2026 12:29:06 -0400 Subject: [PATCH] feat(patreon): capture full post body via adaptive detail-fetch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The feed endpoint (/api/posts) returns `content` empty for many posts, so post bodies — their formatting, inline , and external links — were never captured (the post showed "(no description)"). Enrich an empty feed body from the per-post detail endpoint (/api/posts/{id}) before writing the importer sidecar, memoized by mutating the shared post dict so a multi-image post fetches detail exactly once and fully-seen posts (no fresh download) pay nothing. Best-effort by design: a body we can't fetch returns None and never fails the walk. No-doubling and no-clobber-of-populated-body already hold (post upsert is keyed on external_post_id; an empty body parses to None and isn't applied). First slice of milestone #64 (rich post capture + faithful rendering + external-host downloads). Refs FC #830. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/patreon_client.py | 43 +++++++++++++++++ backend/app/services/patreon_downloader.py | 19 ++++++++ backend/app/services/patreon_ingester.py | 5 +- tests/test_patreon_client.py | 54 ++++++++++++++++++++++ tests/test_patreon_downloader.py | 41 ++++++++++++++++ 5 files changed, 161 insertions(+), 1 deletion(-) diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index b896c4b..98ba7d6 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -562,6 +562,49 @@ class PatreonClient: return current_cursor = next_cursor + # -- detail (full body enrichment) ------------------------------------- + + def fetch_post_detail_content(self, post_id: str) -> str | None: + """Best-effort fetch of a post's full HTML `content` from the per-post + DETAIL endpoint (`/api/posts/{id}`). + + The feed/list endpoint (`/api/posts`) frequently returns `content` as + null even though we request it — the full body (its formatting, inline + ``, and external `` links) only comes back from the + single-post detail resource. The downloader calls this to enrich a post + whose feed body was empty before writing the importer sidecar. + + Best-effort BY DESIGN: the media download is the primary job, so a body + we can't fetch must never fail the walk. Every failure path (no id, + transport error, non-200, non-JSON, missing/empty content) returns None + rather than raising — distinct from the loud drift/auth raises on the + feed path, which gate real downloads. + """ + if not post_id: + return None + if self._request_sleep > 0: + time.sleep(self._request_sleep) # pace the API endpoint (plan #703) + url = f"{_POSTS_URL}/{post_id}" + params = {"fields[post]": "content", "json-api-version": "1.0"} + try: + resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS) + except requests.RequestException as exc: + log.warning("Patreon post-detail fetch failed (post %s): %s", post_id, exc) + return None + if resp.status_code != 200: + log.warning( + "Patreon post-detail fetch HTTP %s (post %s)", resp.status_code, post_id + ) + return None + try: + payload = resp.json() + except ValueError: + return None + data = payload.get("data") if isinstance(payload, dict) else None + attrs = data.get("attributes") if isinstance(data, dict) else None + content = attrs.get("content") if isinstance(attrs, dict) else None + return content if isinstance(content, str) and content.strip() else None + # -- verify ------------------------------------------------------------ def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]: diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index 7abc2df..6990bc6 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -168,10 +168,17 @@ class PatreonDownloader: validate: bool = True, rate_limit: float = 0.0, session: requests.Session | None = None, + content_fetcher: Callable[[str], str | None] | None = None, ): self.images_root = Path(images_root) self.cookies_path = str(cookies_path) if cookies_path else None self._validate = validate + # Best-effort enrichment seam: (post_id) -> full HTML body, or None. The + # feed endpoint often omits `content`; the adapter wires this to + # PatreonClient.fetch_post_detail_content so the sidecar captures the + # real body (formatting + inline + external links). + # None in unit tests / when enrichment isn't wanted. + self._content_fetcher = content_fetcher # Politeness: seconds to sleep before each actual media download (paces # the CDN; honors ImportSettings.download_rate_limit_seconds, the same # value gallery-dl used as its between-downloads `sleep`). 0 = no pacing. @@ -495,6 +502,18 @@ class PatreonDownloader: attrs = post.get("attributes") or {} title = attrs.get("title") content = attrs.get("content") + # The feed/list endpoint frequently returns an empty `content`; the full + # HTML body (formatting + inline + external links) only + # comes from the per-post detail endpoint. Enrich on first write for this + # post and MEMOIZE by mutating the shared `post` dict — so a multi-image + # post fetches detail exactly once, and a fully-seen post (no fresh + # download → no sidecar write) never pays the extra GET. + if (not isinstance(content, str) or not content.strip()) and self._content_fetcher: + fetched = self._content_fetcher(str(post.get("id") or "")) + if fetched: + attrs["content"] = fetched + post["attributes"] = attrs + content = fetched published = attrs.get("published_at") url = attrs.get("url") data = { diff --git a/backend/app/services/patreon_ingester.py b/backend/app/services/patreon_ingester.py index be2c973..c4e8d73 100644 --- a/backend/app/services/patreon_ingester.py +++ b/backend/app/services/patreon_ingester.py @@ -112,7 +112,10 @@ class PatreonIngester(Ingester): downloader if downloader is not None else PatreonDownloader( - self.images_root, cookies_path, validate=validate, rate_limit=rate_limit + self.images_root, cookies_path, validate=validate, rate_limit=rate_limit, + # Enrich empty feed bodies from the per-post detail endpoint, via + # the SAME client (shares its cookie session + request pacing). + content_fetcher=resolved_client.fetch_post_detail_content, ) ) super().__init__( diff --git a/tests/test_patreon_client.py b/tests/test_patreon_client.py index fe46b91..101dd36 100644 --- a/tests/test_patreon_client.py +++ b/tests/test_patreon_client.py @@ -9,6 +9,7 @@ import json from pathlib import Path import pytest +import requests from backend.app.services import patreon_client as pc_mod from backend.app.services.patreon_client import ( @@ -352,3 +353,56 @@ def test_request_sleep_paces_before_fetch(monkeypatch): monkeypatch.setattr(pc_mod.time, "sleep", lambda s: slept.append(s)) client._fetch("5555", None) assert slept == [1.5] + + +# -- fetch_post_detail_content (best-effort full-body enrichment) ---------- + + +def test_fetch_post_detail_content_returns_body(monkeypatch): + payload = {"data": {"id": "42", "attributes": {"content": "

full body

"}}} + captured: dict = {} + client = PatreonClient(cookies_path=None) + + def _get(url, params=None, timeout=None): + captured["url"] = url + captured["params"] = params + return _FakeResp(200, json_data=payload) + + monkeypatch.setattr(client._session, "get", _get) + assert client.fetch_post_detail_content("42") == "

full body

" + assert captured["url"].endswith("/api/posts/42") + assert captured["params"]["fields[post]"] == "content" + + +def test_fetch_post_detail_content_empty_body_is_none(monkeypatch): + payload = {"data": {"attributes": {"content": " "}}} + client = _client_returning(monkeypatch, _FakeResp(200, json_data=payload)) + assert client.fetch_post_detail_content("42") is None + + +def test_fetch_post_detail_content_blank_id_skips_request(monkeypatch): + calls: list = [] + client = PatreonClient(cookies_path=None) + monkeypatch.setattr( + client._session, "get", + lambda *a, **k: calls.append(1) or _FakeResp(200, json_data={}), + ) + assert client.fetch_post_detail_content("") is None + assert calls == [] + + +def test_fetch_post_detail_content_swallows_http_and_transport_errors(monkeypatch): + # Non-200 → None (best-effort; a missing body must never fail the walk). + client = _client_returning(monkeypatch, _FakeResp(500)) + assert client.fetch_post_detail_content("42") is None + # Non-JSON body (HTML challenge) → None, not a raise. + client = _client_returning(monkeypatch, _FakeResp(200, raise_json=True)) + assert client.fetch_post_detail_content("42") is None + # Transport error → None. + client = PatreonClient(cookies_path=None) + + def _boom(*a, **k): + raise requests.ConnectionError("boom") + + monkeypatch.setattr(client._session, "get", _boom) + assert client.fetch_post_detail_content("42") is None diff --git a/tests/test_patreon_downloader.py b/tests/test_patreon_downloader.py index 4b19873..e09608f 100644 --- a/tests/test_patreon_downloader.py +++ b/tests/test_patreon_downloader.py @@ -7,6 +7,7 @@ via monkeypatching `_run_ytdlp`. PURE module → no DB, so unmarked (fast lane). from __future__ import annotations +import json from pathlib import Path import pytest @@ -536,3 +537,43 @@ def test_range_ignored_by_server_refetches_clean(tmp_path, monkeypatch): outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") assert outcomes[0].status == "downloaded" assert outcomes[0].path.read_bytes() == full # clean, not full[:400] + full + + +# -- content enrichment (adaptive detail-fetch of an empty feed body) ------- + + +def test_enriches_empty_content_via_fetcher(tmp_path): + """An empty feed body is backfilled from the detail-fetch seam — once per + post (memoized on the shared dict) — and lands in the importer sidecar.""" + calls: list[str] = [] + + def _fetcher(post_id: str) -> str: + calls.append(post_id) + return "

full body

" + + dl = PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, + session=_FakeSession(), content_fetcher=_fetcher, + ) + post = _post() + post["attributes"]["content"] = "" # feed returned an empty body + dl.download_post(post, [_img("a.png"), _img("b.png")], "artist-x") + + post_dir = tmp_path / "artist-x" / "patreon" / pd_mod._post_dir_name(post) + sc = find_sidecar(post_dir / "01_a.png") + assert sc is not None + assert json.loads(sc.read_text())["content"] == "

full body

" + # Fetched ONCE for the post, not once per media item. + assert calls == ["1001"] + + +def test_no_enrichment_when_feed_body_present(tmp_path): + """A non-empty feed body is trusted as-is; the detail endpoint isn't hit.""" + calls: list[str] = [] + dl = PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, + session=_FakeSession(), + content_fetcher=lambda pid: calls.append(pid) or "x", + ) + dl.download_post(_post(), [_img("a.png")], "artist-x") # _post body is non-empty + assert calls == []