From 976107bbe820a265e340fae8bea36dc81c325aa7 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 15 Jun 2026 00:14:52 -0400 Subject: [PATCH] fix(patreon): read post body from content_json_string (ProseMirror), not the dead content field (#842) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit THE empty-body root cause. Patreon deprecated the flat `content` HTML field — it returns null on the feed AND the detail endpoint, for every post type (confirmed against the live API: all 135 StickySpoodge posts, text_only/ image_file/poll alike). The real body now lives in `content_json_string` (a ProseMirror/TipTap doc), returned only under the DEFAULT post fieldset — a sparse fields[post]=content request omits it. Not credential, not post_type: a request shape gone stale. - NEW utils/prosemirror.py: ProseMirror doc -> HTML (paragraphs, marks bold/italic/underline/strike/code/link, hardBreak, inline images, lists, headings; unknown nodes degrade to children). post_body_html(attrs) = the one resolver: legacy content HTML else convert content_json_string. - patreon_client: add content_json_string to the feed _FIELDS_POST; rewrite fetch_post_detail_content to use the DEFAULT fieldset (no sparse fields[post]) and resolve via post_body_html (replaces the wrong sparse req + full-fetch fallback). - patreon_downloader._write_sidecar_data: resolve body via post_body_html (feed content_json_string) before the detail-fetch; memoize resolved HTML. - tests: prosemirror converter unit tests; client legacy + content_json_string paths; contract pins content_json_string. Inline nodes carry the CDN filehash → bodies now feed Phase-2 localization. Co-Authored-By: Claude Opus 4.8 --- backend/app/services/patreon_client.py | 82 +++++-------- backend/app/services/patreon_downloader.py | 24 ++-- backend/app/utils/prosemirror.py | 129 +++++++++++++++++++++ tests/test_patreon_client.py | 44 +++---- tests/test_patreon_contract.py | 4 +- tests/test_prosemirror.py | 108 +++++++++++++++++ 6 files changed, 302 insertions(+), 89 deletions(-) create mode 100644 backend/app/utils/prosemirror.py create mode 100644 tests/test_prosemirror.py diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index 84a4ffd..dea59df 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -40,6 +40,7 @@ from urllib.parse import parse_qs, urlsplit import requests from ..utils.paths import filehash_from_url, safe_ext +from ..utils.prosemirror import post_body_html log = logging.getLogger(__name__) @@ -84,8 +85,12 @@ _INCLUDE = ( "native_video_insights,user,user_defined_tags,ti_checks" ) _FIELDS_POST = ( - "content,post_file,image,post_type,published_at,title,url,patreon_url," - "current_user_can_view" + # `content` is the legacy flat-HTML body (Patreon now returns it null); + # `content_json_string` is the current ProseMirror-doc body. Request BOTH — + # post_body_html() prefers content (old posts) and falls back to converting + # content_json_string (current posts). #842. + "content,content_json_string,post_file,image,post_type,published_at,title," + "url,patreon_url,current_user_can_view" ) _FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name" _FIELDS_CAMPAIGN = "name,url" @@ -571,29 +576,30 @@ class PatreonClient: # -- detail (full body enrichment) ------------------------------------- def fetch_post_detail_content(self, post_id: str) -> str | None: - """Best-effort fetch of a post's full HTML `content` from the per-post - DETAIL endpoint (`/api/posts/{id}`). + """Best-effort fetch of a post's body (as HTML) from the per-post DETAIL + endpoint (`/api/posts/{id}`). - The feed/list endpoint (`/api/posts`) frequently returns `content` as - null even though we request it — the full body (its formatting, inline - ``, and external `` links) only comes back from the - single-post detail resource. The downloader calls this to enrich a post - whose feed body was empty before writing the importer sidecar. + Patreon deprecated the flat `content` HTML field — it returns null on the + feed AND the detail endpoint, for every post type. The real body now lives + in `content_json_string` (a ProseMirror doc), and is only returned under + the DEFAULT post fieldset: a sparse `fields[post]=content` request OMITS + it (confirmed against the live API 2026-06-15). So we request the default + fieldset (no `fields[post]`) and resolve the body via `post_body_html` + (content → else convert content_json_string). The downloader calls this to + enrich a post whose feed body was empty before writing the sidecar. - Best-effort BY DESIGN: the media download is the primary job, so a body - we can't fetch must never fail the walk. Every failure path (no id, - transport error, non-200, non-JSON, missing/empty content) returns None - rather than raising — distinct from the loud drift/auth raises on the - feed path, which gate real downloads. + Best-effort BY DESIGN: a body we can't fetch must never fail the walk. + Every failure path returns None rather than raising — distinct from the + loud drift/auth raises on the feed path, which gate real downloads. """ if not post_id: return None if self._request_sleep > 0: time.sleep(self._request_sleep) # pace the API endpoint (plan #703) - url = f"{_POSTS_URL}/{post_id}" - params = {"fields[post]": "content", "json-api-version": "1.0"} + # No `fields[post]` — the default fieldset is the only shape that returns + # content_json_string (the body). A sparse fieldset nulls it out. try: - resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS) + resp = self._session.get(f"{_POSTS_URL}/{post_id}", timeout=_TIMEOUT_SECONDS) except requests.RequestException as exc: log.warning("Patreon post-detail fetch failed (post %s): %s", post_id, exc) return None @@ -608,43 +614,13 @@ class PatreonClient: return None data = payload.get("data") if isinstance(payload, dict) else None attrs = data.get("attributes") if isinstance(data, dict) else None - content = attrs.get("content") if isinstance(attrs, dict) else None - if isinstance(content, str) and content.strip(): - log.info("post-detail: fetched %d chars (post %s)", len(content), post_id) - return content - # Sparse `fields[post]=content` came back null/empty. Patreon serves null - # under the sparse fieldset for some post types (polls, embeds/films, - # body-only announcements) even when the body plainly exists. Re-fetch the - # FULL post resource ONCE — only the empty cases pay this extra GET — which - # both DIAGNOSES (logs post_type) and RECOVERS the body when the sparse - # fieldset was the cause (#842). - try: - full = self._session.get( - url, params={"json-api-version": "1.0"}, timeout=_TIMEOUT_SECONDS, - ) - except requests.RequestException as exc: - log.warning("post-detail full re-fetch failed (post %s): %s", post_id, exc) - return None - fa = None - if full.status_code == 200: - try: - fp = full.json() - except ValueError: - fp = None - fdata = fp.get("data") if isinstance(fp, dict) else None - fa = fdata.get("attributes") if isinstance(fdata, dict) else None - fcontent = fa.get("content") if isinstance(fa, dict) else None - ptype = fa.get("post_type") if isinstance(fa, dict) else None - if isinstance(fcontent, str) and fcontent.strip(): - log.warning( - "post-detail: sparse fieldset gave null but FULL fetch has %d chars " - "(post %s, post_type=%s) — using full body", - len(fcontent), post_id, ptype, - ) - return fcontent + body = post_body_html(attrs) + if body and body.strip(): + log.info("post-detail: fetched %d chars (post %s)", len(body), post_id) + return body + ptype = attrs.get("post_type") if isinstance(attrs, dict) else None log.info( - "post-detail: empty/null content (post %s, post_type=%s) even on full " - "fetch — body not in the post resource", post_id, ptype, + "post-detail: no body (post %s, post_type=%s)", post_id, ptype, ) return None diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index 176b546..ddabca7 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -41,6 +41,7 @@ from urllib.parse import urlsplit import requests +from ..utils.prosemirror import post_body_html from .file_validator import is_validatable, quarantine_file, validate_file from .patreon_client import ( _BACKOFF_CAP_SECONDS, @@ -553,19 +554,22 @@ class PatreonDownloader: the per-media sidecar — a media-less post has no source file.""" attrs = post.get("attributes") or {} title = attrs.get("title") - content = attrs.get("content") - # The feed/list endpoint frequently returns an empty `content`; the full - # HTML body (formatting + inline + external links) only - # comes from the per-post detail endpoint. Enrich on first write for this - # post and MEMOIZE by mutating the shared `post` dict — so a multi-image - # post fetches detail exactly once, and a fully-seen post (no fresh - # download → no sidecar write) never pays the extra GET. - if (not isinstance(content, str) or not content.strip()) and self._content_fetcher: + # Resolve the body HTML from the feed attrs: legacy flat `content`, else + # convert the current `content_json_string` ProseMirror doc (#842). + content = post_body_html(attrs) + # The feed/list endpoint frequently returns an empty body; the full body + # only comes from the per-post detail endpoint. Enrich on first write for + # this post and MEMOIZE the RESOLVED HTML by mutating the shared `post` + # dict — so a multi-image post fetches detail at most once, the post-record + # body-length read reuses it, and a fully-seen post (no fresh download → no + # sidecar write) never pays the extra GET. + if (not content or not content.strip()) and self._content_fetcher: fetched = self._content_fetcher(str(post.get("id") or "")) if fetched: - attrs["content"] = fetched - post["attributes"] = attrs content = fetched + if isinstance(content, str) and content.strip(): + attrs["content"] = content + post["attributes"] = attrs published = attrs.get("published_at") url = attrs.get("url") data = { diff --git a/backend/app/utils/prosemirror.py b/backend/app/utils/prosemirror.py new file mode 100644 index 0000000..ac533b8 --- /dev/null +++ b/backend/app/utils/prosemirror.py @@ -0,0 +1,129 @@ +"""Render Patreon's post body to HTML. + +Patreon deprecated the flat `content` HTML field (it returns `null` now, on the +feed AND the detail endpoint, for every post type) and moved the real body to +`content_json_string` — a ProseMirror/TipTap document (a JSON string of typed +nodes + marks), returned only under the DEFAULT post fieldset. This module +converts that doc to the HTML the rest of the pipeline already expects (it then +flows through the existing nh3 sanitizer + inline-image localization). #842. + +Built from the observed Patreon doc shape: doc → [paragraph|heading|image|list…]; +paragraph content → [text(+marks: bold/italic/underline/strike/code/link), +hardBreak, image]. Unknown node types degrade to rendering their children so we +never silently drop text. +""" + +from __future__ import annotations + +import json +from html import escape + +# ProseMirror text-mark type → wrapping HTML tag (link handled separately — it +# needs an href attribute). +_MARK_TAGS = { + "bold": "strong", + "strong": "strong", + "italic": "em", + "em": "em", + "underline": "u", + "strike": "s", + "strikethrough": "s", + "code": "code", +} + + +def post_body_html(attrs: dict | None) -> str | None: + """Resolve a Patreon post's body HTML from its `attributes`. + + Prefers the legacy flat `content` HTML (older posts still carry it); falls + back to converting the `content_json_string` ProseMirror doc (current posts). + Returns None when neither holds a body. The ONE place body-source precedence + lives, so the feed-parse path and the detail-fetch path can't diverge. + """ + if not isinstance(attrs, dict): + return None + content = attrs.get("content") + if isinstance(content, str) and content.strip(): + return content + return prosemirror_to_html(attrs.get("content_json_string")) + + +def prosemirror_to_html(doc_json: str | None) -> str | None: + """Render a ProseMirror/TipTap doc (a JSON string) to HTML, or None for an + empty / blank / unparseable doc.""" + if not isinstance(doc_json, str) or not doc_json.strip(): + return None + try: + doc = json.loads(doc_json) + except (ValueError, TypeError): + return None + if not isinstance(doc, dict): + return None + return _render_nodes(doc.get("content")) or None + + +def _render_nodes(nodes: object) -> str: + if not isinstance(nodes, list): + return "" + return "".join(_render_node(n) for n in nodes if isinstance(n, dict)) + + +def _render_node(node: dict) -> str: + ntype = node.get("type") + if ntype == "text": + return _render_text(node) + if ntype == "hardBreak": + return "
" + if ntype == "image": + return _render_image(node) + if ntype == "horizontalRule": + return "
" + + inner = _render_nodes(node.get("content")) + if ntype == "paragraph": + return f"

{inner}

" + if ntype == "heading": + level = (node.get("attrs") or {}).get("level", 1) + if not isinstance(level, int) or not 1 <= level <= 6: + level = 1 + return f"{inner}" + if ntype == "blockquote": + return f"
{inner}
" + if ntype == "bulletList": + return f"
    {inner}
" + if ntype == "orderedList": + return f"
    {inner}
" + if ntype == "listItem": + return f"
  • {inner}
    • " + if ntype == "codeBlock": + return f"
    {inner}
    " + # Unknown / unhandled container — emit children so text is never dropped. + return inner + + +def _render_text(node: dict) -> str: + text = node.get("text") + if not isinstance(text, str) or not text: + return "" + out = escape(text) + for mark in node.get("marks") or []: + if not isinstance(mark, dict): + continue + mtype = mark.get("type") + if mtype == "link": + href = (mark.get("attrs") or {}).get("href") or "" + out = f'    {out}' + else: + tag = _MARK_TAGS.get(mtype) + if tag: + out = f"<{tag}>{out}" + return out + + +def _render_image(node: dict) -> str: + attrs = node.get("attrs") or {} + src = attrs.get("src") + if not isinstance(src, str) or not src: + return "" + alt = attrs.get("alt") or "" + return f'{escape(str(alt), quote=True)}' diff --git a/tests/test_patreon_client.py b/tests/test_patreon_client.py index 095d218..84edaa3 100644 --- a/tests/test_patreon_client.py +++ b/tests/test_patreon_client.py @@ -358,7 +358,10 @@ def test_request_sleep_paces_before_fetch(monkeypatch): # -- fetch_post_detail_content (best-effort full-body enrichment) ---------- -def test_fetch_post_detail_content_returns_body(monkeypatch): +def test_fetch_post_detail_content_returns_legacy_content(monkeypatch): + """Old posts still carry the flat `content` HTML — returned as-is. The detail + fetch uses the DEFAULT fieldset (no sparse `fields[post]`, which nulls the + body) — #842.""" payload = {"data": {"id": "42", "attributes": {"content": "

    full body

    "}}} captured: dict = {} client = PatreonClient(cookies_path=None) @@ -371,37 +374,28 @@ def test_fetch_post_detail_content_returns_body(monkeypatch): monkeypatch.setattr(client._session, "get", _get) assert client.fetch_post_detail_content("42") == "

    full body

    " assert captured["url"].endswith("/api/posts/42") - assert captured["params"]["fields[post]"] == "content" + # No sparse fieldset — that's what nulled the body on the live API. + assert not captured["params"] or "fields[post]" not in captured["params"] + + +def test_fetch_post_detail_content_converts_content_json_string(monkeypatch): + """Current posts: `content` is null and the body lives in the + `content_json_string` ProseMirror doc → converted to HTML (#842).""" + doc = ( + '{"type":"doc","content":[{"type":"paragraph","content":' + '[{"type":"text","text":"Hello"}]}]}' + ) + payload = {"data": {"attributes": {"content": None, "content_json_string": doc}}} + client = _client_returning(monkeypatch, _FakeResp(200, json_data=payload)) + assert client.fetch_post_detail_content("42") == "

    Hello

    " def test_fetch_post_detail_content_empty_body_is_none(monkeypatch): - payload = {"data": {"attributes": {"content": " "}}} + payload = {"data": {"attributes": {"content": " ", "content_json_string": ""}}} client = _client_returning(monkeypatch, _FakeResp(200, json_data=payload)) assert client.fetch_post_detail_content("42") is None -def test_fetch_post_detail_content_falls_back_to_full_fetch(monkeypatch): - """#842: when the sparse fields[post]=content request returns null (Patreon - does this for polls / embeds / body-only posts), a one-time FULL re-fetch - recovers the body — so inline-only / no-media posts still get their text.""" - client = PatreonClient(cookies_path=None) - seen_params: list = [] - - def _get(url, params=None, timeout=None): - seen_params.append(params or {}) - if "fields[post]" in (params or {}): - return _FakeResp(200, json_data={"data": {"attributes": {"content": None}}}) - return _FakeResp(200, json_data={ - "data": {"attributes": {"content": "

    body via full

    ", "post_type": "poll"}} - }) - - monkeypatch.setattr(client._session, "get", _get) - assert client.fetch_post_detail_content("42") == "

    body via full

    " - # Sparse request first, then a full re-fetch without the sparse fieldset. - assert "fields[post]" in seen_params[0] - assert "fields[post]" not in seen_params[1] - - def test_fetch_post_detail_content_blank_id_skips_request(monkeypatch): calls: list = [] client = PatreonClient(cookies_path=None) diff --git a/tests/test_patreon_contract.py b/tests/test_patreon_contract.py index 1f6a520..33be64d 100644 --- a/tests/test_patreon_contract.py +++ b/tests/test_patreon_contract.py @@ -66,7 +66,9 @@ def test_request_field_set_carries_parser_dependencies(): assert rel in includes, f"missing include relationship {rel}" post_fields = params["fields[post]"] - for field in ("content", "post_file", "image"): + # content_json_string is the CURRENT body field (Patreon nulled flat `content`, + # #842); content kept for legacy posts. Both must stay requested. + for field in ("content", "content_json_string", "post_file", "image"): assert field in post_fields, f"missing post field {field}" assert params["filter[campaign_id]"] == "5555" diff --git a/tests/test_prosemirror.py b/tests/test_prosemirror.py new file mode 100644 index 0000000..ec60262 --- /dev/null +++ b/tests/test_prosemirror.py @@ -0,0 +1,108 @@ +"""Unit tests for the Patreon ProseMirror→HTML body converter (#842). + +Patreon moved the post body from the flat `content` HTML field (now null) to +`content_json_string` (a ProseMirror doc). PURE module → no DB, fast lane. +""" + +from __future__ import annotations + +import json + +from backend.app.utils.prosemirror import post_body_html, prosemirror_to_html + + +def _doc(*nodes) -> str: + return json.dumps({"type": "doc", "content": list(nodes)}) + + +def test_paragraph_text_and_hardbreak(): + doc = _doc({ + "type": "paragraph", + "content": [ + {"type": "text", "text": "Hi there"}, + {"type": "hardBreak"}, + {"type": "text", "text": "line two"}, + ], + }) + assert prosemirror_to_html(doc) == "

    Hi there
    line two

    " + + +def test_marks_bold_italic_link(): + doc = _doc({ + "type": "paragraph", + "content": [ + {"type": "text", "marks": [{"type": "bold"}], "text": "B"}, + {"type": "text", "marks": [{"type": "italic"}], "text": "I"}, + { + "type": "text", + "marks": [{"type": "link", "attrs": {"href": "https://x.test/a"}}], + "text": "L", + }, + ], + }) + assert prosemirror_to_html(doc) == ( + '

    BIL

    ' + ) + + +def test_image_node_emits_img_with_cdn_src(): + doc = _doc({ + "type": "image", + "attrs": {"src": "https://cdn.test/p/post/1/abc/1.gif?token-hash=z", "alt": None}, + }) + out = prosemirror_to_html(doc) + assert out == '' + + +def test_lists_and_heading(): + doc = _doc( + {"type": "heading", "attrs": {"level": 2}, + "content": [{"type": "text", "text": "Status"}]}, + {"type": "bulletList", "content": [ + {"type": "listItem", "content": [ + {"type": "paragraph", "content": [{"type": "text", "text": "one"}]}]}, + ]}, + ) + assert prosemirror_to_html(doc) == ( + "

    Status

    " + ) + + +def test_text_is_html_escaped(): + doc = _doc({"type": "paragraph", + "content": [{"type": "text", "text": "a < b & c > d"}]}) + assert prosemirror_to_html(doc) == "

    a < b & c > d

    " + + +def test_unknown_node_renders_children_not_dropped(): + doc = _doc({"type": "weirdBlock", "content": [ + {"type": "paragraph", "content": [{"type": "text", "text": "keep me"}]}]}) + assert prosemirror_to_html(doc) == "

    keep me

    " + + +def test_empty_and_unparseable_return_none(): + assert prosemirror_to_html(None) is None + assert prosemirror_to_html("") is None + assert prosemirror_to_html(" ") is None + assert prosemirror_to_html("{not json") is None + assert prosemirror_to_html(json.dumps({"type": "doc", "content": []})) is None + + +def test_post_body_html_prefers_legacy_content(): + attrs = {"content": "

    legacy

    ", + "content_json_string": _doc({"type": "paragraph", + "content": [{"type": "text", "text": "new"}]})} + assert post_body_html(attrs) == "

    legacy

    " + + +def test_post_body_html_falls_back_to_json_string(): + attrs = {"content": None, + "content_json_string": _doc({"type": "paragraph", + "content": [{"type": "text", "text": "new"}]})} + assert post_body_html(attrs) == "

    new

    " + + +def test_post_body_html_none_when_both_empty(): + assert post_body_html({"content": None, "content_json_string": None}) is None + assert post_body_html({}) is None + assert post_body_html(None) is None