fix(patreon): read post body from content_json_string (ProseMirror), not the dead content field (#842)

THE empty-body root cause. Patreon deprecated the flat `content` HTML field —
it returns null on the feed AND the detail endpoint, for every post type
(confirmed against the live API: all 135 StickySpoodge posts, text_only/
image_file/poll alike). The real body now lives in `content_json_string` (a
ProseMirror/TipTap doc), returned only under the DEFAULT post fieldset — a sparse
fields[post]=content request omits it. Not credential, not post_type: a request
shape gone stale.

- NEW utils/prosemirror.py: ProseMirror doc -> HTML (paragraphs, marks
  bold/italic/underline/strike/code/link, hardBreak, inline images, lists,
  headings; unknown nodes degrade to children). post_body_html(attrs) = the one
  resolver: legacy content HTML else convert content_json_string.
- patreon_client: add content_json_string to the feed _FIELDS_POST; rewrite
  fetch_post_detail_content to use the DEFAULT fieldset (no sparse fields[post])
  and resolve via post_body_html (replaces the wrong sparse req + full-fetch
  fallback).
- patreon_downloader._write_sidecar_data: resolve body via post_body_html
  (feed content_json_string) before the detail-fetch; memoize resolved HTML.
- tests: prosemirror converter unit tests; client legacy + content_json_string
  paths; contract pins content_json_string.

Inline <img> nodes carry the CDN filehash → bodies now feed Phase-2 localization.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/app/services/patreon_client.py

@@ -40,6 +40,7 @@ from urllib.parse import parse_qs, urlsplit
 import requests
 
 from ..utils.paths import filehash_from_url, safe_ext
+from ..utils.prosemirror import post_body_html
 
 log = logging.getLogger(__name__)
 
@@ -84,8 +85,12 @@ _INCLUDE = (
     "native_video_insights,user,user_defined_tags,ti_checks"
 )
 _FIELDS_POST = (
-    "content,post_file,image,post_type,published_at,title,url,patreon_url,"
+    # `content` is the legacy flat-HTML body (Patreon now returns it null);
-    "current_user_can_view"
+    # `content_json_string` is the current ProseMirror-doc body. Request BOTH —
+    # post_body_html() prefers content (old posts) and falls back to converting
+    # content_json_string (current posts). #842.
+    "content,content_json_string,post_file,image,post_type,published_at,title,"
+    "url,patreon_url,current_user_can_view"
 )
 _FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name"
 _FIELDS_CAMPAIGN = "name,url"
@@ -571,29 +576,30 @@ class PatreonClient:
     # -- detail (full body enrichment) -------------------------------------
 
     def fetch_post_detail_content(self, post_id: str) -> str | None:
-        """Best-effort fetch of a post's full HTML `content` from the per-post
+        """Best-effort fetch of a post's body (as HTML) from the per-post DETAIL
-        DETAIL endpoint (`/api/posts/{id}`).
+        endpoint (`/api/posts/{id}`).
 
-        The feed/list endpoint (`/api/posts`) frequently returns `content` as
+        Patreon deprecated the flat `content` HTML field — it returns null on the
-        null even though we request it — the full body (its formatting, inline
+        feed AND the detail endpoint, for every post type. The real body now lives
-        `<img>`, and external `<a href>` links) only comes back from the
+        in `content_json_string` (a ProseMirror doc), and is only returned under
-        single-post detail resource. The downloader calls this to enrich a post
+        the DEFAULT post fieldset: a sparse `fields[post]=content` request OMITS
-        whose feed body was empty before writing the importer sidecar.
+        it (confirmed against the live API 2026-06-15). So we request the default
+        fieldset (no `fields[post]`) and resolve the body via `post_body_html`
+        (content → else convert content_json_string). The downloader calls this to
+        enrich a post whose feed body was empty before writing the sidecar.
 
-        Best-effort BY DESIGN: the media download is the primary job, so a body
+        Best-effort BY DESIGN: a body we can't fetch must never fail the walk.
-        we can't fetch must never fail the walk. Every failure path (no id,
+        Every failure path returns None rather than raising — distinct from the
-        transport error, non-200, non-JSON, missing/empty content) returns None
+        loud drift/auth raises on the feed path, which gate real downloads.
-        rather than raising — distinct from the loud drift/auth raises on the
-        feed path, which gate real downloads.
         """
         if not post_id:
             return None
         if self._request_sleep > 0:
             time.sleep(self._request_sleep)  # pace the API endpoint (plan #703)
-        url = f"{_POSTS_URL}/{post_id}"
+        # No `fields[post]` — the default fieldset is the only shape that returns
-        params = {"fields[post]": "content", "json-api-version": "1.0"}
+        # content_json_string (the body). A sparse fieldset nulls it out.
         try:
-            resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS)
+            resp = self._session.get(f"{_POSTS_URL}/{post_id}", timeout=_TIMEOUT_SECONDS)
         except requests.RequestException as exc:
             log.warning("Patreon post-detail fetch failed (post %s): %s", post_id, exc)
             return None
@@ -608,43 +614,13 @@ class PatreonClient:
             return None
         data = payload.get("data") if isinstance(payload, dict) else None
         attrs = data.get("attributes") if isinstance(data, dict) else None
-        content = attrs.get("content") if isinstance(attrs, dict) else None
+        body = post_body_html(attrs)
-        if isinstance(content, str) and content.strip():
+        if body and body.strip():
-            log.info("post-detail: fetched %d chars (post %s)", len(content), post_id)
+            log.info("post-detail: fetched %d chars (post %s)", len(body), post_id)
-            return content
+            return body
-        # Sparse `fields[post]=content` came back null/empty. Patreon serves null
+        ptype = attrs.get("post_type") if isinstance(attrs, dict) else None
-        # under the sparse fieldset for some post types (polls, embeds/films,
-        # body-only announcements) even when the body plainly exists. Re-fetch the
-        # FULL post resource ONCE — only the empty cases pay this extra GET — which
-        # both DIAGNOSES (logs post_type) and RECOVERS the body when the sparse
-        # fieldset was the cause (#842).
-        try:
-            full = self._session.get(
-                url, params={"json-api-version": "1.0"}, timeout=_TIMEOUT_SECONDS,
-            )
-        except requests.RequestException as exc:
-            log.warning("post-detail full re-fetch failed (post %s): %s", post_id, exc)
-            return None
-        fa = None
-        if full.status_code == 200:
-            try:
-                fp = full.json()
-            except ValueError:
-                fp = None
-            fdata = fp.get("data") if isinstance(fp, dict) else None
-            fa = fdata.get("attributes") if isinstance(fdata, dict) else None
-        fcontent = fa.get("content") if isinstance(fa, dict) else None
-        ptype = fa.get("post_type") if isinstance(fa, dict) else None
-        if isinstance(fcontent, str) and fcontent.strip():
-            log.warning(
-                "post-detail: sparse fieldset gave null but FULL fetch has %d chars "
-                "(post %s, post_type=%s) — using full body",
-                len(fcontent), post_id, ptype,
-            )
-            return fcontent
         log.info(
-            "post-detail: empty/null content (post %s, post_type=%s) even on full "
+            "post-detail: no body (post %s, post_type=%s)", post_id, ptype,
-            "fetch — body not in the post resource", post_id, ptype,
         )
         return None
 

backend/app/services/patreon_downloader.py

@@ -41,6 +41,7 @@ from urllib.parse import urlsplit
 
 import requests
 
+from ..utils.prosemirror import post_body_html
 from .file_validator import is_validatable, quarantine_file, validate_file
 from .patreon_client import (
     _BACKOFF_CAP_SECONDS,
@@ -553,19 +554,22 @@ class PatreonDownloader:
         the per-media sidecar — a media-less post has no source file."""
         attrs = post.get("attributes") or {}
         title = attrs.get("title")
-        content = attrs.get("content")
+        # Resolve the body HTML from the feed attrs: legacy flat `content`, else
-        # The feed/list endpoint frequently returns an empty `content`; the full
+        # convert the current `content_json_string` ProseMirror doc (#842).
-        # HTML body (formatting + inline <img> + external <a href> links) only
+        content = post_body_html(attrs)
-        # comes from the per-post detail endpoint. Enrich on first write for this
+        # The feed/list endpoint frequently returns an empty body; the full body
-        # post and MEMOIZE by mutating the shared `post` dict — so a multi-image
+        # only comes from the per-post detail endpoint. Enrich on first write for
-        # post fetches detail exactly once, and a fully-seen post (no fresh
+        # this post and MEMOIZE the RESOLVED HTML by mutating the shared `post`
-        # download → no sidecar write) never pays the extra GET.
+        # dict — so a multi-image post fetches detail at most once, the post-record
-        if (not isinstance(content, str) or not content.strip()) and self._content_fetcher:
+        # body-length read reuses it, and a fully-seen post (no fresh download → no
+        # sidecar write) never pays the extra GET.
+        if (not content or not content.strip()) and self._content_fetcher:
             fetched = self._content_fetcher(str(post.get("id") or ""))
             if fetched:
-                attrs["content"] = fetched
-                post["attributes"] = attrs
                 content = fetched
+        if isinstance(content, str) and content.strip():
+            attrs["content"] = content
+            post["attributes"] = attrs
         published = attrs.get("published_at")
         url = attrs.get("url")
         data = {

backend/app/utils/prosemirror.py

@@ -0,0 +1,129 @@
+"""Render Patreon's post body to HTML.
+
+Patreon deprecated the flat `content` HTML field (it returns `null` now, on the
+feed AND the detail endpoint, for every post type) and moved the real body to
+`content_json_string` — a ProseMirror/TipTap document (a JSON string of typed
+nodes + marks), returned only under the DEFAULT post fieldset. This module
+converts that doc to the HTML the rest of the pipeline already expects (it then
+flows through the existing nh3 sanitizer + inline-image localization). #842.
+
+Built from the observed Patreon doc shape: doc → [paragraph|heading|image|list…];
+paragraph content → [text(+marks: bold/italic/underline/strike/code/link),
+hardBreak, image]. Unknown node types degrade to rendering their children so we
+never silently drop text.
+"""
+
+from __future__ import annotations
+
+import json
+from html import escape
+
+# ProseMirror text-mark type → wrapping HTML tag (link handled separately — it
+# needs an href attribute).
+_MARK_TAGS = {
+    "bold": "strong",
+    "strong": "strong",
+    "italic": "em",
+    "em": "em",
+    "underline": "u",
+    "strike": "s",
+    "strikethrough": "s",
+    "code": "code",
+}
+
+
+def post_body_html(attrs: dict | None) -> str | None:
+    """Resolve a Patreon post's body HTML from its `attributes`.
+
+    Prefers the legacy flat `content` HTML (older posts still carry it); falls
+    back to converting the `content_json_string` ProseMirror doc (current posts).
+    Returns None when neither holds a body. The ONE place body-source precedence
+    lives, so the feed-parse path and the detail-fetch path can't diverge.
+    """
+    if not isinstance(attrs, dict):
+        return None
+    content = attrs.get("content")
+    if isinstance(content, str) and content.strip():
+        return content
+    return prosemirror_to_html(attrs.get("content_json_string"))
+
+
+def prosemirror_to_html(doc_json: str | None) -> str | None:
+    """Render a ProseMirror/TipTap doc (a JSON string) to HTML, or None for an
+    empty / blank / unparseable doc."""
+    if not isinstance(doc_json, str) or not doc_json.strip():
+        return None
+    try:
+        doc = json.loads(doc_json)
+    except (ValueError, TypeError):
+        return None
+    if not isinstance(doc, dict):
+        return None
+    return _render_nodes(doc.get("content")) or None
+
+
+def _render_nodes(nodes: object) -> str:
+    if not isinstance(nodes, list):
+        return ""
+    return "".join(_render_node(n) for n in nodes if isinstance(n, dict))
+
+
+def _render_node(node: dict) -> str:
+    ntype = node.get("type")
+    if ntype == "text":
+        return _render_text(node)
+    if ntype == "hardBreak":
+        return "<br>"
+    if ntype == "image":
+        return _render_image(node)
+    if ntype == "horizontalRule":
+        return "<hr>"
+
+    inner = _render_nodes(node.get("content"))
+    if ntype == "paragraph":
+        return f"<p>{inner}</p>"
+    if ntype == "heading":
+        level = (node.get("attrs") or {}).get("level", 1)
+        if not isinstance(level, int) or not 1 <= level <= 6:
+            level = 1
+        return f"<h{level}>{inner}</h{level}>"
+    if ntype == "blockquote":
+        return f"<blockquote>{inner}</blockquote>"
+    if ntype == "bulletList":
+        return f"<ul>{inner}</ul>"
+    if ntype == "orderedList":
+        return f"<ol>{inner}</ol>"
+    if ntype == "listItem":
+        return f"<li>{inner}</li>"
+    if ntype == "codeBlock":
+        return f"<pre><code>{inner}</code></pre>"
+    # Unknown / unhandled container — emit children so text is never dropped.
+    return inner
+
+
+def _render_text(node: dict) -> str:
+    text = node.get("text")
+    if not isinstance(text, str) or not text:
+        return ""
+    out = escape(text)
+    for mark in node.get("marks") or []:
+        if not isinstance(mark, dict):
+            continue
+        mtype = mark.get("type")
+        if mtype == "link":
+            href = (mark.get("attrs") or {}).get("href") or ""
+            out = f'<a href="{escape(str(href), quote=True)}">{out}</a>'
+        else:
+            tag = _MARK_TAGS.get(mtype)
+            if tag:
+                out = f"<{tag}>{out}</{tag}>"
+    return out
+
+
+def _render_image(node: dict) -> str:
+    attrs = node.get("attrs") or {}
+    src = attrs.get("src")
+    if not isinstance(src, str) or not src:
+        return ""
+    alt = attrs.get("alt") or ""
+    return f'<img src="{escape(src, quote=True)}" alt="{escape(str(alt), quote=True)}">'

tests/test_patreon_client.py

@@ -358,7 +358,10 @@ def test_request_sleep_paces_before_fetch(monkeypatch):
 # -- fetch_post_detail_content (best-effort full-body enrichment) ----------
 
 
-def test_fetch_post_detail_content_returns_body(monkeypatch):
+def test_fetch_post_detail_content_returns_legacy_content(monkeypatch):
+    """Old posts still carry the flat `content` HTML — returned as-is. The detail
+    fetch uses the DEFAULT fieldset (no sparse `fields[post]`, which nulls the
+    body) — #842."""
     payload = {"data": {"id": "42", "attributes": {"content": "<p>full body</p>"}}}
     captured: dict = {}
     client = PatreonClient(cookies_path=None)
@@ -371,37 +374,28 @@ def test_fetch_post_detail_content_returns_body(monkeypatch):
     monkeypatch.setattr(client._session, "get", _get)
     assert client.fetch_post_detail_content("42") == "<p>full body</p>"
     assert captured["url"].endswith("/api/posts/42")
-    assert captured["params"]["fields[post]"] == "content"
+    # No sparse fieldset — that's what nulled the body on the live API.
+    assert not captured["params"] or "fields[post]" not in captured["params"]
+
+
+def test_fetch_post_detail_content_converts_content_json_string(monkeypatch):
+    """Current posts: `content` is null and the body lives in the
+    `content_json_string` ProseMirror doc → converted to HTML (#842)."""
+    doc = (
+        '{"type":"doc","content":[{"type":"paragraph","content":'
+        '[{"type":"text","text":"Hello"}]}]}'
+    )
+    payload = {"data": {"attributes": {"content": None, "content_json_string": doc}}}
+    client = _client_returning(monkeypatch, _FakeResp(200, json_data=payload))
+    assert client.fetch_post_detail_content("42") == "<p>Hello</p>"
 
 
 def test_fetch_post_detail_content_empty_body_is_none(monkeypatch):
-    payload = {"data": {"attributes": {"content": "   "}}}
+    payload = {"data": {"attributes": {"content": "   ", "content_json_string": ""}}}
     client = _client_returning(monkeypatch, _FakeResp(200, json_data=payload))
     assert client.fetch_post_detail_content("42") is None
 
 
-def test_fetch_post_detail_content_falls_back_to_full_fetch(monkeypatch):
-    """#842: when the sparse fields[post]=content request returns null (Patreon
-    does this for polls / embeds / body-only posts), a one-time FULL re-fetch
-    recovers the body — so inline-only / no-media posts still get their text."""
-    client = PatreonClient(cookies_path=None)
-    seen_params: list = []
-
-    def _get(url, params=None, timeout=None):
-        seen_params.append(params or {})
-        if "fields[post]" in (params or {}):
-            return _FakeResp(200, json_data={"data": {"attributes": {"content": None}}})
-        return _FakeResp(200, json_data={
-            "data": {"attributes": {"content": "<p>body via full</p>", "post_type": "poll"}}
-        })
-
-    monkeypatch.setattr(client._session, "get", _get)
-    assert client.fetch_post_detail_content("42") == "<p>body via full</p>"
-    # Sparse request first, then a full re-fetch without the sparse fieldset.
-    assert "fields[post]" in seen_params[0]
-    assert "fields[post]" not in seen_params[1]
-
-
 def test_fetch_post_detail_content_blank_id_skips_request(monkeypatch):
     calls: list = []
     client = PatreonClient(cookies_path=None)

tests/test_patreon_contract.py

@@ -66,7 +66,9 @@ def test_request_field_set_carries_parser_dependencies():
         assert rel in includes, f"missing include relationship {rel}"
 
     post_fields = params["fields[post]"]
-    for field in ("content", "post_file", "image"):
+    # content_json_string is the CURRENT body field (Patreon nulled flat `content`,
+    # #842); content kept for legacy posts. Both must stay requested.
+    for field in ("content", "content_json_string", "post_file", "image"):
         assert field in post_fields, f"missing post field {field}"
 
     assert params["filter[campaign_id]"] == "5555"

tests/test_prosemirror.py

@@ -0,0 +1,108 @@
+"""Unit tests for the Patreon ProseMirror→HTML body converter (#842).
+
+Patreon moved the post body from the flat `content` HTML field (now null) to
+`content_json_string` (a ProseMirror doc). PURE module → no DB, fast lane.
+"""
+
+from __future__ import annotations
+
+import json
+
+from backend.app.utils.prosemirror import post_body_html, prosemirror_to_html
+
+
+def _doc(*nodes) -> str:
+    return json.dumps({"type": "doc", "content": list(nodes)})
+
+
+def test_paragraph_text_and_hardbreak():
+    doc = _doc({
+        "type": "paragraph",
+        "content": [
+            {"type": "text", "text": "Hi there"},
+            {"type": "hardBreak"},
+            {"type": "text", "text": "line two"},
+        ],
+    })
+    assert prosemirror_to_html(doc) == "<p>Hi there<br>line two</p>"
+
+
+def test_marks_bold_italic_link():
+    doc = _doc({
+        "type": "paragraph",
+        "content": [
+            {"type": "text", "marks": [{"type": "bold"}], "text": "B"},
+            {"type": "text", "marks": [{"type": "italic"}], "text": "I"},
+            {
+                "type": "text",
+                "marks": [{"type": "link", "attrs": {"href": "https://x.test/a"}}],
+                "text": "L",
+            },
+        ],
+    })
+    assert prosemirror_to_html(doc) == (
+        '<p><strong>B</strong><em>I</em><a href="https://x.test/a">L</a></p>'
+    )
+
+
+def test_image_node_emits_img_with_cdn_src():
+    doc = _doc({
+        "type": "image",
+        "attrs": {"src": "https://cdn.test/p/post/1/abc/1.gif?token-hash=z", "alt": None},
+    })
+    out = prosemirror_to_html(doc)
+    assert out == '<img src="https://cdn.test/p/post/1/abc/1.gif?token-hash=z" alt="">'
+
+
+def test_lists_and_heading():
+    doc = _doc(
+        {"type": "heading", "attrs": {"level": 2},
+         "content": [{"type": "text", "text": "Status"}]},
+        {"type": "bulletList", "content": [
+            {"type": "listItem", "content": [
+                {"type": "paragraph", "content": [{"type": "text", "text": "one"}]}]},
+        ]},
+    )
+    assert prosemirror_to_html(doc) == (
+        "<h2>Status</h2><ul><li><p>one</p></li></ul>"
+    )
+
+
+def test_text_is_html_escaped():
+    doc = _doc({"type": "paragraph",
+                "content": [{"type": "text", "text": "a < b & c > d"}]})
+    assert prosemirror_to_html(doc) == "<p>a &lt; b &amp; c &gt; d</p>"
+
+
+def test_unknown_node_renders_children_not_dropped():
+    doc = _doc({"type": "weirdBlock", "content": [
+        {"type": "paragraph", "content": [{"type": "text", "text": "keep me"}]}]})
+    assert prosemirror_to_html(doc) == "<p>keep me</p>"
+
+
+def test_empty_and_unparseable_return_none():
+    assert prosemirror_to_html(None) is None
+    assert prosemirror_to_html("") is None
+    assert prosemirror_to_html("   ") is None
+    assert prosemirror_to_html("{not json") is None
+    assert prosemirror_to_html(json.dumps({"type": "doc", "content": []})) is None
+
+
+def test_post_body_html_prefers_legacy_content():
+    attrs = {"content": "<p>legacy</p>",
+             "content_json_string": _doc({"type": "paragraph",
+                                          "content": [{"type": "text", "text": "new"}]})}
+    assert post_body_html(attrs) == "<p>legacy</p>"
+
+
+def test_post_body_html_falls_back_to_json_string():
+    attrs = {"content": None,
+             "content_json_string": _doc({"type": "paragraph",
+                                          "content": [{"type": "text", "text": "new"}]})}
+    assert post_body_html(attrs) == "<p>new</p>"
+
+
+def test_post_body_html_none_when_both_empty():
+    assert post_body_html({"content": None, "content_json_string": None}) is None
+    assert post_body_html({}) is None
+    assert post_body_html(None) is None