fix(patreon): read post body from content_json_string (ProseMirror), not the dead content field (#842)

THE empty-body root cause. Patreon deprecated the flat `content` HTML field —
it returns null on the feed AND the detail endpoint, for every post type
(confirmed against the live API: all 135 StickySpoodge posts, text_only/
image_file/poll alike). The real body now lives in `content_json_string` (a
ProseMirror/TipTap doc), returned only under the DEFAULT post fieldset — a sparse
fields[post]=content request omits it. Not credential, not post_type: a request
shape gone stale.

- NEW utils/prosemirror.py: ProseMirror doc -> HTML (paragraphs, marks
  bold/italic/underline/strike/code/link, hardBreak, inline images, lists,
  headings; unknown nodes degrade to children). post_body_html(attrs) = the one
  resolver: legacy content HTML else convert content_json_string.
- patreon_client: add content_json_string to the feed _FIELDS_POST; rewrite
  fetch_post_detail_content to use the DEFAULT fieldset (no sparse fields[post])
  and resolve via post_body_html (replaces the wrong sparse req + full-fetch
  fallback).
- patreon_downloader._write_sidecar_data: resolve body via post_body_html
  (feed content_json_string) before the detail-fetch; memoize resolved HTML.
- tests: prosemirror converter unit tests; client legacy + content_json_string
  paths; contract pins content_json_string.

Inline <img> nodes carry the CDN filehash → bodies now feed Phase-2 localization.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/app/services/patreon_client.py

@@ -40,6 +40,7 @@ from urllib.parse import parse_qs, urlsplit
 import requests
 
 from ..utils.paths import filehash_from_url, safe_ext
+from ..utils.prosemirror import post_body_html
 
 log = logging.getLogger(__name__)
 
@@ -84,8 +85,12 @@ _INCLUDE = (
     "native_video_insights,user,user_defined_tags,ti_checks"
 )
 _FIELDS_POST = (
-    "content,post_file,image,post_type,published_at,title,url,patreon_url,"
-    "current_user_can_view"
+    # `content` is the legacy flat-HTML body (Patreon now returns it null);
+    # `content_json_string` is the current ProseMirror-doc body. Request BOTH —
+    # post_body_html() prefers content (old posts) and falls back to converting
+    # content_json_string (current posts). #842.
+    "content,content_json_string,post_file,image,post_type,published_at,title,"
+    "url,patreon_url,current_user_can_view"
 )
 _FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name"
 _FIELDS_CAMPAIGN = "name,url"
@@ -571,29 +576,30 @@ class PatreonClient:
     # -- detail (full body enrichment) -------------------------------------
 
     def fetch_post_detail_content(self, post_id: str) -> str | None:
-        """Best-effort fetch of a post's full HTML `content` from the per-post
-        DETAIL endpoint (`/api/posts/{id}`).
+        """Best-effort fetch of a post's body (as HTML) from the per-post DETAIL
+        endpoint (`/api/posts/{id}`).
 
-        The feed/list endpoint (`/api/posts`) frequently returns `content` as
-        null even though we request it — the full body (its formatting, inline
-        `<img>`, and external `<a href>` links) only comes back from the
-        single-post detail resource. The downloader calls this to enrich a post
-        whose feed body was empty before writing the importer sidecar.
+        Patreon deprecated the flat `content` HTML field — it returns null on the
+        feed AND the detail endpoint, for every post type. The real body now lives
+        in `content_json_string` (a ProseMirror doc), and is only returned under
+        the DEFAULT post fieldset: a sparse `fields[post]=content` request OMITS
+        it (confirmed against the live API 2026-06-15). So we request the default
+        fieldset (no `fields[post]`) and resolve the body via `post_body_html`
+        (content → else convert content_json_string). The downloader calls this to
+        enrich a post whose feed body was empty before writing the sidecar.
 
-        Best-effort BY DESIGN: the media download is the primary job, so a body
-        we can't fetch must never fail the walk. Every failure path (no id,
-        transport error, non-200, non-JSON, missing/empty content) returns None
-        rather than raising — distinct from the loud drift/auth raises on the
-        feed path, which gate real downloads.
+        Best-effort BY DESIGN: a body we can't fetch must never fail the walk.
+        Every failure path returns None rather than raising — distinct from the
+        loud drift/auth raises on the feed path, which gate real downloads.
         """
         if not post_id:
             return None
         if self._request_sleep > 0:
             time.sleep(self._request_sleep)  # pace the API endpoint (plan #703)
-        url = f"{_POSTS_URL}/{post_id}"
-        params = {"fields[post]": "content", "json-api-version": "1.0"}
+        # No `fields[post]` — the default fieldset is the only shape that returns
+        # content_json_string (the body). A sparse fieldset nulls it out.
         try:
-            resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS)
+            resp = self._session.get(f"{_POSTS_URL}/{post_id}", timeout=_TIMEOUT_SECONDS)
         except requests.RequestException as exc:
             log.warning("Patreon post-detail fetch failed (post %s): %s", post_id, exc)
             return None
@@ -608,43 +614,13 @@ class PatreonClient:
             return None
         data = payload.get("data") if isinstance(payload, dict) else None
         attrs = data.get("attributes") if isinstance(data, dict) else None
-        content = attrs.get("content") if isinstance(attrs, dict) else None
-        if isinstance(content, str) and content.strip():
-            log.info("post-detail: fetched %d chars (post %s)", len(content), post_id)
-            return content
-        # Sparse `fields[post]=content` came back null/empty. Patreon serves null
-        # under the sparse fieldset for some post types (polls, embeds/films,
-        # body-only announcements) even when the body plainly exists. Re-fetch the
-        # FULL post resource ONCE — only the empty cases pay this extra GET — which
-        # both DIAGNOSES (logs post_type) and RECOVERS the body when the sparse
-        # fieldset was the cause (#842).
-        try:
-            full = self._session.get(
-                url, params={"json-api-version": "1.0"}, timeout=_TIMEOUT_SECONDS,
-            )
-        except requests.RequestException as exc:
-            log.warning("post-detail full re-fetch failed (post %s): %s", post_id, exc)
-            return None
-        fa = None
-        if full.status_code == 200:
-            try:
-                fp = full.json()
-            except ValueError:
-                fp = None
-            fdata = fp.get("data") if isinstance(fp, dict) else None
-            fa = fdata.get("attributes") if isinstance(fdata, dict) else None
-        fcontent = fa.get("content") if isinstance(fa, dict) else None
-        ptype = fa.get("post_type") if isinstance(fa, dict) else None
-        if isinstance(fcontent, str) and fcontent.strip():
-            log.warning(
-                "post-detail: sparse fieldset gave null but FULL fetch has %d chars "
-                "(post %s, post_type=%s) — using full body",
-                len(fcontent), post_id, ptype,
-            )
-            return fcontent
+        body = post_body_html(attrs)
+        if body and body.strip():
+            log.info("post-detail: fetched %d chars (post %s)", len(body), post_id)
+            return body
+        ptype = attrs.get("post_type") if isinstance(attrs, dict) else None
         log.info(
-            "post-detail: empty/null content (post %s, post_type=%s) even on full "
-            "fetch — body not in the post resource", post_id, ptype,
+            "post-detail: no body (post %s, post_type=%s)", post_id, ptype,
         )
         return None
 

backend/app/services/patreon_downloader.py

@@ -41,6 +41,7 @@ from urllib.parse import urlsplit
 
 import requests
 
+from ..utils.prosemirror import post_body_html
 from .file_validator import is_validatable, quarantine_file, validate_file
 from .patreon_client import (
     _BACKOFF_CAP_SECONDS,
@@ -553,19 +554,22 @@ class PatreonDownloader:
         the per-media sidecar — a media-less post has no source file."""
         attrs = post.get("attributes") or {}
         title = attrs.get("title")
-        content = attrs.get("content")
-        # The feed/list endpoint frequently returns an empty `content`; the full
-        # HTML body (formatting + inline <img> + external <a href> links) only
-        # comes from the per-post detail endpoint. Enrich on first write for this
-        # post and MEMOIZE by mutating the shared `post` dict — so a multi-image
-        # post fetches detail exactly once, and a fully-seen post (no fresh
-        # download → no sidecar write) never pays the extra GET.
-        if (not isinstance(content, str) or not content.strip()) and self._content_fetcher:
+        # Resolve the body HTML from the feed attrs: legacy flat `content`, else
+        # convert the current `content_json_string` ProseMirror doc (#842).
+        content = post_body_html(attrs)
+        # The feed/list endpoint frequently returns an empty body; the full body
+        # only comes from the per-post detail endpoint. Enrich on first write for
+        # this post and MEMOIZE the RESOLVED HTML by mutating the shared `post`
+        # dict — so a multi-image post fetches detail at most once, the post-record
+        # body-length read reuses it, and a fully-seen post (no fresh download → no
+        # sidecar write) never pays the extra GET.
+        if (not content or not content.strip()) and self._content_fetcher:
             fetched = self._content_fetcher(str(post.get("id") or ""))
             if fetched:
-                attrs["content"] = fetched
-                post["attributes"] = attrs
                 content = fetched
+        if isinstance(content, str) and content.strip():
+            attrs["content"] = content
+            post["attributes"] = attrs
         published = attrs.get("published_at")
         url = attrs.get("url")
         data = {

backend/app/utils/prosemirror.py

@@ -0,0 +1,129 @@
+"""Render Patreon's post body to HTML.
+
+Patreon deprecated the flat `content` HTML field (it returns `null` now, on the
+feed AND the detail endpoint, for every post type) and moved the real body to
+`content_json_string` — a ProseMirror/TipTap document (a JSON string of typed
+nodes + marks), returned only under the DEFAULT post fieldset. This module
+converts that doc to the HTML the rest of the pipeline already expects (it then
+flows through the existing nh3 sanitizer + inline-image localization). #842.
+
+Built from the observed Patreon doc shape: doc → [paragraph|heading|image|list…];
+paragraph content → [text(+marks: bold/italic/underline/strike/code/link),
+hardBreak, image]. Unknown node types degrade to rendering their children so we
+never silently drop text.
+"""
+
+from __future__ import annotations
+
+import json
+from html import escape
+
+# ProseMirror text-mark type → wrapping HTML tag (link handled separately — it
+# needs an href attribute).
+_MARK_TAGS = {
+    "bold": "strong",
+    "strong": "strong",
+    "italic": "em",
+    "em": "em",
+    "underline": "u",
+    "strike": "s",
+    "strikethrough": "s",
+    "code": "code",
+}
+
+
+def post_body_html(attrs: dict | None) -> str | None:
+    """Resolve a Patreon post's body HTML from its `attributes`.
+
+    Prefers the legacy flat `content` HTML (older posts still carry it); falls
+    back to converting the `content_json_string` ProseMirror doc (current posts).
+    Returns None when neither holds a body. The ONE place body-source precedence
+    lives, so the feed-parse path and the detail-fetch path can't diverge.
+    """
+    if not isinstance(attrs, dict):
+        return None
+    content = attrs.get("content")
+    if isinstance(content, str) and content.strip():
+        return content
+    return prosemirror_to_html(attrs.get("content_json_string"))
+
+
+def prosemirror_to_html(doc_json: str | None) -> str | None:
+    """Render a ProseMirror/TipTap doc (a JSON string) to HTML, or None for an
+    empty / blank / unparseable doc."""
+    if not isinstance(doc_json, str) or not doc_json.strip():
+        return None
+    try:
+        doc = json.loads(doc_json)
+    except (ValueError, TypeError):
+        return None
+    if not isinstance(doc, dict):
+        return None
+    return _render_nodes(doc.get("content")) or None
+
+
+def _render_nodes(nodes: object) -> str:
+    if not isinstance(nodes, list):
+        return ""
+    return "".join(_render_node(n) for n in nodes if isinstance(n, dict))
+
+
+def _render_node(node: dict) -> str:
+    ntype = node.get("type")
+    if ntype == "text":
+        return _render_text(node)
+    if ntype == "hardBreak":
+        return "<br>"
+    if ntype == "image":
+        return _render_image(node)
+    if ntype == "horizontalRule":
+        return "<hr>"
+
+    inner = _render_nodes(node.get("content"))
+    if ntype == "paragraph":
+        return f"<p>{inner}</p>"
+    if ntype == "heading":
+        level = (node.get("attrs") or {}).get("level", 1)
+        if not isinstance(level, int) or not 1 <= level <= 6:
+            level = 1
+        return f"<h{level}>{inner}</h{level}>"
+    if ntype == "blockquote":
+        return f"<blockquote>{inner}</blockquote>"
+    if ntype == "bulletList":
+        return f"<ul>{inner}</ul>"
+    if ntype == "orderedList":
+        return f"<ol>{inner}</ol>"
+    if ntype == "listItem":
+        return f"<li>{inner}</li>"
+    if ntype == "codeBlock":
+        return f"<pre><code>{inner}</code></pre>"
+    # Unknown / unhandled container — emit children so text is never dropped.
+    return inner
+
+
+def _render_text(node: dict) -> str:
+    text = node.get("text")
+    if not isinstance(text, str) or not text:
+        return ""
+    out = escape(text)
+    for mark in node.get("marks") or []:
+        if not isinstance(mark, dict):
+            continue
+        mtype = mark.get("type")
+        if mtype == "link":
+            href = (mark.get("attrs") or {}).get("href") or ""
+            out = f'<a href="{escape(str(href), quote=True)}">{out}</a>'
+        else:
+            tag = _MARK_TAGS.get(mtype)
+            if tag:
+                out = f"<{tag}>{out}</{tag}>"
+    return out
+
+
+def _render_image(node: dict) -> str:
+    attrs = node.get("attrs") or {}
+    src = attrs.get("src")
+    if not isinstance(src, str) or not src:
+        return ""
+    alt = attrs.get("alt") or ""
+    return f'<img src="{escape(src, quote=True)}" alt="{escape(str(alt), quote=True)}">'