fix(patreon): read post body from content_json_string (ProseMirror), not the dead content field (#842)

THE empty-body root cause. Patreon deprecated the flat `content` HTML field —
it returns null on the feed AND the detail endpoint, for every post type
(confirmed against the live API: all 135 StickySpoodge posts, text_only/
image_file/poll alike). The real body now lives in `content_json_string` (a
ProseMirror/TipTap doc), returned only under the DEFAULT post fieldset — a sparse
fields[post]=content request omits it. Not credential, not post_type: a request
shape gone stale.

- NEW utils/prosemirror.py: ProseMirror doc -> HTML (paragraphs, marks
  bold/italic/underline/strike/code/link, hardBreak, inline images, lists,
  headings; unknown nodes degrade to children). post_body_html(attrs) = the one
  resolver: legacy content HTML else convert content_json_string.
- patreon_client: add content_json_string to the feed _FIELDS_POST; rewrite
  fetch_post_detail_content to use the DEFAULT fieldset (no sparse fields[post])
  and resolve via post_body_html (replaces the wrong sparse req + full-fetch
  fallback).
- patreon_downloader._write_sidecar_data: resolve body via post_body_html
  (feed content_json_string) before the detail-fetch; memoize resolved HTML.
- tests: prosemirror converter unit tests; client legacy + content_json_string
  paths; contract pins content_json_string.

Inline <img> nodes carry the CDN filehash → bodies now feed Phase-2 localization.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/app/services/patreon_client.py

@@ -40,6 +40,7 @@ from urllib.parse import parse_qs, urlsplit
 import requests
 
 from ..utils.paths import filehash_from_url, safe_ext
+from ..utils.prosemirror import post_body_html
 
 log = logging.getLogger(__name__)
 
@@ -84,8 +85,12 @@ _INCLUDE = (
     "native_video_insights,user,user_defined_tags,ti_checks"
 )
 _FIELDS_POST = (
-    "content,post_file,image,post_type,published_at,title,url,patreon_url,"
-    "current_user_can_view"
+    # `content` is the legacy flat-HTML body (Patreon now returns it null);
+    # `content_json_string` is the current ProseMirror-doc body. Request BOTH —
+    # post_body_html() prefers content (old posts) and falls back to converting
+    # content_json_string (current posts). #842.
+    "content,content_json_string,post_file,image,post_type,published_at,title,"
+    "url,patreon_url,current_user_can_view"
 )
 _FIELDS_MEDIA = "id,image_urls,download_url,metadata,file_name"
 _FIELDS_CAMPAIGN = "name,url"
@@ -571,29 +576,30 @@ class PatreonClient:
     # -- detail (full body enrichment) -------------------------------------
 
     def fetch_post_detail_content(self, post_id: str) -> str | None:
-        """Best-effort fetch of a post's full HTML `content` from the per-post
-        DETAIL endpoint (`/api/posts/{id}`).
+        """Best-effort fetch of a post's body (as HTML) from the per-post DETAIL
+        endpoint (`/api/posts/{id}`).
 
-        The feed/list endpoint (`/api/posts`) frequently returns `content` as
-        null even though we request it — the full body (its formatting, inline
-        `<img>`, and external `<a href>` links) only comes back from the
-        single-post detail resource. The downloader calls this to enrich a post
-        whose feed body was empty before writing the importer sidecar.
+        Patreon deprecated the flat `content` HTML field — it returns null on the
+        feed AND the detail endpoint, for every post type. The real body now lives
+        in `content_json_string` (a ProseMirror doc), and is only returned under
+        the DEFAULT post fieldset: a sparse `fields[post]=content` request OMITS
+        it (confirmed against the live API 2026-06-15). So we request the default
+        fieldset (no `fields[post]`) and resolve the body via `post_body_html`
+        (content → else convert content_json_string). The downloader calls this to
+        enrich a post whose feed body was empty before writing the sidecar.
 
-        Best-effort BY DESIGN: the media download is the primary job, so a body
-        we can't fetch must never fail the walk. Every failure path (no id,
-        transport error, non-200, non-JSON, missing/empty content) returns None
-        rather than raising — distinct from the loud drift/auth raises on the
-        feed path, which gate real downloads.
+        Best-effort BY DESIGN: a body we can't fetch must never fail the walk.
+        Every failure path returns None rather than raising — distinct from the
+        loud drift/auth raises on the feed path, which gate real downloads.
         """
         if not post_id:
             return None
         if self._request_sleep > 0:
             time.sleep(self._request_sleep)  # pace the API endpoint (plan #703)
-        url = f"{_POSTS_URL}/{post_id}"
-        params = {"fields[post]": "content", "json-api-version": "1.0"}
+        # No `fields[post]` — the default fieldset is the only shape that returns
+        # content_json_string (the body). A sparse fieldset nulls it out.
         try:
-            resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS)
+            resp = self._session.get(f"{_POSTS_URL}/{post_id}", timeout=_TIMEOUT_SECONDS)
         except requests.RequestException as exc:
             log.warning("Patreon post-detail fetch failed (post %s): %s", post_id, exc)
             return None
@@ -608,43 +614,13 @@ class PatreonClient:
             return None
         data = payload.get("data") if isinstance(payload, dict) else None
         attrs = data.get("attributes") if isinstance(data, dict) else None
-        content = attrs.get("content") if isinstance(attrs, dict) else None
-        if isinstance(content, str) and content.strip():
-            log.info("post-detail: fetched %d chars (post %s)", len(content), post_id)
-            return content
-        # Sparse `fields[post]=content` came back null/empty. Patreon serves null
-        # under the sparse fieldset for some post types (polls, embeds/films,
-        # body-only announcements) even when the body plainly exists. Re-fetch the
-        # FULL post resource ONCE — only the empty cases pay this extra GET — which
-        # both DIAGNOSES (logs post_type) and RECOVERS the body when the sparse
-        # fieldset was the cause (#842).
-        try:
-            full = self._session.get(
-                url, params={"json-api-version": "1.0"}, timeout=_TIMEOUT_SECONDS,
-            )
-        except requests.RequestException as exc:
-            log.warning("post-detail full re-fetch failed (post %s): %s", post_id, exc)
-            return None
-        fa = None
-        if full.status_code == 200:
-            try:
-                fp = full.json()
-            except ValueError:
-                fp = None
-            fdata = fp.get("data") if isinstance(fp, dict) else None
-            fa = fdata.get("attributes") if isinstance(fdata, dict) else None
-        fcontent = fa.get("content") if isinstance(fa, dict) else None
-        ptype = fa.get("post_type") if isinstance(fa, dict) else None
-        if isinstance(fcontent, str) and fcontent.strip():
-            log.warning(
-                "post-detail: sparse fieldset gave null but FULL fetch has %d chars "
-                "(post %s, post_type=%s) — using full body",
-                len(fcontent), post_id, ptype,
-            )
-            return fcontent
+        body = post_body_html(attrs)
+        if body and body.strip():
+            log.info("post-detail: fetched %d chars (post %s)", len(body), post_id)
+            return body
+        ptype = attrs.get("post_type") if isinstance(attrs, dict) else None
         log.info(
-            "post-detail: empty/null content (post %s, post_type=%s) even on full "
-            "fetch — body not in the post resource", post_id, ptype,
+            "post-detail: no body (post %s, post_type=%s)", post_id, ptype,
         )
         return None
 

backend/app/services/patreon_downloader.py

@@ -41,6 +41,7 @@ from urllib.parse import urlsplit
 
 import requests
 
+from ..utils.prosemirror import post_body_html
 from .file_validator import is_validatable, quarantine_file, validate_file
 from .patreon_client import (
     _BACKOFF_CAP_SECONDS,
@@ -553,19 +554,22 @@ class PatreonDownloader:
         the per-media sidecar — a media-less post has no source file."""
         attrs = post.get("attributes") or {}
         title = attrs.get("title")
-        content = attrs.get("content")
-        # The feed/list endpoint frequently returns an empty `content`; the full
-        # HTML body (formatting + inline <img> + external <a href> links) only
-        # comes from the per-post detail endpoint. Enrich on first write for this
-        # post and MEMOIZE by mutating the shared `post` dict — so a multi-image
-        # post fetches detail exactly once, and a fully-seen post (no fresh
-        # download → no sidecar write) never pays the extra GET.
-        if (not isinstance(content, str) or not content.strip()) and self._content_fetcher:
+        # Resolve the body HTML from the feed attrs: legacy flat `content`, else
+        # convert the current `content_json_string` ProseMirror doc (#842).
+        content = post_body_html(attrs)
+        # The feed/list endpoint frequently returns an empty body; the full body
+        # only comes from the per-post detail endpoint. Enrich on first write for
+        # this post and MEMOIZE the RESOLVED HTML by mutating the shared `post`
+        # dict — so a multi-image post fetches detail at most once, the post-record
+        # body-length read reuses it, and a fully-seen post (no fresh download → no
+        # sidecar write) never pays the extra GET.
+        if (not content or not content.strip()) and self._content_fetcher:
             fetched = self._content_fetcher(str(post.get("id") or ""))
             if fetched:
-                attrs["content"] = fetched
-                post["attributes"] = attrs
                 content = fetched
+        if isinstance(content, str) and content.strip():
+            attrs["content"] = content
+            post["attributes"] = attrs
         published = attrs.get("published_at")
         url = attrs.get("url")
         data = {