fix(patreon): read post body from content_json_string (ProseMirror), not the dead content field (#842)

THE empty-body root cause. Patreon deprecated the flat `content` HTML field —
it returns null on the feed AND the detail endpoint, for every post type
(confirmed against the live API: all 135 StickySpoodge posts, text_only/
image_file/poll alike). The real body now lives in `content_json_string` (a
ProseMirror/TipTap doc), returned only under the DEFAULT post fieldset — a sparse
fields[post]=content request omits it. Not credential, not post_type: a request
shape gone stale.

- NEW utils/prosemirror.py: ProseMirror doc -> HTML (paragraphs, marks
  bold/italic/underline/strike/code/link, hardBreak, inline images, lists,
  headings; unknown nodes degrade to children). post_body_html(attrs) = the one
  resolver: legacy content HTML else convert content_json_string.
- patreon_client: add content_json_string to the feed _FIELDS_POST; rewrite
  fetch_post_detail_content to use the DEFAULT fieldset (no sparse fields[post])
  and resolve via post_body_html (replaces the wrong sparse req + full-fetch
  fallback).
- patreon_downloader._write_sidecar_data: resolve body via post_body_html
  (feed content_json_string) before the detail-fetch; memoize resolved HTML.
- tests: prosemirror converter unit tests; client legacy + content_json_string
  paths; contract pins content_json_string.

Inline <img> nodes carry the CDN filehash → bodies now feed Phase-2 localization.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/test_patreon_client.py

@@ -358,7 +358,10 @@ def test_request_sleep_paces_before_fetch(monkeypatch):
 # -- fetch_post_detail_content (best-effort full-body enrichment) ----------
 
 
-def test_fetch_post_detail_content_returns_body(monkeypatch):
+def test_fetch_post_detail_content_returns_legacy_content(monkeypatch):
+    """Old posts still carry the flat `content` HTML — returned as-is. The detail
+    fetch uses the DEFAULT fieldset (no sparse `fields[post]`, which nulls the
+    body) — #842."""
     payload = {"data": {"id": "42", "attributes": {"content": "<p>full body</p>"}}}
     captured: dict = {}
     client = PatreonClient(cookies_path=None)
@@ -371,37 +374,28 @@ def test_fetch_post_detail_content_returns_body(monkeypatch):
     monkeypatch.setattr(client._session, "get", _get)
     assert client.fetch_post_detail_content("42") == "<p>full body</p>"
     assert captured["url"].endswith("/api/posts/42")
-    assert captured["params"]["fields[post]"] == "content"
+    # No sparse fieldset — that's what nulled the body on the live API.
+    assert not captured["params"] or "fields[post]" not in captured["params"]
+
+
+def test_fetch_post_detail_content_converts_content_json_string(monkeypatch):
+    """Current posts: `content` is null and the body lives in the
+    `content_json_string` ProseMirror doc → converted to HTML (#842)."""
+    doc = (
+        '{"type":"doc","content":[{"type":"paragraph","content":'
+        '[{"type":"text","text":"Hello"}]}]}'
+    )
+    payload = {"data": {"attributes": {"content": None, "content_json_string": doc}}}
+    client = _client_returning(monkeypatch, _FakeResp(200, json_data=payload))
+    assert client.fetch_post_detail_content("42") == "<p>Hello</p>"
 
 
 def test_fetch_post_detail_content_empty_body_is_none(monkeypatch):
-    payload = {"data": {"attributes": {"content": "   "}}}
+    payload = {"data": {"attributes": {"content": "   ", "content_json_string": ""}}}
     client = _client_returning(monkeypatch, _FakeResp(200, json_data=payload))
     assert client.fetch_post_detail_content("42") is None
 
 
-def test_fetch_post_detail_content_falls_back_to_full_fetch(monkeypatch):
-    """#842: when the sparse fields[post]=content request returns null (Patreon
-    does this for polls / embeds / body-only posts), a one-time FULL re-fetch
-    recovers the body — so inline-only / no-media posts still get their text."""
-    client = PatreonClient(cookies_path=None)
-    seen_params: list = []
-
-    def _get(url, params=None, timeout=None):
-        seen_params.append(params or {})
-        if "fields[post]" in (params or {}):
-            return _FakeResp(200, json_data={"data": {"attributes": {"content": None}}})
-        return _FakeResp(200, json_data={
-            "data": {"attributes": {"content": "<p>body via full</p>", "post_type": "poll"}}
-        })
-
-    monkeypatch.setattr(client._session, "get", _get)
-    assert client.fetch_post_detail_content("42") == "<p>body via full</p>"
-    # Sparse request first, then a full re-fetch without the sparse fieldset.
-    assert "fields[post]" in seen_params[0]
-    assert "fields[post]" not in seen_params[1]
-
-
 def test_fetch_post_detail_content_blank_id_skips_request(monkeypatch):
     calls: list = []
     client = PatreonClient(cookies_path=None)

tests/test_patreon_contract.py

@@ -66,7 +66,9 @@ def test_request_field_set_carries_parser_dependencies():
         assert rel in includes, f"missing include relationship {rel}"
 
     post_fields = params["fields[post]"]
-    for field in ("content", "post_file", "image"):
+    # content_json_string is the CURRENT body field (Patreon nulled flat `content`,
+    # #842); content kept for legacy posts. Both must stay requested.
+    for field in ("content", "content_json_string", "post_file", "image"):
         assert field in post_fields, f"missing post field {field}"
 
     assert params["filter[campaign_id]"] == "5555"

tests/test_prosemirror.py

@@ -0,0 +1,108 @@
+"""Unit tests for the Patreon ProseMirror→HTML body converter (#842).
+
+Patreon moved the post body from the flat `content` HTML field (now null) to
+`content_json_string` (a ProseMirror doc). PURE module → no DB, fast lane.
+"""
+
+from __future__ import annotations
+
+import json
+
+from backend.app.utils.prosemirror import post_body_html, prosemirror_to_html
+
+
+def _doc(*nodes) -> str:
+    return json.dumps({"type": "doc", "content": list(nodes)})
+
+
+def test_paragraph_text_and_hardbreak():
+    doc = _doc({
+        "type": "paragraph",
+        "content": [
+            {"type": "text", "text": "Hi there"},
+            {"type": "hardBreak"},
+            {"type": "text", "text": "line two"},
+        ],
+    })
+    assert prosemirror_to_html(doc) == "<p>Hi there<br>line two</p>"
+
+
+def test_marks_bold_italic_link():
+    doc = _doc({
+        "type": "paragraph",
+        "content": [
+            {"type": "text", "marks": [{"type": "bold"}], "text": "B"},
+            {"type": "text", "marks": [{"type": "italic"}], "text": "I"},
+            {
+                "type": "text",
+                "marks": [{"type": "link", "attrs": {"href": "https://x.test/a"}}],
+                "text": "L",
+            },
+        ],
+    })
+    assert prosemirror_to_html(doc) == (
+        '<p><strong>B</strong><em>I</em><a href="https://x.test/a">L</a></p>'
+    )
+
+
+def test_image_node_emits_img_with_cdn_src():
+    doc = _doc({
+        "type": "image",
+        "attrs": {"src": "https://cdn.test/p/post/1/abc/1.gif?token-hash=z", "alt": None},
+    })
+    out = prosemirror_to_html(doc)
+    assert out == '<img src="https://cdn.test/p/post/1/abc/1.gif?token-hash=z" alt="">'
+
+
+def test_lists_and_heading():
+    doc = _doc(
+        {"type": "heading", "attrs": {"level": 2},
+         "content": [{"type": "text", "text": "Status"}]},
+        {"type": "bulletList", "content": [
+            {"type": "listItem", "content": [
+                {"type": "paragraph", "content": [{"type": "text", "text": "one"}]}]},
+        ]},
+    )
+    assert prosemirror_to_html(doc) == (
+        "<h2>Status</h2><ul><li><p>one</p></li></ul>"
+    )
+
+
+def test_text_is_html_escaped():
+    doc = _doc({"type": "paragraph",
+                "content": [{"type": "text", "text": "a < b & c > d"}]})
+    assert prosemirror_to_html(doc) == "<p>a &lt; b &amp; c &gt; d</p>"
+
+
+def test_unknown_node_renders_children_not_dropped():
+    doc = _doc({"type": "weirdBlock", "content": [
+        {"type": "paragraph", "content": [{"type": "text", "text": "keep me"}]}]})
+    assert prosemirror_to_html(doc) == "<p>keep me</p>"
+
+
+def test_empty_and_unparseable_return_none():
+    assert prosemirror_to_html(None) is None
+    assert prosemirror_to_html("") is None
+    assert prosemirror_to_html("   ") is None
+    assert prosemirror_to_html("{not json") is None
+    assert prosemirror_to_html(json.dumps({"type": "doc", "content": []})) is None
+
+
+def test_post_body_html_prefers_legacy_content():
+    attrs = {"content": "<p>legacy</p>",
+             "content_json_string": _doc({"type": "paragraph",
+                                          "content": [{"type": "text", "text": "new"}]})}
+    assert post_body_html(attrs) == "<p>legacy</p>"
+
+
+def test_post_body_html_falls_back_to_json_string():
+    attrs = {"content": None,
+             "content_json_string": _doc({"type": "paragraph",
+                                          "content": [{"type": "text", "text": "new"}]})}
+    assert post_body_html(attrs) == "<p>new</p>"
+
+
+def test_post_body_html_none_when_both_empty():
+    assert post_body_html({"content": None, "content_json_string": None}) is None
+    assert post_body_html({}) is None
+    assert post_body_html(None) is None