fix(patreon): read post body from content_json_string (ProseMirror), not the dead content field (#842)
THE empty-body root cause. Patreon deprecated the flat `content` HTML field — it returns null on the feed AND the detail endpoint, for every post type (confirmed against the live API: all 135 StickySpoodge posts, text_only/ image_file/poll alike). The real body now lives in `content_json_string` (a ProseMirror/TipTap doc), returned only under the DEFAULT post fieldset — a sparse fields[post]=content request omits it. Not credential, not post_type: a request shape gone stale. - NEW utils/prosemirror.py: ProseMirror doc -> HTML (paragraphs, marks bold/italic/underline/strike/code/link, hardBreak, inline images, lists, headings; unknown nodes degrade to children). post_body_html(attrs) = the one resolver: legacy content HTML else convert content_json_string. - patreon_client: add content_json_string to the feed _FIELDS_POST; rewrite fetch_post_detail_content to use the DEFAULT fieldset (no sparse fields[post]) and resolve via post_body_html (replaces the wrong sparse req + full-fetch fallback). - patreon_downloader._write_sidecar_data: resolve body via post_body_html (feed content_json_string) before the detail-fetch; memoize resolved HTML. - tests: prosemirror converter unit tests; client legacy + content_json_string paths; contract pins content_json_string. Inline <img> nodes carry the CDN filehash → bodies now feed Phase-2 localization. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -358,7 +358,10 @@ def test_request_sleep_paces_before_fetch(monkeypatch):
|
||||
# -- fetch_post_detail_content (best-effort full-body enrichment) ----------
|
||||
|
||||
|
||||
def test_fetch_post_detail_content_returns_body(monkeypatch):
|
||||
def test_fetch_post_detail_content_returns_legacy_content(monkeypatch):
|
||||
"""Old posts still carry the flat `content` HTML — returned as-is. The detail
|
||||
fetch uses the DEFAULT fieldset (no sparse `fields[post]`, which nulls the
|
||||
body) — #842."""
|
||||
payload = {"data": {"id": "42", "attributes": {"content": "<p>full body</p>"}}}
|
||||
captured: dict = {}
|
||||
client = PatreonClient(cookies_path=None)
|
||||
@@ -371,37 +374,28 @@ def test_fetch_post_detail_content_returns_body(monkeypatch):
|
||||
monkeypatch.setattr(client._session, "get", _get)
|
||||
assert client.fetch_post_detail_content("42") == "<p>full body</p>"
|
||||
assert captured["url"].endswith("/api/posts/42")
|
||||
assert captured["params"]["fields[post]"] == "content"
|
||||
# No sparse fieldset — that's what nulled the body on the live API.
|
||||
assert not captured["params"] or "fields[post]" not in captured["params"]
|
||||
|
||||
|
||||
def test_fetch_post_detail_content_converts_content_json_string(monkeypatch):
|
||||
"""Current posts: `content` is null and the body lives in the
|
||||
`content_json_string` ProseMirror doc → converted to HTML (#842)."""
|
||||
doc = (
|
||||
'{"type":"doc","content":[{"type":"paragraph","content":'
|
||||
'[{"type":"text","text":"Hello"}]}]}'
|
||||
)
|
||||
payload = {"data": {"attributes": {"content": None, "content_json_string": doc}}}
|
||||
client = _client_returning(monkeypatch, _FakeResp(200, json_data=payload))
|
||||
assert client.fetch_post_detail_content("42") == "<p>Hello</p>"
|
||||
|
||||
|
||||
def test_fetch_post_detail_content_empty_body_is_none(monkeypatch):
|
||||
payload = {"data": {"attributes": {"content": " "}}}
|
||||
payload = {"data": {"attributes": {"content": " ", "content_json_string": ""}}}
|
||||
client = _client_returning(monkeypatch, _FakeResp(200, json_data=payload))
|
||||
assert client.fetch_post_detail_content("42") is None
|
||||
|
||||
|
||||
def test_fetch_post_detail_content_falls_back_to_full_fetch(monkeypatch):
|
||||
"""#842: when the sparse fields[post]=content request returns null (Patreon
|
||||
does this for polls / embeds / body-only posts), a one-time FULL re-fetch
|
||||
recovers the body — so inline-only / no-media posts still get their text."""
|
||||
client = PatreonClient(cookies_path=None)
|
||||
seen_params: list = []
|
||||
|
||||
def _get(url, params=None, timeout=None):
|
||||
seen_params.append(params or {})
|
||||
if "fields[post]" in (params or {}):
|
||||
return _FakeResp(200, json_data={"data": {"attributes": {"content": None}}})
|
||||
return _FakeResp(200, json_data={
|
||||
"data": {"attributes": {"content": "<p>body via full</p>", "post_type": "poll"}}
|
||||
})
|
||||
|
||||
monkeypatch.setattr(client._session, "get", _get)
|
||||
assert client.fetch_post_detail_content("42") == "<p>body via full</p>"
|
||||
# Sparse request first, then a full re-fetch without the sparse fieldset.
|
||||
assert "fields[post]" in seen_params[0]
|
||||
assert "fields[post]" not in seen_params[1]
|
||||
|
||||
|
||||
def test_fetch_post_detail_content_blank_id_skips_request(monkeypatch):
|
||||
calls: list = []
|
||||
client = PatreonClient(cookies_path=None)
|
||||
|
||||
Reference in New Issue
Block a user