976107bbe8
THE empty-body root cause. Patreon deprecated the flat `content` HTML field — it returns null on the feed AND the detail endpoint, for every post type (confirmed against the live API: all 135 StickySpoodge posts, text_only/ image_file/poll alike). The real body now lives in `content_json_string` (a ProseMirror/TipTap doc), returned only under the DEFAULT post fieldset — a sparse fields[post]=content request omits it. Not credential, not post_type: a request shape gone stale. - NEW utils/prosemirror.py: ProseMirror doc -> HTML (paragraphs, marks bold/italic/underline/strike/code/link, hardBreak, inline images, lists, headings; unknown nodes degrade to children). post_body_html(attrs) = the one resolver: legacy content HTML else convert content_json_string. - patreon_client: add content_json_string to the feed _FIELDS_POST; rewrite fetch_post_detail_content to use the DEFAULT fieldset (no sparse fields[post]) and resolve via post_body_html (replaces the wrong sparse req + full-fetch fallback). - patreon_downloader._write_sidecar_data: resolve body via post_body_html (feed content_json_string) before the detail-fetch; memoize resolved HTML. - tests: prosemirror converter unit tests; client legacy + content_json_string paths; contract pins content_json_string. Inline <img> nodes carry the CDN filehash → bodies now feed Phase-2 localization. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
130 lines
4.3 KiB
Python
130 lines
4.3 KiB
Python
"""Render Patreon's post body to HTML.
|
|
|
|
Patreon deprecated the flat `content` HTML field (it returns `null` now, on the
|
|
feed AND the detail endpoint, for every post type) and moved the real body to
|
|
`content_json_string` — a ProseMirror/TipTap document (a JSON string of typed
|
|
nodes + marks), returned only under the DEFAULT post fieldset. This module
|
|
converts that doc to the HTML the rest of the pipeline already expects (it then
|
|
flows through the existing nh3 sanitizer + inline-image localization). #842.
|
|
|
|
Built from the observed Patreon doc shape: doc → [paragraph|heading|image|list…];
|
|
paragraph content → [text(+marks: bold/italic/underline/strike/code/link),
|
|
hardBreak, image]. Unknown node types degrade to rendering their children so we
|
|
never silently drop text.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from html import escape
|
|
|
|
# ProseMirror text-mark type → wrapping HTML tag (link handled separately — it
|
|
# needs an href attribute).
|
|
_MARK_TAGS = {
|
|
"bold": "strong",
|
|
"strong": "strong",
|
|
"italic": "em",
|
|
"em": "em",
|
|
"underline": "u",
|
|
"strike": "s",
|
|
"strikethrough": "s",
|
|
"code": "code",
|
|
}
|
|
|
|
|
|
def post_body_html(attrs: dict | None) -> str | None:
|
|
"""Resolve a Patreon post's body HTML from its `attributes`.
|
|
|
|
Prefers the legacy flat `content` HTML (older posts still carry it); falls
|
|
back to converting the `content_json_string` ProseMirror doc (current posts).
|
|
Returns None when neither holds a body. The ONE place body-source precedence
|
|
lives, so the feed-parse path and the detail-fetch path can't diverge.
|
|
"""
|
|
if not isinstance(attrs, dict):
|
|
return None
|
|
content = attrs.get("content")
|
|
if isinstance(content, str) and content.strip():
|
|
return content
|
|
return prosemirror_to_html(attrs.get("content_json_string"))
|
|
|
|
|
|
def prosemirror_to_html(doc_json: str | None) -> str | None:
|
|
"""Render a ProseMirror/TipTap doc (a JSON string) to HTML, or None for an
|
|
empty / blank / unparseable doc."""
|
|
if not isinstance(doc_json, str) or not doc_json.strip():
|
|
return None
|
|
try:
|
|
doc = json.loads(doc_json)
|
|
except (ValueError, TypeError):
|
|
return None
|
|
if not isinstance(doc, dict):
|
|
return None
|
|
return _render_nodes(doc.get("content")) or None
|
|
|
|
|
|
def _render_nodes(nodes: object) -> str:
|
|
if not isinstance(nodes, list):
|
|
return ""
|
|
return "".join(_render_node(n) for n in nodes if isinstance(n, dict))
|
|
|
|
|
|
def _render_node(node: dict) -> str:
|
|
ntype = node.get("type")
|
|
if ntype == "text":
|
|
return _render_text(node)
|
|
if ntype == "hardBreak":
|
|
return "<br>"
|
|
if ntype == "image":
|
|
return _render_image(node)
|
|
if ntype == "horizontalRule":
|
|
return "<hr>"
|
|
|
|
inner = _render_nodes(node.get("content"))
|
|
if ntype == "paragraph":
|
|
return f"<p>{inner}</p>"
|
|
if ntype == "heading":
|
|
level = (node.get("attrs") or {}).get("level", 1)
|
|
if not isinstance(level, int) or not 1 <= level <= 6:
|
|
level = 1
|
|
return f"<h{level}>{inner}</h{level}>"
|
|
if ntype == "blockquote":
|
|
return f"<blockquote>{inner}</blockquote>"
|
|
if ntype == "bulletList":
|
|
return f"<ul>{inner}</ul>"
|
|
if ntype == "orderedList":
|
|
return f"<ol>{inner}</ol>"
|
|
if ntype == "listItem":
|
|
return f"<li>{inner}</li>"
|
|
if ntype == "codeBlock":
|
|
return f"<pre><code>{inner}</code></pre>"
|
|
# Unknown / unhandled container — emit children so text is never dropped.
|
|
return inner
|
|
|
|
|
|
def _render_text(node: dict) -> str:
|
|
text = node.get("text")
|
|
if not isinstance(text, str) or not text:
|
|
return ""
|
|
out = escape(text)
|
|
for mark in node.get("marks") or []:
|
|
if not isinstance(mark, dict):
|
|
continue
|
|
mtype = mark.get("type")
|
|
if mtype == "link":
|
|
href = (mark.get("attrs") or {}).get("href") or ""
|
|
out = f'<a href="{escape(str(href), quote=True)}">{out}</a>'
|
|
else:
|
|
tag = _MARK_TAGS.get(mtype)
|
|
if tag:
|
|
out = f"<{tag}>{out}</{tag}>"
|
|
return out
|
|
|
|
|
|
def _render_image(node: dict) -> str:
|
|
attrs = node.get("attrs") or {}
|
|
src = attrs.get("src")
|
|
if not isinstance(src, str) or not src:
|
|
return ""
|
|
alt = attrs.get("alt") or ""
|
|
return f'<img src="{escape(src, quote=True)}" alt="{escape(str(alt), quote=True)}">'
|