feat(patreon): capture full post body via adaptive detail-fetch
CI / lint (push) Successful in 4s
CI / frontend-build (push) Successful in 21s
CI / backend-lint-and-test (push) Successful in 38s
CI / integration (push) Successful in 3m24s

The feed endpoint (/api/posts) returns `content` empty for many posts, so post
bodies — their formatting, inline <img>, and external <a href> links — were
never captured (the post showed "(no description)"). Enrich an empty feed body
from the per-post detail endpoint (/api/posts/{id}) before writing the importer
sidecar, memoized by mutating the shared post dict so a multi-image post fetches
detail exactly once and fully-seen posts (no fresh download) pay nothing.
Best-effort by design: a body we can't fetch returns None and never fails the
walk. No-doubling and no-clobber-of-populated-body already hold (post upsert is
keyed on external_post_id; an empty body parses to None and isn't applied).

First slice of milestone #64 (rich post capture + faithful rendering +
external-host downloads). Refs FC #830.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-14 12:29:06 -04:00
parent 7fcef53d5b
commit 2c67c27044
5 changed files with 161 additions and 1 deletions
+54
View File
@@ -9,6 +9,7 @@ import json
from pathlib import Path
import pytest
import requests
from backend.app.services import patreon_client as pc_mod
from backend.app.services.patreon_client import (
@@ -352,3 +353,56 @@ def test_request_sleep_paces_before_fetch(monkeypatch):
monkeypatch.setattr(pc_mod.time, "sleep", lambda s: slept.append(s))
client._fetch("5555", None)
assert slept == [1.5]
# -- fetch_post_detail_content (best-effort full-body enrichment) ----------
def test_fetch_post_detail_content_returns_body(monkeypatch):
payload = {"data": {"id": "42", "attributes": {"content": "<p>full body</p>"}}}
captured: dict = {}
client = PatreonClient(cookies_path=None)
def _get(url, params=None, timeout=None):
captured["url"] = url
captured["params"] = params
return _FakeResp(200, json_data=payload)
monkeypatch.setattr(client._session, "get", _get)
assert client.fetch_post_detail_content("42") == "<p>full body</p>"
assert captured["url"].endswith("/api/posts/42")
assert captured["params"]["fields[post]"] == "content"
def test_fetch_post_detail_content_empty_body_is_none(monkeypatch):
payload = {"data": {"attributes": {"content": " "}}}
client = _client_returning(monkeypatch, _FakeResp(200, json_data=payload))
assert client.fetch_post_detail_content("42") is None
def test_fetch_post_detail_content_blank_id_skips_request(monkeypatch):
calls: list = []
client = PatreonClient(cookies_path=None)
monkeypatch.setattr(
client._session, "get",
lambda *a, **k: calls.append(1) or _FakeResp(200, json_data={}),
)
assert client.fetch_post_detail_content("") is None
assert calls == []
def test_fetch_post_detail_content_swallows_http_and_transport_errors(monkeypatch):
# Non-200 → None (best-effort; a missing body must never fail the walk).
client = _client_returning(monkeypatch, _FakeResp(500))
assert client.fetch_post_detail_content("42") is None
# Non-JSON body (HTML challenge) → None, not a raise.
client = _client_returning(monkeypatch, _FakeResp(200, raise_json=True))
assert client.fetch_post_detail_content("42") is None
# Transport error → None.
client = PatreonClient(cookies_path=None)
def _boom(*a, **k):
raise requests.ConnectionError("boom")
monkeypatch.setattr(client._session, "get", _boom)
assert client.fetch_post_detail_content("42") is None
+41
View File
@@ -7,6 +7,7 @@ via monkeypatching `_run_ytdlp`. PURE module → no DB, so unmarked (fast lane).
from __future__ import annotations
import json
from pathlib import Path
import pytest
@@ -536,3 +537,43 @@ def test_range_ignored_by_server_refetches_clean(tmp_path, monkeypatch):
outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
assert outcomes[0].status == "downloaded"
assert outcomes[0].path.read_bytes() == full # clean, not full[:400] + full
# -- content enrichment (adaptive detail-fetch of an empty feed body) -------
def test_enriches_empty_content_via_fetcher(tmp_path):
"""An empty feed body is backfilled from the detail-fetch seam — once per
post (memoized on the shared dict) — and lands in the importer sidecar."""
calls: list[str] = []
def _fetcher(post_id: str) -> str:
calls.append(post_id)
return "<p>full body</p>"
dl = PatreonDownloader(
images_root=tmp_path, cookies_path=None, validate=False,
session=_FakeSession(), content_fetcher=_fetcher,
)
post = _post()
post["attributes"]["content"] = "" # feed returned an empty body
dl.download_post(post, [_img("a.png"), _img("b.png")], "artist-x")
post_dir = tmp_path / "artist-x" / "patreon" / pd_mod._post_dir_name(post)
sc = find_sidecar(post_dir / "01_a.png")
assert sc is not None
assert json.loads(sc.read_text())["content"] == "<p>full body</p>"
# Fetched ONCE for the post, not once per media item.
assert calls == ["1001"]
def test_no_enrichment_when_feed_body_present(tmp_path):
"""A non-empty feed body is trusted as-is; the detail endpoint isn't hit."""
calls: list[str] = []
dl = PatreonDownloader(
images_root=tmp_path, cookies_path=None, validate=False,
session=_FakeSession(),
content_fetcher=lambda pid: calls.append(pid) or "x",
)
dl.download_post(_post(), [_img("a.png")], "artist-x") # _post body is non-empty
assert calls == []