Merge dev → main: #830 rich post capture + external-host downloads (+ #768/#789/#739) #102
@@ -562,6 +562,49 @@ class PatreonClient:
|
|||||||
return
|
return
|
||||||
current_cursor = next_cursor
|
current_cursor = next_cursor
|
||||||
|
|
||||||
|
# -- detail (full body enrichment) -------------------------------------
|
||||||
|
|
||||||
|
def fetch_post_detail_content(self, post_id: str) -> str | None:
|
||||||
|
"""Best-effort fetch of a post's full HTML `content` from the per-post
|
||||||
|
DETAIL endpoint (`/api/posts/{id}`).
|
||||||
|
|
||||||
|
The feed/list endpoint (`/api/posts`) frequently returns `content` as
|
||||||
|
null even though we request it — the full body (its formatting, inline
|
||||||
|
`<img>`, and external `<a href>` links) only comes back from the
|
||||||
|
single-post detail resource. The downloader calls this to enrich a post
|
||||||
|
whose feed body was empty before writing the importer sidecar.
|
||||||
|
|
||||||
|
Best-effort BY DESIGN: the media download is the primary job, so a body
|
||||||
|
we can't fetch must never fail the walk. Every failure path (no id,
|
||||||
|
transport error, non-200, non-JSON, missing/empty content) returns None
|
||||||
|
rather than raising — distinct from the loud drift/auth raises on the
|
||||||
|
feed path, which gate real downloads.
|
||||||
|
"""
|
||||||
|
if not post_id:
|
||||||
|
return None
|
||||||
|
if self._request_sleep > 0:
|
||||||
|
time.sleep(self._request_sleep) # pace the API endpoint (plan #703)
|
||||||
|
url = f"{_POSTS_URL}/{post_id}"
|
||||||
|
params = {"fields[post]": "content", "json-api-version": "1.0"}
|
||||||
|
try:
|
||||||
|
resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS)
|
||||||
|
except requests.RequestException as exc:
|
||||||
|
log.warning("Patreon post-detail fetch failed (post %s): %s", post_id, exc)
|
||||||
|
return None
|
||||||
|
if resp.status_code != 200:
|
||||||
|
log.warning(
|
||||||
|
"Patreon post-detail fetch HTTP %s (post %s)", resp.status_code, post_id
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
payload = resp.json()
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
data = payload.get("data") if isinstance(payload, dict) else None
|
||||||
|
attrs = data.get("attributes") if isinstance(data, dict) else None
|
||||||
|
content = attrs.get("content") if isinstance(attrs, dict) else None
|
||||||
|
return content if isinstance(content, str) and content.strip() else None
|
||||||
|
|
||||||
# -- verify ------------------------------------------------------------
|
# -- verify ------------------------------------------------------------
|
||||||
|
|
||||||
def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
|
def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
|
||||||
|
|||||||
@@ -168,10 +168,17 @@ class PatreonDownloader:
|
|||||||
validate: bool = True,
|
validate: bool = True,
|
||||||
rate_limit: float = 0.0,
|
rate_limit: float = 0.0,
|
||||||
session: requests.Session | None = None,
|
session: requests.Session | None = None,
|
||||||
|
content_fetcher: Callable[[str], str | None] | None = None,
|
||||||
):
|
):
|
||||||
self.images_root = Path(images_root)
|
self.images_root = Path(images_root)
|
||||||
self.cookies_path = str(cookies_path) if cookies_path else None
|
self.cookies_path = str(cookies_path) if cookies_path else None
|
||||||
self._validate = validate
|
self._validate = validate
|
||||||
|
# Best-effort enrichment seam: (post_id) -> full HTML body, or None. The
|
||||||
|
# feed endpoint often omits `content`; the adapter wires this to
|
||||||
|
# PatreonClient.fetch_post_detail_content so the sidecar captures the
|
||||||
|
# real body (formatting + inline <img> + external <a href> links).
|
||||||
|
# None in unit tests / when enrichment isn't wanted.
|
||||||
|
self._content_fetcher = content_fetcher
|
||||||
# Politeness: seconds to sleep before each actual media download (paces
|
# Politeness: seconds to sleep before each actual media download (paces
|
||||||
# the CDN; honors ImportSettings.download_rate_limit_seconds, the same
|
# the CDN; honors ImportSettings.download_rate_limit_seconds, the same
|
||||||
# value gallery-dl used as its between-downloads `sleep`). 0 = no pacing.
|
# value gallery-dl used as its between-downloads `sleep`). 0 = no pacing.
|
||||||
@@ -495,6 +502,18 @@ class PatreonDownloader:
|
|||||||
attrs = post.get("attributes") or {}
|
attrs = post.get("attributes") or {}
|
||||||
title = attrs.get("title")
|
title = attrs.get("title")
|
||||||
content = attrs.get("content")
|
content = attrs.get("content")
|
||||||
|
# The feed/list endpoint frequently returns an empty `content`; the full
|
||||||
|
# HTML body (formatting + inline <img> + external <a href> links) only
|
||||||
|
# comes from the per-post detail endpoint. Enrich on first write for this
|
||||||
|
# post and MEMOIZE by mutating the shared `post` dict — so a multi-image
|
||||||
|
# post fetches detail exactly once, and a fully-seen post (no fresh
|
||||||
|
# download → no sidecar write) never pays the extra GET.
|
||||||
|
if (not isinstance(content, str) or not content.strip()) and self._content_fetcher:
|
||||||
|
fetched = self._content_fetcher(str(post.get("id") or ""))
|
||||||
|
if fetched:
|
||||||
|
attrs["content"] = fetched
|
||||||
|
post["attributes"] = attrs
|
||||||
|
content = fetched
|
||||||
published = attrs.get("published_at")
|
published = attrs.get("published_at")
|
||||||
url = attrs.get("url")
|
url = attrs.get("url")
|
||||||
data = {
|
data = {
|
||||||
|
|||||||
@@ -112,7 +112,10 @@ class PatreonIngester(Ingester):
|
|||||||
downloader
|
downloader
|
||||||
if downloader is not None
|
if downloader is not None
|
||||||
else PatreonDownloader(
|
else PatreonDownloader(
|
||||||
self.images_root, cookies_path, validate=validate, rate_limit=rate_limit
|
self.images_root, cookies_path, validate=validate, rate_limit=rate_limit,
|
||||||
|
# Enrich empty feed bodies from the per-post detail endpoint, via
|
||||||
|
# the SAME client (shares its cookie session + request pacing).
|
||||||
|
content_fetcher=resolved_client.fetch_post_detail_content,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
super().__init__(
|
super().__init__(
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import json
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
import requests
|
||||||
|
|
||||||
from backend.app.services import patreon_client as pc_mod
|
from backend.app.services import patreon_client as pc_mod
|
||||||
from backend.app.services.patreon_client import (
|
from backend.app.services.patreon_client import (
|
||||||
@@ -352,3 +353,56 @@ def test_request_sleep_paces_before_fetch(monkeypatch):
|
|||||||
monkeypatch.setattr(pc_mod.time, "sleep", lambda s: slept.append(s))
|
monkeypatch.setattr(pc_mod.time, "sleep", lambda s: slept.append(s))
|
||||||
client._fetch("5555", None)
|
client._fetch("5555", None)
|
||||||
assert slept == [1.5]
|
assert slept == [1.5]
|
||||||
|
|
||||||
|
|
||||||
|
# -- fetch_post_detail_content (best-effort full-body enrichment) ----------
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_post_detail_content_returns_body(monkeypatch):
|
||||||
|
payload = {"data": {"id": "42", "attributes": {"content": "<p>full body</p>"}}}
|
||||||
|
captured: dict = {}
|
||||||
|
client = PatreonClient(cookies_path=None)
|
||||||
|
|
||||||
|
def _get(url, params=None, timeout=None):
|
||||||
|
captured["url"] = url
|
||||||
|
captured["params"] = params
|
||||||
|
return _FakeResp(200, json_data=payload)
|
||||||
|
|
||||||
|
monkeypatch.setattr(client._session, "get", _get)
|
||||||
|
assert client.fetch_post_detail_content("42") == "<p>full body</p>"
|
||||||
|
assert captured["url"].endswith("/api/posts/42")
|
||||||
|
assert captured["params"]["fields[post]"] == "content"
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_post_detail_content_empty_body_is_none(monkeypatch):
|
||||||
|
payload = {"data": {"attributes": {"content": " "}}}
|
||||||
|
client = _client_returning(monkeypatch, _FakeResp(200, json_data=payload))
|
||||||
|
assert client.fetch_post_detail_content("42") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_post_detail_content_blank_id_skips_request(monkeypatch):
|
||||||
|
calls: list = []
|
||||||
|
client = PatreonClient(cookies_path=None)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
client._session, "get",
|
||||||
|
lambda *a, **k: calls.append(1) or _FakeResp(200, json_data={}),
|
||||||
|
)
|
||||||
|
assert client.fetch_post_detail_content("") is None
|
||||||
|
assert calls == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_post_detail_content_swallows_http_and_transport_errors(monkeypatch):
|
||||||
|
# Non-200 → None (best-effort; a missing body must never fail the walk).
|
||||||
|
client = _client_returning(monkeypatch, _FakeResp(500))
|
||||||
|
assert client.fetch_post_detail_content("42") is None
|
||||||
|
# Non-JSON body (HTML challenge) → None, not a raise.
|
||||||
|
client = _client_returning(monkeypatch, _FakeResp(200, raise_json=True))
|
||||||
|
assert client.fetch_post_detail_content("42") is None
|
||||||
|
# Transport error → None.
|
||||||
|
client = PatreonClient(cookies_path=None)
|
||||||
|
|
||||||
|
def _boom(*a, **k):
|
||||||
|
raise requests.ConnectionError("boom")
|
||||||
|
|
||||||
|
monkeypatch.setattr(client._session, "get", _boom)
|
||||||
|
assert client.fetch_post_detail_content("42") is None
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ via monkeypatching `_run_ytdlp`. PURE module → no DB, so unmarked (fast lane).
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -536,3 +537,43 @@ def test_range_ignored_by_server_refetches_clean(tmp_path, monkeypatch):
|
|||||||
outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
|
outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
|
||||||
assert outcomes[0].status == "downloaded"
|
assert outcomes[0].status == "downloaded"
|
||||||
assert outcomes[0].path.read_bytes() == full # clean, not full[:400] + full
|
assert outcomes[0].path.read_bytes() == full # clean, not full[:400] + full
|
||||||
|
|
||||||
|
|
||||||
|
# -- content enrichment (adaptive detail-fetch of an empty feed body) -------
|
||||||
|
|
||||||
|
|
||||||
|
def test_enriches_empty_content_via_fetcher(tmp_path):
|
||||||
|
"""An empty feed body is backfilled from the detail-fetch seam — once per
|
||||||
|
post (memoized on the shared dict) — and lands in the importer sidecar."""
|
||||||
|
calls: list[str] = []
|
||||||
|
|
||||||
|
def _fetcher(post_id: str) -> str:
|
||||||
|
calls.append(post_id)
|
||||||
|
return "<p>full body</p>"
|
||||||
|
|
||||||
|
dl = PatreonDownloader(
|
||||||
|
images_root=tmp_path, cookies_path=None, validate=False,
|
||||||
|
session=_FakeSession(), content_fetcher=_fetcher,
|
||||||
|
)
|
||||||
|
post = _post()
|
||||||
|
post["attributes"]["content"] = "" # feed returned an empty body
|
||||||
|
dl.download_post(post, [_img("a.png"), _img("b.png")], "artist-x")
|
||||||
|
|
||||||
|
post_dir = tmp_path / "artist-x" / "patreon" / pd_mod._post_dir_name(post)
|
||||||
|
sc = find_sidecar(post_dir / "01_a.png")
|
||||||
|
assert sc is not None
|
||||||
|
assert json.loads(sc.read_text())["content"] == "<p>full body</p>"
|
||||||
|
# Fetched ONCE for the post, not once per media item.
|
||||||
|
assert calls == ["1001"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_enrichment_when_feed_body_present(tmp_path):
|
||||||
|
"""A non-empty feed body is trusted as-is; the detail endpoint isn't hit."""
|
||||||
|
calls: list[str] = []
|
||||||
|
dl = PatreonDownloader(
|
||||||
|
images_root=tmp_path, cookies_path=None, validate=False,
|
||||||
|
session=_FakeSession(),
|
||||||
|
content_fetcher=lambda pid: calls.append(pid) or "x",
|
||||||
|
)
|
||||||
|
dl.download_post(_post(), [_img("a.png")], "artist-x") # _post body is non-empty
|
||||||
|
assert calls == []
|
||||||
|
|||||||
Reference in New Issue
Block a user