feat(patreon): native JSON-API client — ingester build step 1 (plan #697)
PatreonClient: cookie-auth requests session, /api/posts cursor pagination, JSON:API included flattening, per-post media extraction (images/image_large/ attachments/postfile/content) with filehash dedup, loud drift detection. Zero per-file HEADs — every media URL+file_name comes from the API. Not yet wired into download_service (later step). Pure-parsing unit tests + fixture. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+121
@@ -0,0 +1,121 @@
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "1001",
|
||||
"type": "post",
|
||||
"attributes": {
|
||||
"title": "Image gallery post",
|
||||
"published_at": "2026-05-01T12:00:00.000+00:00",
|
||||
"post_type": "image_file",
|
||||
"content": "<p>just a gallery</p>",
|
||||
"url": "https://www.patreon.com/posts/1001",
|
||||
"image": {
|
||||
"large_url": "https://c10.patreonusercontent.com/4/large/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg",
|
||||
"url": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg"
|
||||
}
|
||||
},
|
||||
"relationships": {
|
||||
"images": {
|
||||
"data": [
|
||||
{"id": "9001", "type": "media"},
|
||||
{"id": "9002", "type": "media"}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "1002",
|
||||
"type": "post",
|
||||
"attributes": {
|
||||
"title": "Attachment post",
|
||||
"published_at": "2026-04-15T09:30:00.000+00:00",
|
||||
"post_type": "text_only",
|
||||
"content": "<p>here is a zip</p>",
|
||||
"url": "https://www.patreon.com/posts/1002",
|
||||
"post_file": {
|
||||
"name": "bonus-pack.zip",
|
||||
"url": "https://www.patreon.com/file?h=cccccccccccccccccccccccccccccccc&i=1002"
|
||||
}
|
||||
},
|
||||
"relationships": {
|
||||
"attachments_media": {
|
||||
"data": [
|
||||
{"id": "9003", "type": "media"}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "1003",
|
||||
"type": "post",
|
||||
"attributes": {
|
||||
"title": "Inline content post",
|
||||
"published_at": "2026-04-01T08:00:00.000+00:00",
|
||||
"post_type": "text_only",
|
||||
"content": "<p>look</p><figure><img src=\"https://c10.patreonusercontent.com/4/orig/dddddddddddddddddddddddddddddddd/inline.png?token=x&v=2\" alt=\"inline\"></figure>",
|
||||
"url": "https://www.patreon.com/posts/1003"
|
||||
},
|
||||
"relationships": {}
|
||||
},
|
||||
{
|
||||
"id": "1004",
|
||||
"type": "post",
|
||||
"attributes": {
|
||||
"title": "Video post",
|
||||
"published_at": "2026-03-20T18:45:00.000+00:00",
|
||||
"post_type": "video_external_file",
|
||||
"content": "<p>watch</p>",
|
||||
"url": "https://www.patreon.com/posts/1004",
|
||||
"post_file": {
|
||||
"name": "clip.m3u8",
|
||||
"url": "https://stream.mux.com/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee.m3u8?token=jwt"
|
||||
}
|
||||
},
|
||||
"relationships": {}
|
||||
}
|
||||
],
|
||||
"included": [
|
||||
{
|
||||
"id": "9001",
|
||||
"type": "media",
|
||||
"attributes": {
|
||||
"file_name": "gallery-one.jpg",
|
||||
"download_url": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg",
|
||||
"image_urls": {
|
||||
"original": "https://c10.patreonusercontent.com/4/orig/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg",
|
||||
"default": "https://c10.patreonusercontent.com/4/default/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/gallery-one.jpg"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "9002",
|
||||
"type": "media",
|
||||
"attributes": {
|
||||
"file_name": "gallery-two.jpg",
|
||||
"download_url": "https://c10.patreonusercontent.com/4/orig/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/gallery-two.jpg",
|
||||
"image_urls": {
|
||||
"original": "https://c10.patreonusercontent.com/4/orig/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/gallery-two.jpg"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "9003",
|
||||
"type": "media",
|
||||
"attributes": {
|
||||
"file_name": "bonus-pack.zip",
|
||||
"download_url": "https://www.patreon.com/file/download?h=cccccccccccccccccccccccccccccccc"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "5555",
|
||||
"type": "campaign",
|
||||
"attributes": {
|
||||
"name": "Example Creator",
|
||||
"url": "https://www.patreon.com/example"
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": {
|
||||
"next": "https://www.patreon.com/api/posts?filter%5Bcampaign_id%5D=5555&page%5Bcursor%5D=NEXTCURSOR123&sort=-published_at"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
"""PatreonClient tests — parsing only, no network.
|
||||
|
||||
The fixture is loaded from disk and fed directly to the pure parsing methods
|
||||
(_transform / extract_media / _validate_response / parse_cursor_from_url); no
|
||||
live requests call is exercised here.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.app.services.patreon_client import (
|
||||
MediaItem,
|
||||
PatreonClient,
|
||||
PatreonDriftError,
|
||||
parse_cursor_from_url,
|
||||
)
|
||||
|
||||
_FIXTURE = Path(__file__).parent / "fixtures" / "patreon_posts_page1.json"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def response():
|
||||
return json.loads(_FIXTURE.read_text())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
# No cookies file → empty session, but we never make a request in tests.
|
||||
return PatreonClient(cookies_path=None)
|
||||
|
||||
|
||||
def _post_by_id(response, post_id):
|
||||
for post in response["data"]:
|
||||
if post["id"] == post_id:
|
||||
return post
|
||||
raise AssertionError(f"no post {post_id} in fixture")
|
||||
|
||||
|
||||
def test_transform_indexes_included(client, response):
|
||||
index = client._transform(response)
|
||||
assert index[("media", "9001")]["file_name"] == "gallery-one.jpg"
|
||||
assert index[("media", "9003")]["file_name"] == "bonus-pack.zip"
|
||||
assert index[("campaign", "5555")]["name"] == "Example Creator"
|
||||
# Unknown keys are absent, not errors.
|
||||
assert ("media", "0000") not in index
|
||||
|
||||
|
||||
def test_extract_gallery_dedups_image_large(client, response):
|
||||
index = client._transform(response)
|
||||
post = _post_by_id(response, "1001")
|
||||
items = client.extract_media(post, index)
|
||||
|
||||
# Two gallery images; the image_large cover shares filehash aaaa... with
|
||||
# gallery image 9001 → collapses to one, leaving exactly 2 items.
|
||||
assert len(items) == 2
|
||||
assert all(isinstance(m, MediaItem) for m in items)
|
||||
|
||||
kinds = [m.kind for m in items]
|
||||
assert kinds == ["images", "images"] # image_large deduped away
|
||||
|
||||
filenames = {m.filename for m in items}
|
||||
assert filenames == {"gallery-one.jpg", "gallery-two.jpg"}
|
||||
|
||||
hashes = {m.filehash for m in items}
|
||||
assert hashes == {"a" * 32, "b" * 32}
|
||||
assert all(m.post_id == "1001" for m in items)
|
||||
|
||||
|
||||
def test_extract_attachment_and_postfile(client, response):
|
||||
index = client._transform(response)
|
||||
post = _post_by_id(response, "1002")
|
||||
items = client.extract_media(post, index)
|
||||
|
||||
by_kind = {m.kind: m for m in items}
|
||||
# attachments_media → "attachments", plus the post_file → "postfile".
|
||||
assert set(by_kind) == {"attachments", "postfile"}
|
||||
assert by_kind["attachments"].filename == "bonus-pack.zip"
|
||||
assert by_kind["attachments"].filehash == "c" * 32
|
||||
assert by_kind["postfile"].filename == "bonus-pack.zip"
|
||||
# postfile URL also carries the same hash, but they're different kinds in
|
||||
# different relationships; both are kept because dedup is by filehash and
|
||||
# the attachment is encountered first... assert the post_file collapsed.
|
||||
# (attachments resolved before postfile, same hash → postfile dropped.)
|
||||
# So only ONE survives with that hash:
|
||||
surviving_c = [m for m in items if m.filehash == "c" * 32]
|
||||
assert len(surviving_c) == 1
|
||||
|
||||
|
||||
def test_extract_inline_content_img(client, response):
|
||||
index = client._transform(response)
|
||||
post = _post_by_id(response, "1003")
|
||||
items = client.extract_media(post, index)
|
||||
|
||||
assert len(items) == 1
|
||||
item = items[0]
|
||||
assert item.kind == "content"
|
||||
assert item.filehash == "d" * 32
|
||||
# & in the src must be unescaped to & .
|
||||
assert "&" not in item.url
|
||||
assert "token=x&v=2" in item.url
|
||||
|
||||
|
||||
def test_extract_video_postfile(client, response):
|
||||
index = client._transform(response)
|
||||
post = _post_by_id(response, "1004")
|
||||
items = client.extract_media(post, index)
|
||||
|
||||
assert len(items) == 1
|
||||
item = items[0]
|
||||
assert item.kind == "postfile"
|
||||
assert item.url.startswith("https://stream.mux.com/")
|
||||
assert item.filehash == "e" * 32
|
||||
|
||||
|
||||
def test_parse_cursor_from_url_extracts_cursor(response):
|
||||
next_url = response["links"]["next"]
|
||||
assert parse_cursor_from_url(next_url) == "NEXTCURSOR123"
|
||||
|
||||
|
||||
def test_parse_cursor_from_url_none_when_absent():
|
||||
assert parse_cursor_from_url("https://www.patreon.com/api/posts?sort=-x") is None
|
||||
assert parse_cursor_from_url(None) is None
|
||||
|
||||
|
||||
def test_iter_posts_yields_triples_and_paginates(client, response, monkeypatch):
|
||||
# Page 1 = the fixture (4 posts, links.next → NEXTCURSOR123); page 2 = one
|
||||
# post, no links.next → stop. Monkeypatch _fetch so no network is touched.
|
||||
page2 = {
|
||||
"data": [{"id": "1005", "type": "post", "attributes": {"title": "older"}}],
|
||||
"included": [],
|
||||
"links": {},
|
||||
}
|
||||
|
||||
def fake_fetch(campaign_id, cursor):
|
||||
return page2 if cursor == "NEXTCURSOR123" else response
|
||||
|
||||
monkeypatch.setattr(client, "_fetch", fake_fetch)
|
||||
|
||||
seen = list(client.iter_posts("5555"))
|
||||
# 4 posts on page 1 + 1 on page 2.
|
||||
assert [p["id"] for p, _idx, _cur in seen] == ["1001", "1002", "1003", "1004", "1005"]
|
||||
# page_cursor is the cursor that FETCHED the post's page.
|
||||
cursors = {p["id"]: cur for p, _idx, cur in seen}
|
||||
assert cursors["1001"] is None and cursors["1005"] == "NEXTCURSOR123"
|
||||
# included_index is the page's flattened resources, ready for extract_media.
|
||||
page1_index = next(idx for p, idx, _cur in seen if p["id"] == "1001")
|
||||
assert page1_index[("media", "9001")]["file_name"] == "gallery-one.jpg"
|
||||
|
||||
|
||||
def test_missing_data_raises_drift(client):
|
||||
with pytest.raises(PatreonDriftError):
|
||||
client._validate_response({"included": []})
|
||||
|
||||
|
||||
def test_non_list_data_raises_drift(client):
|
||||
with pytest.raises(PatreonDriftError):
|
||||
client._validate_response({"data": {"id": "1"}})
|
||||
|
||||
|
||||
def test_media_missing_file_name_raises_drift(client):
|
||||
# A media resource referenced by a gallery relationship but lacking
|
||||
# file_name is drift from the contract.
|
||||
post = {
|
||||
"id": "2000",
|
||||
"type": "post",
|
||||
"attributes": {"title": "broken"},
|
||||
"relationships": {"images": {"data": [{"id": "7777", "type": "media"}]}},
|
||||
}
|
||||
index = {
|
||||
("media", "7777"): {
|
||||
"download_url": "https://c10.patreonusercontent.com/4/orig/"
|
||||
+ "f" * 32
|
||||
+ "/x.jpg"
|
||||
}
|
||||
}
|
||||
with pytest.raises(PatreonDriftError):
|
||||
client.extract_media(post, index)
|
||||
|
||||
|
||||
def test_media_referenced_but_absent_raises_drift(client):
|
||||
post = {
|
||||
"id": "2001",
|
||||
"type": "post",
|
||||
"attributes": {"title": "dangling"},
|
||||
"relationships": {"images": {"data": [{"id": "8888", "type": "media"}]}},
|
||||
}
|
||||
with pytest.raises(PatreonDriftError):
|
||||
client.extract_media(post, {})
|
||||
Reference in New Issue
Block a user