Files
FabledCurator/tests/test_patreon_client.py
T
bvandeusen 682beafbc5
CI / lint (push) Successful in 2s
CI / backend-lint-and-test (push) Successful in 12s
CI / frontend-build (push) Successful in 28s
CI / integration (push) Successful in 2m58s
feat(patreon): drift detection + error categorization — build step 4 (plan #697)
Typed, loud failure mapping for the native Patreon ingester so a changed
API shape or expired auth never silently zero-downloads as "success".

- New ErrorType.API_DRIFT (free varchar error_type col → no migration):
  distinct from auth so the operator knows the fix is updating the
  ingester, not rotating cookies.
- patreon_client: PatreonAPIError carries status_code; new PatreonAuthError
  for 401/403 + HTML-login/non-JSON bodies (reclassified from drift —
  expired-session is auth, actionable as "rotate cookies").
- patreon_ingester._failure_result maps: PatreonAuthError→AUTH_ERROR,
  PatreonDriftError→API_DRIFT ("Patreon API changed — ingester needs
  update"), HTTP 429→RATE_LIMITED, 404→NOT_FOUND, other HTTP→HTTP_ERROR,
  transport→NETWORK_ERROR. (429 thus drives the platform cooldown.)
- FailingSourcesCard: api_drift chip (red) + hint.

Contract test (new test_patreon_contract.py): the recorded /api/posts
fixture must parse end-to-end (no drift, 5 media across 4 posts) AND the
request params must still carry every field the parser depends on
(file_name, image_urls/download_url, images/attachments_media/media
includes, content/post_file/image post fields) — a trim of either trips a
red build. Plus client HTTP-status classification tests and ingester
error-type mapping tests.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-05 21:56:58 -04:00

241 lines
8.1 KiB
Python

"""PatreonClient tests — parsing only, no network.
The fixture is loaded from disk and fed directly to the pure parsing methods
(_transform / extract_media / _validate_response / parse_cursor_from_url); no
live requests call is exercised here.
"""
import json
from pathlib import Path
import pytest
from backend.app.services.patreon_client import (
MediaItem,
PatreonAPIError,
PatreonAuthError,
PatreonClient,
PatreonDriftError,
parse_cursor_from_url,
)
_FIXTURE = Path(__file__).parent / "fixtures" / "patreon_posts_page1.json"
@pytest.fixture
def response():
return json.loads(_FIXTURE.read_text())
@pytest.fixture
def client():
# No cookies file → empty session, but we never make a request in tests.
return PatreonClient(cookies_path=None)
def _post_by_id(response, post_id):
for post in response["data"]:
if post["id"] == post_id:
return post
raise AssertionError(f"no post {post_id} in fixture")
def test_transform_indexes_included(client, response):
index = client._transform(response)
assert index[("media", "9001")]["file_name"] == "gallery-one.jpg"
assert index[("media", "9003")]["file_name"] == "bonus-pack.zip"
assert index[("campaign", "5555")]["name"] == "Example Creator"
# Unknown keys are absent, not errors.
assert ("media", "0000") not in index
def test_extract_gallery_dedups_image_large(client, response):
index = client._transform(response)
post = _post_by_id(response, "1001")
items = client.extract_media(post, index)
# Two gallery images; the image_large cover shares filehash aaaa... with
# gallery image 9001 → collapses to one, leaving exactly 2 items.
assert len(items) == 2
assert all(isinstance(m, MediaItem) for m in items)
kinds = [m.kind for m in items]
assert kinds == ["images", "images"] # image_large deduped away
filenames = {m.filename for m in items}
assert filenames == {"gallery-one.jpg", "gallery-two.jpg"}
hashes = {m.filehash for m in items}
assert hashes == {"a" * 32, "b" * 32}
assert all(m.post_id == "1001" for m in items)
def test_extract_attachment_postfile_same_file_collapses(client, response):
index = client._transform(response)
post = _post_by_id(response, "1002")
items = client.extract_media(post, index)
# The attachment (attachments_media 9003) and the post_file are the SAME
# zip — both carry filehash c…. attachments resolve before postfile, so the
# postfile collapses into the attachment by filehash → exactly one survives.
# (postfile-as-a-kind is exercised separately by the video post test.)
assert len(items) == 1
item = items[0]
assert item.kind == "attachments"
assert item.filename == "bonus-pack.zip"
assert item.filehash == "c" * 32
def test_extract_inline_content_img(client, response):
index = client._transform(response)
post = _post_by_id(response, "1003")
items = client.extract_media(post, index)
assert len(items) == 1
item = items[0]
assert item.kind == "content"
assert item.filehash == "d" * 32
# &amp; in the src must be unescaped to & .
assert "&amp;" not in item.url
assert "token=x&v=2" in item.url
def test_extract_video_postfile(client, response):
index = client._transform(response)
post = _post_by_id(response, "1004")
items = client.extract_media(post, index)
assert len(items) == 1
item = items[0]
assert item.kind == "postfile"
assert item.url.startswith("https://stream.mux.com/")
assert item.filehash == "e" * 32
def test_parse_cursor_from_url_extracts_cursor(response):
next_url = response["links"]["next"]
assert parse_cursor_from_url(next_url) == "NEXTCURSOR123"
def test_parse_cursor_from_url_none_when_absent():
assert parse_cursor_from_url("https://www.patreon.com/api/posts?sort=-x") is None
assert parse_cursor_from_url(None) is None
def test_iter_posts_yields_triples_and_paginates(client, response, monkeypatch):
# Page 1 = the fixture (4 posts, links.next → NEXTCURSOR123); page 2 = one
# post, no links.next → stop. Monkeypatch _fetch so no network is touched.
page2 = {
"data": [{"id": "1005", "type": "post", "attributes": {"title": "older"}}],
"included": [],
"links": {},
}
def fake_fetch(campaign_id, cursor):
return page2 if cursor == "NEXTCURSOR123" else response
monkeypatch.setattr(client, "_fetch", fake_fetch)
seen = list(client.iter_posts("5555"))
# 4 posts on page 1 + 1 on page 2.
assert [p["id"] for p, _idx, _cur in seen] == ["1001", "1002", "1003", "1004", "1005"]
# page_cursor is the cursor that FETCHED the post's page.
cursors = {p["id"]: cur for p, _idx, cur in seen}
assert cursors["1001"] is None and cursors["1005"] == "NEXTCURSOR123"
# included_index is the page's flattened resources, ready for extract_media.
page1_index = next(idx for p, idx, _cur in seen if p["id"] == "1001")
assert page1_index[("media", "9001")]["file_name"] == "gallery-one.jpg"
def test_missing_data_raises_drift(client):
with pytest.raises(PatreonDriftError):
client._validate_response({"included": []})
def test_non_list_data_raises_drift(client):
with pytest.raises(PatreonDriftError):
client._validate_response({"data": {"id": "1"}})
def test_media_missing_file_name_raises_drift(client):
# A media resource referenced by a gallery relationship but lacking
# file_name is drift from the contract.
post = {
"id": "2000",
"type": "post",
"attributes": {"title": "broken"},
"relationships": {"images": {"data": [{"id": "7777", "type": "media"}]}},
}
index = {
("media", "7777"): {
"download_url": "https://c10.patreonusercontent.com/4/orig/"
+ "f" * 32
+ "/x.jpg"
}
}
with pytest.raises(PatreonDriftError):
client.extract_media(post, index)
def test_media_referenced_but_absent_raises_drift(client):
post = {
"id": "2001",
"type": "post",
"attributes": {"title": "dangling"},
"relationships": {"images": {"data": [{"id": "8888", "type": "media"}]}},
}
with pytest.raises(PatreonDriftError):
client.extract_media(post, {})
# -- HTTP status classification (plan #697 step 4) -------------------------
class _FakeResp:
def __init__(self, status_code, *, json_data=None, raise_json=False):
self.status_code = status_code
self._json_data = json_data
self._raise_json = raise_json
def json(self):
if self._raise_json:
raise ValueError("Expecting value: line 1 column 1")
return self._json_data
def _client_returning(monkeypatch, resp):
client = PatreonClient(cookies_path=None)
monkeypatch.setattr(client._session, "get", lambda *a, **k: resp)
return client
@pytest.mark.parametrize("status", [401, 403])
def test_fetch_auth_status_raises_auth_error(monkeypatch, status):
client = _client_returning(monkeypatch, _FakeResp(status))
with pytest.raises(PatreonAuthError) as ei:
client._fetch("5555", None)
assert ei.value.status_code == status
def test_fetch_non_json_body_is_auth_not_drift(monkeypatch):
# An HTML login/challenge page (non-JSON) means the session expired — auth,
# actionable as "rotate cookies", NOT API drift.
client = _client_returning(monkeypatch, _FakeResp(200, raise_json=True))
with pytest.raises(PatreonAuthError):
client._fetch("5555", None)
@pytest.mark.parametrize("status", [404, 429, 500])
def test_fetch_other_status_raises_api_error_with_code(monkeypatch, status):
client = _client_returning(monkeypatch, _FakeResp(status))
with pytest.raises(PatreonAPIError) as ei:
client._fetch("5555", None)
assert ei.value.status_code == status
# Not auth — the ingester maps these by status, not to auth_error.
assert not isinstance(ei.value, PatreonAuthError)
def test_fetch_ok_returns_payload(monkeypatch):
client = _client_returning(monkeypatch, _FakeResp(200, json_data={"data": []}))
assert client._fetch("5555", None) == {"data": []}