feat(patreon): drift detection + error categorization — build step 4 (plan #697)
Typed, loud failure mapping for the native Patreon ingester so a changed
API shape or expired auth never silently zero-downloads as "success".
- New ErrorType.API_DRIFT (free varchar error_type col → no migration):
distinct from auth so the operator knows the fix is updating the
ingester, not rotating cookies.
- patreon_client: PatreonAPIError carries status_code; new PatreonAuthError
for 401/403 + HTML-login/non-JSON bodies (reclassified from drift —
expired-session is auth, actionable as "rotate cookies").
- patreon_ingester._failure_result maps: PatreonAuthError→AUTH_ERROR,
PatreonDriftError→API_DRIFT ("Patreon API changed — ingester needs
update"), HTTP 429→RATE_LIMITED, 404→NOT_FOUND, other HTTP→HTTP_ERROR,
transport→NETWORK_ERROR. (429 thus drives the platform cooldown.)
- FailingSourcesCard: api_drift chip (red) + hint.
Contract test (new test_patreon_contract.py): the recorded /api/posts
fixture must parse end-to-end (no drift, 5 media across 4 posts) AND the
request params must still carry every field the parser depends on
(file_name, image_urls/download_url, images/attachments_media/media
includes, content/post_file/image post fields) — a trim of either trips a
red build. Plus client HTTP-status classification tests and ingester
error-type mapping tests.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -13,7 +13,12 @@ from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from backend.app.models import Artist, PatreonSeenMedia, Source
|
||||
from backend.app.services.gallery_dl import ErrorType
|
||||
from backend.app.services.patreon_client import MediaItem, PatreonDriftError
|
||||
from backend.app.services.patreon_client import (
|
||||
MediaItem,
|
||||
PatreonAPIError,
|
||||
PatreonAuthError,
|
||||
PatreonDriftError,
|
||||
)
|
||||
from backend.app.services.patreon_downloader import MediaOutcome
|
||||
from backend.app.services.patreon_ingester import PatreonIngester, _ledger_key
|
||||
|
||||
@@ -302,20 +307,63 @@ async def test_mark_seen_is_idempotent(source_id, sync_engine, tmp_path):
|
||||
assert _count_ledger(sync_engine, source_id) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drift_fails_loud(source_id, sync_engine, tmp_path):
|
||||
client = _FakeClient([], raise_exc=PatreonDriftError("missing top-level 'data' key"))
|
||||
downloader = _FakeDownloader(tmp_path)
|
||||
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
||||
|
||||
result = ing.run(
|
||||
def _run_with_client_error(source_id, sync_engine, tmp_path, exc):
|
||||
client = _FakeClient([], raise_exc=exc)
|
||||
ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path))
|
||||
return ing.run(
|
||||
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="tick",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drift_maps_to_api_drift_and_fails_loud(source_id, sync_engine, tmp_path):
|
||||
result = _run_with_client_error(
|
||||
source_id, sync_engine, tmp_path,
|
||||
PatreonDriftError("missing top-level 'data' key"),
|
||||
)
|
||||
assert result.success is False
|
||||
assert result.error_type == ErrorType.API_DRIFT
|
||||
assert "Patreon API changed" in (result.error_message or "")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auth_error_maps_to_auth_error(source_id, sync_engine, tmp_path):
|
||||
result = _run_with_client_error(
|
||||
source_id, sync_engine, tmp_path,
|
||||
PatreonAuthError("HTTP 403 — auth rejected", status_code=403),
|
||||
)
|
||||
assert result.success is False
|
||||
assert result.error_type == ErrorType.AUTH_ERROR
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rate_limit_status_maps_to_rate_limited(source_id, sync_engine, tmp_path):
|
||||
result = _run_with_client_error(
|
||||
source_id, sync_engine, tmp_path,
|
||||
PatreonAPIError("HTTP 429", status_code=429),
|
||||
)
|
||||
assert result.error_type == ErrorType.RATE_LIMITED
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_not_found_status_maps_to_not_found(source_id, sync_engine, tmp_path):
|
||||
result = _run_with_client_error(
|
||||
source_id, sync_engine, tmp_path,
|
||||
PatreonAPIError("HTTP 404", status_code=404),
|
||||
)
|
||||
assert result.error_type == ErrorType.NOT_FOUND
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transport_error_maps_to_network_error(source_id, sync_engine, tmp_path):
|
||||
result = _run_with_client_error(
|
||||
source_id, sync_engine, tmp_path,
|
||||
PatreonAPIError("connection reset"), # no status_code → transport
|
||||
)
|
||||
assert result.error_type == ErrorType.NETWORK_ERROR
|
||||
|
||||
|
||||
def test_ledger_key_video_has_no_filehash():
|
||||
video = MediaItem(
|
||||
url="https://stream.mux.com/abc.m3u8", filename="clip.mp4",
|
||||
|
||||
Reference in New Issue
Block a user