diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 072a19c..ba628f7 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -155,6 +155,11 @@ class Ingester: # comes from the per-media sidecar (gallery-dl path). See [[post-first-ingest-contract]]. post_record_key = getattr(self.client, "post_record_key", None) write_post_record = getattr(self.downloader, "write_post_record", None) + # #874: optional client seam — skip tier-gated posts (the account can't + # view them, so Patreon serves only blurred locked-preview media) ENTIRELY: + # no media download, no post-record stub. Absent on stub/not-yet-migrated + # clients → nothing is ever treated as gated. + post_is_gated = getattr(self.client, "post_is_gated", None) start = time.monotonic() last_live = start # plan #709: last live-progress write timestamp log_lines: list[str] = [] @@ -177,6 +182,9 @@ class Ingester: # this walk; posts_with_body = how many yielded a non-empty body. posts_recorded = 0 posts_with_body = 0 + # Tier-gated posts skipped entirely this walk (#874) — surfaced in the + # run summary for diagnostics ("a lot of these now"). + gated_skipped = 0 # Net-new posts THIS chunk for the live progress badge (plan #704 #5); # excludes the re-walked resume page so _backfill_posts stays a monotonic # absolute across chunks instead of an inflating sum. posts_processed @@ -270,6 +278,20 @@ class Ingester: # resume_cursor None, so everything counts. if not (resume_cursor and page_cursor == resume_cursor): chunk_new_posts += 1 + # Tier-gated post (#874): the account can't fully view it, so + # Patreon serves only blurred locked-preview media. Skip it + # ENTIRELY — no media download AND no post-record stub (operator + # decision: gated content leaves no trace; a later walk re-ingests + # it for real once access is gained). Skipped BEFORE the + # post-record block so gated posts never inflate the #862 body + # canary's sample. post_is_gated gates only on an explicit + # current_user_can_view=False (missing/None → viewable). + if post_is_gated and post_is_gated(post): + gated_skipped += 1 + log_lines.append( + f" post {post.get('id')} — gated (skipped, no access)" + ) + continue # Capture the post body + external links ONCE per post (gated by # the synthetic post key in the seen-ledger), for EVERY post — # whether or not it has downloadable media. This is what makes a @@ -449,6 +471,7 @@ class Ingester: # threshold, surfacing the ratio makes a partial extraction regression # visible in the Raw stdout (e.g. "bodies 3/180" reads as off). + (f", bodies {posts_with_body}/{posts_recorded}" if posts_recorded else "") + + (f", {gated_skipped} gated-skipped" if gated_skipped else "") + (", reached end" if reached_bottom else "") + (", time-boxed" if budget_hit else "") ) @@ -534,6 +557,10 @@ class Ingester: sample: list[dict] = [] unset = object() last_page: object = unset + # #874: same gated-post gate as run() — the preview must not count + # blurred locked-preview media as "new", or it would overstate a gated + # source's backlog (preview/apply parity, rule 93). + post_is_gated = getattr(self.client, "post_is_gated", None) for post, included, page_cursor in self.client.iter_posts( campaign_id, cursor=None ): @@ -545,6 +572,8 @@ class Ingester: pages_scanned = page_limit break posts_scanned += 1 + if post_is_gated and post_is_gated(post): + continue media = self.client.extract_media(post, included) if not media: continue diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index dea59df..907ce73 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -524,6 +524,23 @@ class PatreonClient: "date": published if isinstance(published, str) else None, } + @staticmethod + def post_is_gated(post: dict) -> bool: + """True when the authenticated account CANNOT view this post's content + (#874). Patreon serves only BLURRED locked-preview thumbnails for + paywalled / insufficient-tier posts, and `current_user_can_view` on the + post attributes is the access flag (it IS in `_FIELDS_POST`). The walk + skips a gated post ENTIRELY — no media, no post-record stub — so those + unusable previews never get downloaded. + + Gate ONLY on an explicit `current_user_can_view == False`. A missing / + None flag (older posts, a sparse fieldset, or API drift) is treated as + viewable, so we never over-filter accessible posts on an absent field. + Part of the client contract the core consumes via getattr — an optional + seam, so stub clients / not-yet-migrated platforms simply never gate.""" + attrs = post.get("attributes") or {} + return attrs.get("current_user_can_view") is False + @staticmethod def post_record_key(post: dict) -> tuple[str, str] | None: """`(ledger_key, post_id)` for a media-less post's seen-ledger entry, or diff --git a/tests/test_patreon_client.py b/tests/test_patreon_client.py index 84edaa3..2106237 100644 --- a/tests/test_patreon_client.py +++ b/tests/test_patreon_client.py @@ -435,3 +435,27 @@ def test_post_record_key_returns_synthetic_key_and_id(): def test_post_record_key_none_without_id(): assert PatreonClient.post_record_key({}) is None assert PatreonClient.post_record_key({"id": None}) is None + + +# -- post_is_gated (#874 tier-gated access filter) -------------------------- + + +def test_post_is_gated_only_on_explicit_false(): + # The blurred-preview case: account can't view the post → gate it. + assert PatreonClient.post_is_gated( + {"attributes": {"current_user_can_view": False}} + ) is True + + +def test_post_is_gated_viewable_and_missing_are_not_gated(): + # Explicit True, a missing flag, an explicit None, and a missing attributes + # block ALL count as viewable — gate only on an explicit False so an absent + # field never over-filters accessible posts. + assert PatreonClient.post_is_gated( + {"attributes": {"current_user_can_view": True}} + ) is False + assert PatreonClient.post_is_gated({"attributes": {}}) is False + assert PatreonClient.post_is_gated( + {"attributes": {"current_user_can_view": None}} + ) is False + assert PatreonClient.post_is_gated({}) is False diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index c042478..64613cf 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -50,12 +50,15 @@ class _FakeClient: """Stub PatreonClient. `pages` is a list of (page_cursor, [posts]); each post is (post_id, [MediaItem]). `raise_on_first` lets a test trip drift.""" - def __init__(self, pages, raise_exc=None, empty_body=False): + def __init__(self, pages, raise_exc=None, empty_body=False, gated=None): self._pages = pages self._raise_exc = raise_exc # empty_body simulates a body-field schema break: every post comes back # with no content (the #862 canary's trip condition). self._empty_body = empty_body + # #874: post_ids the account can't view → current_user_can_view=False, + # so the walk must skip them entirely (no media, no post-record). + self._gated = set(gated or ()) self.consumed_posts = 0 def iter_posts(self, campaign_id, cursor=None): @@ -75,6 +78,7 @@ class _FakeClient: "title": f"Post {post_id}", "post_type": "image_file", "content": content, + "current_user_can_view": post_id not in self._gated, }, }, {}, @@ -87,6 +91,10 @@ class _FakeClient: def post_meta(self, post): return {"title": post.get("id"), "date": None} + @staticmethod + def post_is_gated(post): + return (post.get("attributes") or {}).get("current_user_can_view") is False + @staticmethod def post_record_key(post): pid = str(post.get("id") or "") @@ -657,6 +665,52 @@ async def test_backfill_skips_already_captured_post_but_recapture_forces_it( ) assert len(res_recap.post_record_paths) == 1 # gate bypassed → recaptured assert downloader2.post_records == 1 + + +@pytest.mark.asyncio +async def test_gated_post_skipped_entirely_no_media_no_record( + source_id, sync_engine, tmp_path, +): + """#874: a tier-gated post (current_user_can_view=False) is skipped ENTIRELY + — no media download AND no post-record stub — while an accessible post in the + same walk is fully ingested. Patreon serves only blurred locked-preview media + for gated posts; downloading it produced unusable images.""" + gm = _media("gated", 1) + am = _media("open", 1) + client = _FakeClient([(None, [("gated", [gm]), ("open", [am])])], gated={"gated"}) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="backfill", + ) + + # Only the accessible post's media downloaded; the gated post contributed + # nothing — no file, no post-record. + assert result.files_downloaded == 1 + assert [p for p in result.written_paths if "gated" in p] == [] + assert downloader.download_calls == 1 + assert len(result.post_record_paths) == 1 # only the open post + assert downloader.post_records == 1 + # The gated post left NO trace in the seen-ledger (no media key, no post key): + # only the open post's media key + synthetic post key are recorded. + assert _count_ledger(sync_engine, source_id) == 2 + + +@pytest.mark.asyncio +async def test_preview_excludes_gated_posts(source_id, sync_engine, tmp_path): + """#874 preview/apply parity (rule 93): the dry-run must not count a gated + post's blurred preview media as new, or it overstates a gated source's + backlog.""" + gm = _media("gated", 1) + am = _media("open", 1) + client = _FakeClient([(None, [("gated", [gm]), ("open", [am])])], gated={"gated"}) + ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path)) + + preview = ing.preview(source_id, "c1") + assert preview["total_new"] == 1 # only the open post + assert [s for s in preview["sample"] if s["title"] == "gated"] == [] assert res_recap.files_downloaded == 0 # no media re-download assert downloader2.download_calls == 0 # The on-disk media is surfaced as a (path, CDN url) relink pair.