fix(ingest): skip tier-gated Patreon posts entirely (#874)

Patreon serves only blurred locked-preview thumbnails for posts the
authenticated account can't fully view; the native ingester was
downloading those as real media. current_user_can_view was already in
_FIELDS_POST but never read.

Add PatreonClient.post_is_gated (gate ONLY on explicit
current_user_can_view=False; missing/None → viewable, never over-filter)
and skip gated posts at the top of the ingest_core run() and preview()
loops — no media download AND no post-record stub (operator: 'no stub
for gated content'). Skipped before the post-record block so gated posts
never inflate the #862 body canary; surfaced as 'N gated-skipped' in the
run summary. Same gate in preview() for preview/apply parity (rule 93).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/app/services/ingest_core.py

@@ -155,6 +155,11 @@ class Ingester:
         # comes from the per-media sidecar (gallery-dl path). See [[post-first-ingest-contract]].
         post_record_key = getattr(self.client, "post_record_key", None)
         write_post_record = getattr(self.downloader, "write_post_record", None)
+        # #874: optional client seam — skip tier-gated posts (the account can't
+        # view them, so Patreon serves only blurred locked-preview media) ENTIRELY:
+        # no media download, no post-record stub. Absent on stub/not-yet-migrated
+        # clients → nothing is ever treated as gated.
+        post_is_gated = getattr(self.client, "post_is_gated", None)
         start = time.monotonic()
         last_live = start  # plan #709: last live-progress write timestamp
         log_lines: list[str] = []
@@ -177,6 +182,9 @@ class Ingester:
         # this walk; posts_with_body = how many yielded a non-empty body.
         posts_recorded = 0
         posts_with_body = 0
+        # Tier-gated posts skipped entirely this walk (#874) — surfaced in the
+        # run summary for diagnostics ("a lot of these now").
+        gated_skipped = 0
         # Net-new posts THIS chunk for the live progress badge (plan #704 #5);
         # excludes the re-walked resume page so _backfill_posts stays a monotonic
         # absolute across chunks instead of an inflating sum. posts_processed
@@ -270,6 +278,20 @@ class Ingester:
                 # resume_cursor None, so everything counts.
                 if not (resume_cursor and page_cursor == resume_cursor):
                     chunk_new_posts += 1
+                # Tier-gated post (#874): the account can't fully view it, so
+                # Patreon serves only blurred locked-preview media. Skip it
+                # ENTIRELY — no media download AND no post-record stub (operator
+                # decision: gated content leaves no trace; a later walk re-ingests
+                # it for real once access is gained). Skipped BEFORE the
+                # post-record block so gated posts never inflate the #862 body
+                # canary's sample. post_is_gated gates only on an explicit
+                # current_user_can_view=False (missing/None → viewable).
+                if post_is_gated and post_is_gated(post):
+                    gated_skipped += 1
+                    log_lines.append(
+                        f"  post {post.get('id')} — gated (skipped, no access)"
+                    )
+                    continue
                 # Capture the post body + external links ONCE per post (gated by
                 # the synthetic post key in the seen-ledger), for EVERY post —
                 # whether or not it has downloadable media. This is what makes a
@@ -449,6 +471,7 @@ class Ingester:
             # threshold, surfacing the ratio makes a partial extraction regression
             # visible in the Raw stdout (e.g. "bodies 3/180" reads as off).
             + (f", bodies {posts_with_body}/{posts_recorded}" if posts_recorded else "")
+            + (f", {gated_skipped} gated-skipped" if gated_skipped else "")
             + (", reached end" if reached_bottom else "")
             + (", time-boxed" if budget_hit else "")
         )
@@ -534,6 +557,10 @@ class Ingester:
         sample: list[dict] = []
         unset = object()
         last_page: object = unset
+        # #874: same gated-post gate as run() — the preview must not count
+        # blurred locked-preview media as "new", or it would overstate a gated
+        # source's backlog (preview/apply parity, rule 93).
+        post_is_gated = getattr(self.client, "post_is_gated", None)
         for post, included, page_cursor in self.client.iter_posts(
             campaign_id, cursor=None
         ):
@@ -545,6 +572,8 @@ class Ingester:
                     pages_scanned = page_limit
                     break
             posts_scanned += 1
+            if post_is_gated and post_is_gated(post):
+                continue
             media = self.client.extract_media(post, included)
             if not media:
                 continue

backend/app/services/patreon_client.py

@@ -524,6 +524,23 @@ class PatreonClient:
             "date": published if isinstance(published, str) else None,
         }
 
+    @staticmethod
+    def post_is_gated(post: dict) -> bool:
+        """True when the authenticated account CANNOT view this post's content
+        (#874). Patreon serves only BLURRED locked-preview thumbnails for
+        paywalled / insufficient-tier posts, and `current_user_can_view` on the
+        post attributes is the access flag (it IS in `_FIELDS_POST`). The walk
+        skips a gated post ENTIRELY — no media, no post-record stub — so those
+        unusable previews never get downloaded.
+
+        Gate ONLY on an explicit `current_user_can_view == False`. A missing /
+        None flag (older posts, a sparse fieldset, or API drift) is treated as
+        viewable, so we never over-filter accessible posts on an absent field.
+        Part of the client contract the core consumes via getattr — an optional
+        seam, so stub clients / not-yet-migrated platforms simply never gate."""
+        attrs = post.get("attributes") or {}
+        return attrs.get("current_user_can_view") is False
+
     @staticmethod
     def post_record_key(post: dict) -> tuple[str, str] | None:
         """`(ledger_key, post_id)` for a media-less post's seen-ledger entry, or

tests/test_patreon_client.py

@@ -435,3 +435,27 @@ def test_post_record_key_returns_synthetic_key_and_id():
 def test_post_record_key_none_without_id():
     assert PatreonClient.post_record_key({}) is None
     assert PatreonClient.post_record_key({"id": None}) is None
+
+
+# -- post_is_gated (#874 tier-gated access filter) --------------------------
+
+
+def test_post_is_gated_only_on_explicit_false():
+    # The blurred-preview case: account can't view the post → gate it.
+    assert PatreonClient.post_is_gated(
+        {"attributes": {"current_user_can_view": False}}
+    ) is True
+
+
+def test_post_is_gated_viewable_and_missing_are_not_gated():
+    # Explicit True, a missing flag, an explicit None, and a missing attributes
+    # block ALL count as viewable — gate only on an explicit False so an absent
+    # field never over-filters accessible posts.
+    assert PatreonClient.post_is_gated(
+        {"attributes": {"current_user_can_view": True}}
+    ) is False
+    assert PatreonClient.post_is_gated({"attributes": {}}) is False
+    assert PatreonClient.post_is_gated(
+        {"attributes": {"current_user_can_view": None}}
+    ) is False
+    assert PatreonClient.post_is_gated({}) is False

tests/test_patreon_ingester.py

@@ -50,12 +50,15 @@ class _FakeClient:
     """Stub PatreonClient. `pages` is a list of (page_cursor, [posts]); each post
     is (post_id, [MediaItem]). `raise_on_first` lets a test trip drift."""
 
-    def __init__(self, pages, raise_exc=None, empty_body=False):
+    def __init__(self, pages, raise_exc=None, empty_body=False, gated=None):
         self._pages = pages
         self._raise_exc = raise_exc
         # empty_body simulates a body-field schema break: every post comes back
         # with no content (the #862 canary's trip condition).
         self._empty_body = empty_body
+        # #874: post_ids the account can't view → current_user_can_view=False,
+        # so the walk must skip them entirely (no media, no post-record).
+        self._gated = set(gated or ())
         self.consumed_posts = 0
 
     def iter_posts(self, campaign_id, cursor=None):
@@ -75,6 +78,7 @@ class _FakeClient:
                             "title": f"Post {post_id}",
                             "post_type": "image_file",
                             "content": content,
+                            "current_user_can_view": post_id not in self._gated,
                         },
                     },
                     {},
@@ -87,6 +91,10 @@ class _FakeClient:
     def post_meta(self, post):
         return {"title": post.get("id"), "date": None}
 
+    @staticmethod
+    def post_is_gated(post):
+        return (post.get("attributes") or {}).get("current_user_can_view") is False
+
     @staticmethod
     def post_record_key(post):
         pid = str(post.get("id") or "")
@@ -657,6 +665,52 @@ async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
     )
     assert len(res_recap.post_record_paths) == 1    # gate bypassed → recaptured
     assert downloader2.post_records == 1
+
+
+@pytest.mark.asyncio
+async def test_gated_post_skipped_entirely_no_media_no_record(
+    source_id, sync_engine, tmp_path,
+):
+    """#874: a tier-gated post (current_user_can_view=False) is skipped ENTIRELY
+    — no media download AND no post-record stub — while an accessible post in the
+    same walk is fully ingested. Patreon serves only blurred locked-preview media
+    for gated posts; downloading it produced unusable images."""
+    gm = _media("gated", 1)
+    am = _media("open", 1)
+    client = _FakeClient([(None, [("gated", [gm]), ("open", [am])])], gated={"gated"})
+    downloader = _FakeDownloader(tmp_path)
+    ing = _ingester(sync_engine, tmp_path, client, downloader)
+
+    result = ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="backfill",
+    )
+
+    # Only the accessible post's media downloaded; the gated post contributed
+    # nothing — no file, no post-record.
+    assert result.files_downloaded == 1
+    assert [p for p in result.written_paths if "gated" in p] == []
+    assert downloader.download_calls == 1
+    assert len(result.post_record_paths) == 1     # only the open post
+    assert downloader.post_records == 1
+    # The gated post left NO trace in the seen-ledger (no media key, no post key):
+    # only the open post's media key + synthetic post key are recorded.
+    assert _count_ledger(sync_engine, source_id) == 2
+
+
+@pytest.mark.asyncio
+async def test_preview_excludes_gated_posts(source_id, sync_engine, tmp_path):
+    """#874 preview/apply parity (rule 93): the dry-run must not count a gated
+    post's blurred preview media as new, or it overstates a gated source's
+    backlog."""
+    gm = _media("gated", 1)
+    am = _media("open", 1)
+    client = _FakeClient([(None, [("gated", [gm]), ("open", [am])])], gated={"gated"})
+    ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path))
+
+    preview = ing.preview(source_id, "c1")
+    assert preview["total_new"] == 1                       # only the open post
+    assert [s for s in preview["sample"] if s["title"] == "gated"] == []
     assert res_recap.files_downloaded == 0          # no media re-download
     assert downloader2.download_calls == 0
     # The on-disk media is surfaced as a (path, CDN url) relink pair.