fix(ingest): skip tier-gated Patreon posts entirely (#874)
Patreon serves only blurred locked-preview thumbnails for posts the authenticated account can't fully view; the native ingester was downloading those as real media. current_user_can_view was already in _FIELDS_POST but never read. Add PatreonClient.post_is_gated (gate ONLY on explicit current_user_can_view=False; missing/None → viewable, never over-filter) and skip gated posts at the top of the ingest_core run() and preview() loops — no media download AND no post-record stub (operator: 'no stub for gated content'). Skipped before the post-record block so gated posts never inflate the #862 body canary; surfaced as 'N gated-skipped' in the run summary. Same gate in preview() for preview/apply parity (rule 93). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -155,6 +155,11 @@ class Ingester:
|
||||
# comes from the per-media sidecar (gallery-dl path). See [[post-first-ingest-contract]].
|
||||
post_record_key = getattr(self.client, "post_record_key", None)
|
||||
write_post_record = getattr(self.downloader, "write_post_record", None)
|
||||
# #874: optional client seam — skip tier-gated posts (the account can't
|
||||
# view them, so Patreon serves only blurred locked-preview media) ENTIRELY:
|
||||
# no media download, no post-record stub. Absent on stub/not-yet-migrated
|
||||
# clients → nothing is ever treated as gated.
|
||||
post_is_gated = getattr(self.client, "post_is_gated", None)
|
||||
start = time.monotonic()
|
||||
last_live = start # plan #709: last live-progress write timestamp
|
||||
log_lines: list[str] = []
|
||||
@@ -177,6 +182,9 @@ class Ingester:
|
||||
# this walk; posts_with_body = how many yielded a non-empty body.
|
||||
posts_recorded = 0
|
||||
posts_with_body = 0
|
||||
# Tier-gated posts skipped entirely this walk (#874) — surfaced in the
|
||||
# run summary for diagnostics ("a lot of these now").
|
||||
gated_skipped = 0
|
||||
# Net-new posts THIS chunk for the live progress badge (plan #704 #5);
|
||||
# excludes the re-walked resume page so _backfill_posts stays a monotonic
|
||||
# absolute across chunks instead of an inflating sum. posts_processed
|
||||
@@ -270,6 +278,20 @@ class Ingester:
|
||||
# resume_cursor None, so everything counts.
|
||||
if not (resume_cursor and page_cursor == resume_cursor):
|
||||
chunk_new_posts += 1
|
||||
# Tier-gated post (#874): the account can't fully view it, so
|
||||
# Patreon serves only blurred locked-preview media. Skip it
|
||||
# ENTIRELY — no media download AND no post-record stub (operator
|
||||
# decision: gated content leaves no trace; a later walk re-ingests
|
||||
# it for real once access is gained). Skipped BEFORE the
|
||||
# post-record block so gated posts never inflate the #862 body
|
||||
# canary's sample. post_is_gated gates only on an explicit
|
||||
# current_user_can_view=False (missing/None → viewable).
|
||||
if post_is_gated and post_is_gated(post):
|
||||
gated_skipped += 1
|
||||
log_lines.append(
|
||||
f" post {post.get('id')} — gated (skipped, no access)"
|
||||
)
|
||||
continue
|
||||
# Capture the post body + external links ONCE per post (gated by
|
||||
# the synthetic post key in the seen-ledger), for EVERY post —
|
||||
# whether or not it has downloadable media. This is what makes a
|
||||
@@ -449,6 +471,7 @@ class Ingester:
|
||||
# threshold, surfacing the ratio makes a partial extraction regression
|
||||
# visible in the Raw stdout (e.g. "bodies 3/180" reads as off).
|
||||
+ (f", bodies {posts_with_body}/{posts_recorded}" if posts_recorded else "")
|
||||
+ (f", {gated_skipped} gated-skipped" if gated_skipped else "")
|
||||
+ (", reached end" if reached_bottom else "")
|
||||
+ (", time-boxed" if budget_hit else "")
|
||||
)
|
||||
@@ -534,6 +557,10 @@ class Ingester:
|
||||
sample: list[dict] = []
|
||||
unset = object()
|
||||
last_page: object = unset
|
||||
# #874: same gated-post gate as run() — the preview must not count
|
||||
# blurred locked-preview media as "new", or it would overstate a gated
|
||||
# source's backlog (preview/apply parity, rule 93).
|
||||
post_is_gated = getattr(self.client, "post_is_gated", None)
|
||||
for post, included, page_cursor in self.client.iter_posts(
|
||||
campaign_id, cursor=None
|
||||
):
|
||||
@@ -545,6 +572,8 @@ class Ingester:
|
||||
pages_scanned = page_limit
|
||||
break
|
||||
posts_scanned += 1
|
||||
if post_is_gated and post_is_gated(post):
|
||||
continue
|
||||
media = self.client.extract_media(post, included)
|
||||
if not media:
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user