diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 365dda8..d8ee3b7 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -57,6 +57,19 @@ _ERROR_MAX = 1000 # page-tied update would lurch). Trivial churn (~one single-row UPDATE / 5s). _LIVE_PROGRESS_INTERVAL = 5.0 +# Post-body schema-drift canary (#862). Patreon's body lives in +# content/content_json_string with NO post_type gate, so a field rename (as +# content→content_json_string already was) zeroes EVERY body at once — across +# every artist, every walk. If a native walk records at least this many posts +# and extracts a body from NONE of them, treat it as that break (fail the run +# API_DRIFT) rather than silently archiving empties. A *fraction* threshold would +# false-positive on gallery/art creators who legitimately post images with no +# caption, so the gate is "zero across a minimum sample": a real creator nearly +# always has SOME text across this many posts, a broken parser has none. Set high +# enough that a small tick (a few new posts) can't trip it — only a backfill / +# recapture (the operator's schema-test flow) reaches the sample. +_CANARY_MIN_SAMPLE = 30 + class Ingester: """Generic native-ingest orchestration. Subclass with a platform adapter @@ -149,6 +162,12 @@ class Ingester: dead_lettered = 0 skipped_count = 0 posts_processed = 0 + # Post-body schema-drift canary counters (#862, native ingester only — + # gallery-dl walks never enter the post-record block below so these stay 0 + # and the canary can't fire there). posts_recorded = post-records attempted + # this walk; posts_with_body = how many yielded a non-empty body. + posts_recorded = 0 + posts_with_body = 0 # Net-new posts THIS chunk for the live progress badge (plan #704 #5); # excludes the re-walked resume page so _backfill_posts stays a monotonic # absolute across chunks instead of an inflating sum. posts_processed @@ -261,6 +280,9 @@ class Ingester: ) if pkey not in already: rec = write_post_record(post, artist_slug) + posts_recorded += 1 + if rec.body_chars: + posts_with_body += 1 if rec.path is not None: post_records.append(str(rec.path)) self._mark_seen(source_id, [(pkey, ppid)]) @@ -414,6 +436,10 @@ class Ingester: f"{dead_lettered} dead-lettered, {errors} error(s), " f"{posts_processed} post(s), {len(post_records)} post-record(s), " f"{len(relink)} relinked" + # Body-capture health (#862): even below the canary's red-alarm + # threshold, surfacing the ratio makes a partial extraction regression + # visible in the Raw stdout (e.g. "bodies 3/180" reads as off). + + (f", bodies {posts_with_body}/{posts_recorded}" if posts_recorded else "") + (", reached end" if reached_bottom else "") + (", time-boxed" if budget_hit else "") ) @@ -439,6 +465,27 @@ class Ingester: error_message="Chunk timed out with no progress", ) + # Post-body schema-drift canary (#862): a native walk recorded a + # meaningful sample of posts but extracted a body from NONE of them. Since + # the body field has no post_type gate, that's the signature of Patreon + # renaming/restructuring the body field (as content→content_json_string + # already was) — fail RED (API_DRIFT: "fix is the field-set/parser, not + # creds") so the breakage screams instead of silently archiving empties. + # Only reached on an otherwise-clean walk (timeout/stop/error returned + # above), so it never masks a more specific failure. + if posts_recorded >= _CANARY_MIN_SAMPLE and posts_with_body == 0: + msg = ( + f"Post-body canary: extracted a body from 0 of {posts_recorded} " + "posts — Patreon's body field shape likely changed; the ingester " + "needs a field-set/parser update." + ) + log_lines.append(msg) + log.error("%s (artist=%s)", msg, artist_slug) + return _result( + success=False, return_code=-1, + error_type=ErrorType.API_DRIFT, error_message=msg, + ) + # Normal success: reached the bottom, or a tick that early-outed. rc 0 + # error_type None is REQUIRED for a backfill/recovery walk that reached # the bottom to be marked COMPLETE by diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index f503264..c042478 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -18,6 +18,7 @@ from backend.app.models import ( Source, ) from backend.app.services.gallery_dl import ErrorType +from backend.app.services.ingest_core import _CANARY_MIN_SAMPLE from backend.app.services.patreon_client import ( MediaItem, PatreonAPIError, @@ -49,9 +50,12 @@ class _FakeClient: """Stub PatreonClient. `pages` is a list of (page_cursor, [posts]); each post is (post_id, [MediaItem]). `raise_on_first` lets a test trip drift.""" - def __init__(self, pages, raise_exc=None): + def __init__(self, pages, raise_exc=None, empty_body=False): self._pages = pages self._raise_exc = raise_exc + # empty_body simulates a body-field schema break: every post comes back + # with no content (the #862 canary's trip condition). + self._empty_body = empty_body self.consumed_posts = 0 def iter_posts(self, campaign_id, cursor=None): @@ -62,6 +66,7 @@ class _FakeClient: self.consumed_posts += 1 # Carry feed attributes (title/post_type/content) like the real # client — ingest_core reads these for the per-post diagnostic. + content = "" if self._empty_body else f"

body {post_id}

" yield ( { "id": post_id, @@ -69,7 +74,7 @@ class _FakeClient: "attributes": { "title": f"Post {post_id}", "post_type": "image_file", - "content": f"

body {post_id}

", + "content": content, }, }, {}, @@ -941,3 +946,60 @@ async def test_recovery_bypasses_gate_and_re_records(source_id, sync_engine, tmp url="https://patreon.com/ingest", mode="recovery") assert len(result.post_record_paths) == 1 assert dl2.post_records == 1 + + +# --- post-body schema-drift canary (#862) -------------------------------- + + +def _media_less_posts(n): + # A single page of n media-less posts; each still gets a post-record, so the + # canary counts all n. Unique ids keep the seen-ledger gate from collapsing them. + return [(None, [(f"c{i}", []) for i in range(n)])] + + +@pytest.mark.asyncio +async def test_body_canary_fails_run_when_all_bodies_empty(source_id, sync_engine, tmp_path): + """#862: a native walk that records >= _CANARY_MIN_SAMPLE posts but extracts a + body from NONE of them is the signature of a Patreon body-field rename (as + content→content_json_string was) — fail the run RED (API_DRIFT) instead of + silently archiving empties.""" + client = _FakeClient(_media_less_posts(_CANARY_MIN_SAMPLE), empty_body=True) + ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path)) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="backfill", + ) + assert result.success is False + assert result.error_type == ErrorType.API_DRIFT + assert "canary" in (result.error_message or "").lower() + + +@pytest.mark.asyncio +async def test_body_canary_silent_when_some_bodies_present(source_id, sync_engine, tmp_path): + """No false alarm: a large walk that DOES capture bodies completes normally + (normal operation — e.g. a gallery creator who writes captions).""" + client = _FakeClient(_media_less_posts(_CANARY_MIN_SAMPLE)) # bodies present + ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path)) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="backfill", + ) + assert result.success is True + assert result.error_type is None + + +@pytest.mark.asyncio +async def test_body_canary_silent_below_min_sample(source_id, sync_engine, tmp_path): + """Tick safety: a small walk (a few new captionless posts) is below the sample + floor and must NOT trip the canary, even with every body empty.""" + client = _FakeClient(_media_less_posts(_CANARY_MIN_SAMPLE - 1), empty_body=True) + ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path)) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="backfill", + ) + assert result.success is True + assert result.error_type is None