From 796e92540aebc9d97da914f105bfb3e4edb9fcc2 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 14 Jun 2026 12:44:46 -0400 Subject: [PATCH] feat(patreon): capture media-less/text-only posts (post-only records) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Today the ingest core does `if not media: continue`, so a post with no downloadable media (a pure-text post — which often holds the ONLY copy of an external mega/gdrive/pixeldrain link) never upserts a Post. Now the native ingester emits a post-only sidecar (`_post.json`) for every media-less post, gated through the seen-ledger via a synthetic `post:` key so the body is detail-fetched + recorded ONCE (not re-walked every tick); recovery bypasses the gate. Phase 3 imports these via Importer.upsert_post_record, keyed on external_post_id so it UPDATES the same Post a media import would create — never doubles, never clobbers a populated body with an empty one. - gallery_dl.py: DownloadResult.post_record_paths (default []; gallery-dl path unaffected — all constructions are keyword). - ingest_core.py: media-less branch (optional client/downloader seams via getattr; stub clients in tests skip it as before). - patreon_client.py: post_record_key(post). patreon_downloader.py: write_post_record + _write_sidecar_data refactor (shared serializer). - importer.py: upsert_post_record. download_service.py: phase-3 import loop. - tests: client/downloader/ingester (gate + recovery)/importer (no-double). Slice 0b of milestone #64. Refs FC #830. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/download_service.py | 16 ++++++ backend/app/services/gallery_dl.py | 6 ++ backend/app/services/importer.py | 60 ++++++++++++++++++++ backend/app/services/ingest_core.py | 24 ++++++++ backend/app/services/patreon_client.py | 13 +++++ backend/app/services/patreon_downloader.py | 20 ++++++- tests/test_patreon_client.py | 13 +++++ tests/test_patreon_downloader.py | 34 ++++++++++++ tests/test_patreon_ingester.py | 64 ++++++++++++++++++++++ tests/test_sidecar_import.py | 47 +++++++++++++++- 10 files changed, 294 insertions(+), 3 deletions(-) diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index 0fc04e6..03df0c3 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -380,6 +380,22 @@ class DownloadService: else: import_summary["errors"] += 1 + # Post-only records: media-less posts (pure-text) the native ingester + # captured so the artist archive is complete. Upsert each (keyed on + # external_post_id → updates the same Post a media import would create, + # never doubles). No file to clean up; the sidecar stays on disk. + for rec_str in dl_result.post_record_paths or []: + rec_path = Path(rec_str) + if not rec_path.exists(): # noqa: ASYNC240 + continue + + def _upsert(p=rec_path): + return self.importer.upsert_post_record( + p, artist=artist, source=source_row, + ) + + await loop.run_in_executor(None, _upsert) + ev = (await self.async_session.execute( select(DownloadEvent).where(DownloadEvent.id == event_id) )).scalar_one() diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 3add0e1..274ba86 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -147,6 +147,12 @@ class DownloadResult: files_quarantined: int = 0 quarantined_paths: list[str] = field(default_factory=list) written_paths: list[str] = field(default_factory=list) + # Native ingester only: post-only sidecar paths for posts that have NO + # downloadable media (pure-text posts), so the importer can still upsert the + # Post + its body. Empty on the gallery-dl path. Phase 3 imports these via + # Importer.upsert_post_record (keyed on external_post_id → updates, never + # doubles, the same Post a media import would create). + post_record_paths: list[str] = field(default_factory=list) stdout: str = "" stderr: str = "" return_code: int = 0 diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index d5d2185..280b3be 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -710,6 +710,66 @@ class Importer: self.session.commit() return ImportResult(status="refreshed", image_id=existing.id) + def upsert_post_record( + self, sidecar: Path, *, artist: Artist | None = None, + source: Source | None = None, + ) -> bool: + """Upsert the Post for a post-ONLY sidecar (a media-less post), so the + artist archive includes text posts (their body + external links). + + Reuses `_find_or_create_post` (keyed on external_post_id) so it UPDATES + the SAME Post a media import would create — never doubles. Fields are + FILLED, never clobbered with empty: parse_sidecar yields None for empty + values, and a None field is left untouched (an empty feed body never + wipes a populated one). Returns True if a Post was upserted, False if the + sidecar was unusable (parse failure / no artist).""" + try: + data = json.loads(sidecar.read_text("utf-8")) + if not isinstance(data, dict): + raise ValueError("sidecar JSON is not an object") + except Exception as exc: + log.warning("post-record sidecar parse failed for %s: %s", sidecar, exc) + return False + + sd = parse_sidecar(data) + + if artist is None: + name = self._sidecar_artist_name(data) + artist = self._upsert_artist(name) if name else None + if artist is None: + log.warning("post-record sidecar %s has no artist; skipping", sidecar) + return False + + if source is not None: + src = source + else: + platform = sd.platform or "unknown" + src = self._lookup_source_for_sidecar( + artist_id=artist.id, platform=platform, + ) + + epid = sd.external_post_id or sidecar.stem + post = self._find_or_create_post( + source_id=src.id if src else None, + external_post_id=epid, + artist_id=artist.id, + ) + if post.artist_id is None: + post.artist_id = artist.id + if sd.post_url is not None: + post.post_url = sd.post_url + if sd.post_title is not None: + post.post_title = sd.post_title + if sd.post_date is not None: + post.post_date = sd.post_date + if sd.description is not None: + post.description = sd.description + if sd.attachment_count is not None: + post.attachment_count = sd.attachment_count + post.raw_metadata = sd.raw + self.session.commit() + return True + def attach_in_place( self, path: Path, diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 7de059c..ae49dce 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -118,10 +118,16 @@ class Ingester: # tick has no resumable backfill state. checkpoint = mode in ("backfill", "recovery") ledger_key = self._ledger_key + # Optional seams (Patreon native ingester): capture pure-text posts that + # have NO downloadable media so the artist archive is complete. Absent on + # stub clients/downloaders (unit tests) → media-less posts skipped as before. + post_record_key = getattr(self.client, "post_record_key", None) + write_post_record = getattr(self.downloader, "write_post_record", None) start = time.monotonic() last_live = start # plan #709: last live-progress write timestamp log_lines: list[str] = [] written: list[str] = [] + post_records: list[str] = [] quarantined_paths: list[str] = [] downloaded = 0 errors = 0 @@ -158,6 +164,7 @@ class Ingester: files_quarantined=quarantined, quarantined_paths=list(quarantined_paths), written_paths=written, + post_record_paths=list(post_records), stdout="\n".join(log_lines), stderr="", return_code=return_code, @@ -222,6 +229,23 @@ class Ingester: chunk_new_posts += 1 media = self.client.extract_media(post, included) if not media: + # No downloadable media — still capture the post itself + # (title/body/external links) so text-only posts aren't lost. + # Gate via the seen-ledger (synthetic post key) so a text post + # is detail-fetched + recorded ONCE, not re-walked every tick. + if post_record_key and write_post_record: + rk = post_record_key(post) + if rk is not None: + pkey, ppid = rk + already = ( + set() if bypass_seen + else self._seen_keys(source_id, [pkey]) + ) + if pkey not in already: + rec_path = write_post_record(post, artist_slug) + if rec_path is not None: + post_records.append(str(rec_path)) + self._mark_seen(source_id, [(pkey, ppid)]) continue keys = [ledger_key(m) for m in media] diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index 98ba7d6..af56aca 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -526,6 +526,19 @@ class PatreonClient: "date": published if isinstance(published, str) else None, } + @staticmethod + def post_record_key(post: dict) -> tuple[str, str] | None: + """`(ledger_key, post_id)` for a media-less post's seen-ledger entry, or + None when the post has no id. The synthetic `post:` key lets the + generic core gate text-post capture through the SAME seen-ledger as media + — so a text post's body is detail-fetched + recorded ONCE, not re-fetched + every tick. Part of the client contract the core uses via getattr.""" + pid = post.get("id") + pid = str(pid) if pid is not None else "" + if not pid: + return None + return (f"post:{pid}", pid) + # -- iteration --------------------------------------------------------- def iter_posts( diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index 6990bc6..42540cb 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -499,6 +499,12 @@ class PatreonDownloader: registers no derive_post_url, so `url` is trusted as the permalink — we pass the post's attributes.url. """ + return self._write_sidecar_data(post, media_path.with_suffix(".json")) + + def _write_sidecar_data(self, post: dict, sidecar_path: Path) -> Path: + """Serialize the post's metadata to `sidecar_path`. Shared by the + per-media sidecar (next to each file) and the post-only sidecar + (`write_post_record`, for media-less posts).""" attrs = post.get("attributes") or {} title = attrs.get("title") content = attrs.get("content") @@ -524,6 +530,18 @@ class PatreonDownloader: "published_at": published if isinstance(published, str) else None, "url": url if isinstance(url, str) else None, } - sidecar_path = media_path.with_suffix(".json") sidecar_path.write_text(json.dumps(data, indent=2)) return sidecar_path + + def write_post_record(self, post: dict, artist_slug: str) -> Path | None: + """Write a post-ONLY sidecar (no media file) for a media-less post, so + the importer can still upsert the Post + its body — text posts often hold + the only copy of an external link. Named `_post.json`: the + leading underscore keeps it from colliding with a media sidecar + (`_.json`) and from being resolved as some media file's sidecar + by find_sidecar. Returns the path, or None when the post has no id.""" + if not str(post.get("id") or ""): + return None + post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post) + post_dir.mkdir(parents=True, exist_ok=True) + return self._write_sidecar_data(post, post_dir / "_post.json") diff --git a/tests/test_patreon_client.py b/tests/test_patreon_client.py index 101dd36..dd40aa0 100644 --- a/tests/test_patreon_client.py +++ b/tests/test_patreon_client.py @@ -406,3 +406,16 @@ def test_fetch_post_detail_content_swallows_http_and_transport_errors(monkeypatc monkeypatch.setattr(client._session, "get", _boom) assert client.fetch_post_detail_content("42") is None + + +# -- post_record_key (media-less post seen-ledger gate) -------------------- + + +def test_post_record_key_returns_synthetic_key_and_id(): + assert PatreonClient.post_record_key({"id": "987"}) == ("post:987", "987") + assert PatreonClient.post_record_key({"id": 987}) == ("post:987", "987") + + +def test_post_record_key_none_without_id(): + assert PatreonClient.post_record_key({}) is None + assert PatreonClient.post_record_key({"id": None}) is None diff --git a/tests/test_patreon_downloader.py b/tests/test_patreon_downloader.py index e09608f..3a1d15f 100644 --- a/tests/test_patreon_downloader.py +++ b/tests/test_patreon_downloader.py @@ -577,3 +577,37 @@ def test_no_enrichment_when_feed_body_present(tmp_path): ) dl.download_post(_post(), [_img("a.png")], "artist-x") # _post body is non-empty assert calls == [] + + +# -- write_post_record (media-less / text-only post capture) ---------------- + + +def test_write_post_record_writes_enriched_post_only_sidecar(tmp_path): + calls: list[str] = [] + + def _fetcher(post_id: str) -> str: + calls.append(post_id) + return "

text post body

" + + dl = PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, + session=_FakeSession(), content_fetcher=_fetcher, + ) + post = _post() + post["attributes"]["content"] = "" # media-less post, empty feed body + sc = dl.write_post_record(post, "artist-x") + + assert sc is not None + assert sc.name == "_post.json" # can't collide with a media `NN_*.json` + data = json.loads(sc.read_text()) + assert data["id"] == "1001" + assert data["content"] == "

text post body

" + assert calls == ["1001"] + + +def test_write_post_record_none_without_post_id(tmp_path): + dl = PatreonDownloader( + images_root=tmp_path, cookies_path=None, validate=False, + session=_FakeSession(), + ) + assert dl.write_post_record({"id": "", "attributes": {}}, "artist-x") is None diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index b96fa00..a7fbff1 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -68,6 +68,11 @@ class _FakeClient: def post_meta(self, post): return {"title": post.get("id"), "date": None} + @staticmethod + def post_record_key(post): + pid = str(post.get("id") or "") + return (f"post:{pid}", pid) if pid else None + class _FakeDownloader: """Stub PatreonDownloader. Honors the injected is_seen (tier-1); media in @@ -80,6 +85,7 @@ class _FakeDownloader: self.quarantine = set(quarantine or ()) self.error = set(error or ()) self.download_calls = 0 + self.post_records = 0 def download_post(self, post, media_items, artist_slug, *, is_seen, should_stop=lambda: False): @@ -106,6 +112,12 @@ class _FakeDownloader: outcomes.append(MediaOutcome(media=m, status="downloaded", path=p, error=None)) return outcomes + def write_post_record(self, post, artist_slug): + self.post_records += 1 + p = self.tmp_path / f"{post.get('id')}__post.json" + p.write_text("{}") + return p + @pytest.fixture async def source_id(db): @@ -743,3 +755,55 @@ async def test_verify_patreon_credential_delegates_to_client(monkeypatch): ok, msg = await mod.verify_patreon_credential("https://patreon.com/x", None, {}) assert ok is True assert msg == "ok:123" + + +# --- media-less (text-only) post capture ---------------------------------- + + +@pytest.mark.asyncio +async def test_tick_captures_media_less_post_once(source_id, sync_engine, tmp_path): + """A post with NO downloadable media still gets a post-only record, gated by + the seen-ledger (synthetic post key) so a second walk doesn't re-record it.""" + client = _FakeClient([(None, [("ptext", [])])]) + downloader = _FakeDownloader(tmp_path) + ing = _ingester(sync_engine, tmp_path, client, downloader) + + result = ing.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + assert result.success is True + assert len(result.post_record_paths) == 1 + assert downloader.post_records == 1 + # The synthetic `post:ptext` key was marked seen (gates re-capture). + assert _count_ledger(sync_engine, source_id) == 1 + + # Second walk: already recorded → gated, no re-write, no new ledger row. + client2 = _FakeClient([(None, [("ptext", [])])]) + downloader2 = _FakeDownloader(tmp_path) + ing2 = _ingester(sync_engine, tmp_path, client2, downloader2) + result2 = ing2.run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", + ) + assert result2.post_record_paths == [] + assert downloader2.post_records == 0 + assert _count_ledger(sync_engine, source_id) == 1 + + +@pytest.mark.asyncio +async def test_recovery_bypasses_gate_and_re_records(source_id, sync_engine, tmp_path): + """Recovery bypasses the seen-ledger, so a media-less post is re-recorded + (matches recovery semantics for media).""" + client = _FakeClient([(None, [("ptext", [])])]) + ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path)) + ing.run(source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick") + + client2 = _FakeClient([(None, [("ptext", [])])]) + dl2 = _FakeDownloader(tmp_path) + ing2 = _ingester(sync_engine, tmp_path, client2, dl2) + result = ing2.run(source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="recovery") + assert len(result.post_record_paths) == 1 + assert dl2.post_records == 1 diff --git a/tests/test_sidecar_import.py b/tests/test_sidecar_import.py index 0457238..2b7fef3 100644 --- a/tests/test_sidecar_import.py +++ b/tests/test_sidecar_import.py @@ -8,6 +8,7 @@ from PIL import Image from sqlalchemy import func, select from backend.app.models import ( + Artist, ImageProvenance, ImageRecord, ImportSettings, @@ -164,8 +165,6 @@ def test_no_artist_anywhere_skips_provenance(importer, import_layout): def test_sidecar_artist_used_when_no_folder_artist(importer, import_layout): - from backend.app.models import Artist - import_root, _ = import_layout m = import_root / "e.jpg" # root → no folder artist _split(m, "v") @@ -183,3 +182,47 @@ def test_sidecar_artist_used_when_no_folder_artist(importer, import_layout): assert importer.session.execute( select(func.count()).select_from(Source) ).scalar_one() == 0 + + +def test_upsert_post_record_creates_media_less_post(importer, import_layout): + """A post-only sidecar (no media) upserts a Post WITH its body — text posts + are captured even when they have nothing to download.""" + import_root, _ = import_layout + artist = Artist(name="Alice", slug="alice") + importer.session.add(artist) + importer.session.flush() + sc = import_root / "Alice" / "_post.json" + sc.parent.mkdir(parents=True, exist_ok=True) + sc.write_text(json.dumps({ + "category": "patreon", "id": 777, + "url": "https://patreon.com/posts/777", "title": "Text only", + "content": "

links below: mega

", + "published_at": "2023-08-01T00:00:00Z", + })) + assert importer.upsert_post_record(sc, artist=artist) is True + post = importer.session.execute(select(Post)).scalar_one() + assert post.external_post_id == "777" + assert post.post_title == "Text only" + assert post.description == "

links below: mega

" + assert post.post_url == "https://patreon.com/posts/777" + assert post.post_date is not None + + +def test_upsert_post_record_idempotent_no_double(importer, import_layout): + """Re-running upsert_post_record UPDATES the same Post (keyed on + external_post_id) — never doubles.""" + import_root, _ = import_layout + artist = Artist(name="Bob", slug="bob") + importer.session.add(artist) + importer.session.flush() + sc = import_root / "Bob" / "_post.json" + sc.parent.mkdir(parents=True, exist_ok=True) + sc.write_text(json.dumps({ + "category": "patreon", "id": 888, "title": "T", + "content": "

body

", "url": "https://patreon.com/posts/888", + })) + assert importer.upsert_post_record(sc, artist=artist) is True + assert importer.upsert_post_record(sc, artist=artist) is True + assert importer.session.execute( + select(func.count()).select_from(Post) + ).scalar_one() == 1