feat(patreon): capture media-less/text-only posts (post-only records)

Today the ingest core does `if not media: continue`, so a post with no
downloadable media (a pure-text post — which often holds the ONLY copy of an
external mega/gdrive/pixeldrain link) never upserts a Post. Now the native
ingester emits a post-only sidecar (`_post.json`) for every media-less post,
gated through the seen-ledger via a synthetic `post:<id>` key so the body is
detail-fetched + recorded ONCE (not re-walked every tick); recovery bypasses
the gate. Phase 3 imports these via Importer.upsert_post_record, keyed on
external_post_id so it UPDATES the same Post a media import would create —
never doubles, never clobbers a populated body with an empty one.

- gallery_dl.py: DownloadResult.post_record_paths (default []; gallery-dl path
  unaffected — all constructions are keyword).
- ingest_core.py: media-less branch (optional client/downloader seams via
  getattr; stub clients in tests skip it as before).
- patreon_client.py: post_record_key(post). patreon_downloader.py:
  write_post_record + _write_sidecar_data refactor (shared serializer).
- importer.py: upsert_post_record. download_service.py: phase-3 import loop.
- tests: client/downloader/ingester (gate + recovery)/importer (no-double).

Slice 0b of milestone #64. Refs FC #830.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/app/services/download_service.py

@@ -380,6 +380,22 @@ class DownloadService:
             else:
                 import_summary["errors"] += 1
 
+        # Post-only records: media-less posts (pure-text) the native ingester
+        # captured so the artist archive is complete. Upsert each (keyed on
+        # external_post_id → updates the same Post a media import would create,
+        # never doubles). No file to clean up; the sidecar stays on disk.
+        for rec_str in dl_result.post_record_paths or []:
+            rec_path = Path(rec_str)
+            if not rec_path.exists():  # noqa: ASYNC240
+                continue
+
+            def _upsert(p=rec_path):
+                return self.importer.upsert_post_record(
+                    p, artist=artist, source=source_row,
+                )
+
+            await loop.run_in_executor(None, _upsert)
+
         ev = (await self.async_session.execute(
             select(DownloadEvent).where(DownloadEvent.id == event_id)
         )).scalar_one()

backend/app/services/gallery_dl.py

@@ -147,6 +147,12 @@ class DownloadResult:
     files_quarantined: int = 0
     quarantined_paths: list[str] = field(default_factory=list)
     written_paths: list[str] = field(default_factory=list)
+    # Native ingester only: post-only sidecar paths for posts that have NO
+    # downloadable media (pure-text posts), so the importer can still upsert the
+    # Post + its body. Empty on the gallery-dl path. Phase 3 imports these via
+    # Importer.upsert_post_record (keyed on external_post_id → updates, never
+    # doubles, the same Post a media import would create).
+    post_record_paths: list[str] = field(default_factory=list)
     stdout: str = ""
     stderr: str = ""
     return_code: int = 0

backend/app/services/importer.py

@@ -710,6 +710,66 @@ class Importer:
         self.session.commit()
         return ImportResult(status="refreshed", image_id=existing.id)
 
+    def upsert_post_record(
+        self, sidecar: Path, *, artist: Artist | None = None,
+        source: Source | None = None,
+    ) -> bool:
+        """Upsert the Post for a post-ONLY sidecar (a media-less post), so the
+        artist archive includes text posts (their body + external links).
+
+        Reuses `_find_or_create_post` (keyed on external_post_id) so it UPDATES
+        the SAME Post a media import would create — never doubles. Fields are
+        FILLED, never clobbered with empty: parse_sidecar yields None for empty
+        values, and a None field is left untouched (an empty feed body never
+        wipes a populated one). Returns True if a Post was upserted, False if the
+        sidecar was unusable (parse failure / no artist)."""
+        try:
+            data = json.loads(sidecar.read_text("utf-8"))
+            if not isinstance(data, dict):
+                raise ValueError("sidecar JSON is not an object")
+        except Exception as exc:
+            log.warning("post-record sidecar parse failed for %s: %s", sidecar, exc)
+            return False
+
+        sd = parse_sidecar(data)
+
+        if artist is None:
+            name = self._sidecar_artist_name(data)
+            artist = self._upsert_artist(name) if name else None
+        if artist is None:
+            log.warning("post-record sidecar %s has no artist; skipping", sidecar)
+            return False
+
+        if source is not None:
+            src = source
+        else:
+            platform = sd.platform or "unknown"
+            src = self._lookup_source_for_sidecar(
+                artist_id=artist.id, platform=platform,
+            )
+
+        epid = sd.external_post_id or sidecar.stem
+        post = self._find_or_create_post(
+            source_id=src.id if src else None,
+            external_post_id=epid,
+            artist_id=artist.id,
+        )
+        if post.artist_id is None:
+            post.artist_id = artist.id
+        if sd.post_url is not None:
+            post.post_url = sd.post_url
+        if sd.post_title is not None:
+            post.post_title = sd.post_title
+        if sd.post_date is not None:
+            post.post_date = sd.post_date
+        if sd.description is not None:
+            post.description = sd.description
+        if sd.attachment_count is not None:
+            post.attachment_count = sd.attachment_count
+        post.raw_metadata = sd.raw
+        self.session.commit()
+        return True
+
     def attach_in_place(
         self,
         path: Path,

backend/app/services/ingest_core.py

@@ -118,10 +118,16 @@ class Ingester:
         # tick has no resumable backfill state.
         checkpoint = mode in ("backfill", "recovery")
         ledger_key = self._ledger_key
+        # Optional seams (Patreon native ingester): capture pure-text posts that
+        # have NO downloadable media so the artist archive is complete. Absent on
+        # stub clients/downloaders (unit tests) → media-less posts skipped as before.
+        post_record_key = getattr(self.client, "post_record_key", None)
+        write_post_record = getattr(self.downloader, "write_post_record", None)
         start = time.monotonic()
         last_live = start  # plan #709: last live-progress write timestamp
         log_lines: list[str] = []
         written: list[str] = []
+        post_records: list[str] = []
         quarantined_paths: list[str] = []
         downloaded = 0
         errors = 0
@@ -158,6 +164,7 @@ class Ingester:
                 files_quarantined=quarantined,
                 quarantined_paths=list(quarantined_paths),
                 written_paths=written,
+                post_record_paths=list(post_records),
                 stdout="\n".join(log_lines),
                 stderr="",
                 return_code=return_code,
@@ -222,6 +229,23 @@ class Ingester:
                     chunk_new_posts += 1
                 media = self.client.extract_media(post, included)
                 if not media:
+                    # No downloadable media — still capture the post itself
+                    # (title/body/external links) so text-only posts aren't lost.
+                    # Gate via the seen-ledger (synthetic post key) so a text post
+                    # is detail-fetched + recorded ONCE, not re-walked every tick.
+                    if post_record_key and write_post_record:
+                        rk = post_record_key(post)
+                        if rk is not None:
+                            pkey, ppid = rk
+                            already = (
+                                set() if bypass_seen
+                                else self._seen_keys(source_id, [pkey])
+                            )
+                            if pkey not in already:
+                                rec_path = write_post_record(post, artist_slug)
+                                if rec_path is not None:
+                                    post_records.append(str(rec_path))
+                                    self._mark_seen(source_id, [(pkey, ppid)])
                     continue
 
                 keys = [ledger_key(m) for m in media]

backend/app/services/patreon_client.py

@@ -526,6 +526,19 @@ class PatreonClient:
             "date": published if isinstance(published, str) else None,
         }
 
+    @staticmethod
+    def post_record_key(post: dict) -> tuple[str, str] | None:
+        """`(ledger_key, post_id)` for a media-less post's seen-ledger entry, or
+        None when the post has no id. The synthetic `post:<id>` key lets the
+        generic core gate text-post capture through the SAME seen-ledger as media
+        — so a text post's body is detail-fetched + recorded ONCE, not re-fetched
+        every tick. Part of the client contract the core uses via getattr."""
+        pid = post.get("id")
+        pid = str(pid) if pid is not None else ""
+        if not pid:
+            return None
+        return (f"post:{pid}", pid)
+
     # -- iteration ---------------------------------------------------------
 
     def iter_posts(

backend/app/services/patreon_downloader.py

@@ -499,6 +499,12 @@ class PatreonDownloader:
         registers no derive_post_url, so `url` is trusted as the permalink — we
         pass the post's attributes.url.
         """
+        return self._write_sidecar_data(post, media_path.with_suffix(".json"))
+
+    def _write_sidecar_data(self, post: dict, sidecar_path: Path) -> Path:
+        """Serialize the post's metadata to `sidecar_path`. Shared by the
+        per-media sidecar (next to each file) and the post-only sidecar
+        (`write_post_record`, for media-less posts)."""
         attrs = post.get("attributes") or {}
         title = attrs.get("title")
         content = attrs.get("content")
@@ -524,6 +530,18 @@ class PatreonDownloader:
             "published_at": published if isinstance(published, str) else None,
             "url": url if isinstance(url, str) else None,
         }
-        sidecar_path = media_path.with_suffix(".json")
         sidecar_path.write_text(json.dumps(data, indent=2))
         return sidecar_path
+
+    def write_post_record(self, post: dict, artist_slug: str) -> Path | None:
+        """Write a post-ONLY sidecar (no media file) for a media-less post, so
+        the importer can still upsert the Post + its body — text posts often hold
+        the only copy of an external <a href> link. Named `_post.json`: the
+        leading underscore keeps it from colliding with a media sidecar
+        (`<NN>_<stem>.json`) and from being resolved as some media file's sidecar
+        by find_sidecar. Returns the path, or None when the post has no id."""
+        if not str(post.get("id") or ""):
+            return None
+        post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post)
+        post_dir.mkdir(parents=True, exist_ok=True)
+        return self._write_sidecar_data(post, post_dir / "_post.json")

tests/test_patreon_client.py

@@ -406,3 +406,16 @@ def test_fetch_post_detail_content_swallows_http_and_transport_errors(monkeypatc
 
     monkeypatch.setattr(client._session, "get", _boom)
     assert client.fetch_post_detail_content("42") is None
+
+
+# -- post_record_key (media-less post seen-ledger gate) --------------------
+
+
+def test_post_record_key_returns_synthetic_key_and_id():
+    assert PatreonClient.post_record_key({"id": "987"}) == ("post:987", "987")
+    assert PatreonClient.post_record_key({"id": 987}) == ("post:987", "987")
+
+
+def test_post_record_key_none_without_id():
+    assert PatreonClient.post_record_key({}) is None
+    assert PatreonClient.post_record_key({"id": None}) is None

tests/test_patreon_downloader.py

@@ -577,3 +577,37 @@ def test_no_enrichment_when_feed_body_present(tmp_path):
     )
     dl.download_post(_post(), [_img("a.png")], "artist-x")  # _post body is non-empty
     assert calls == []
+
+
+# -- write_post_record (media-less / text-only post capture) ----------------
+
+
+def test_write_post_record_writes_enriched_post_only_sidecar(tmp_path):
+    calls: list[str] = []
+
+    def _fetcher(post_id: str) -> str:
+        calls.append(post_id)
+        return "<p>text post body</p>"
+
+    dl = PatreonDownloader(
+        images_root=tmp_path, cookies_path=None, validate=False,
+        session=_FakeSession(), content_fetcher=_fetcher,
+    )
+    post = _post()
+    post["attributes"]["content"] = ""  # media-less post, empty feed body
+    sc = dl.write_post_record(post, "artist-x")
+
+    assert sc is not None
+    assert sc.name == "_post.json"  # can't collide with a media `NN_*.json`
+    data = json.loads(sc.read_text())
+    assert data["id"] == "1001"
+    assert data["content"] == "<p>text post body</p>"
+    assert calls == ["1001"]
+
+
+def test_write_post_record_none_without_post_id(tmp_path):
+    dl = PatreonDownloader(
+        images_root=tmp_path, cookies_path=None, validate=False,
+        session=_FakeSession(),
+    )
+    assert dl.write_post_record({"id": "", "attributes": {}}, "artist-x") is None

tests/test_patreon_ingester.py

@@ -68,6 +68,11 @@ class _FakeClient:
     def post_meta(self, post):
         return {"title": post.get("id"), "date": None}
 
+    @staticmethod
+    def post_record_key(post):
+        pid = str(post.get("id") or "")
+        return (f"post:{pid}", pid) if pid else None
+
 
 class _FakeDownloader:
     """Stub PatreonDownloader. Honors the injected is_seen (tier-1); media in
@@ -80,6 +85,7 @@ class _FakeDownloader:
         self.quarantine = set(quarantine or ())
         self.error = set(error or ())
         self.download_calls = 0
+        self.post_records = 0
 
     def download_post(self, post, media_items, artist_slug, *, is_seen,
                       should_stop=lambda: False):
@@ -106,6 +112,12 @@ class _FakeDownloader:
                 outcomes.append(MediaOutcome(media=m, status="downloaded", path=p, error=None))
         return outcomes
 
+    def write_post_record(self, post, artist_slug):
+        self.post_records += 1
+        p = self.tmp_path / f"{post.get('id')}__post.json"
+        p.write_text("{}")
+        return p
+
 
 @pytest.fixture
 async def source_id(db):
@@ -743,3 +755,55 @@ async def test_verify_patreon_credential_delegates_to_client(monkeypatch):
     ok, msg = await mod.verify_patreon_credential("https://patreon.com/x", None, {})
     assert ok is True
     assert msg == "ok:123"
+
+
+# --- media-less (text-only) post capture ----------------------------------
+
+
+@pytest.mark.asyncio
+async def test_tick_captures_media_less_post_once(source_id, sync_engine, tmp_path):
+    """A post with NO downloadable media still gets a post-only record, gated by
+    the seen-ledger (synthetic post key) so a second walk doesn't re-record it."""
+    client = _FakeClient([(None, [("ptext", [])])])
+    downloader = _FakeDownloader(tmp_path)
+    ing = _ingester(sync_engine, tmp_path, client, downloader)
+
+    result = ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="tick",
+    )
+    assert result.success is True
+    assert len(result.post_record_paths) == 1
+    assert downloader.post_records == 1
+    # The synthetic `post:ptext` key was marked seen (gates re-capture).
+    assert _count_ledger(sync_engine, source_id) == 1
+
+    # Second walk: already recorded → gated, no re-write, no new ledger row.
+    client2 = _FakeClient([(None, [("ptext", [])])])
+    downloader2 = _FakeDownloader(tmp_path)
+    ing2 = _ingester(sync_engine, tmp_path, client2, downloader2)
+    result2 = ing2.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="tick",
+    )
+    assert result2.post_record_paths == []
+    assert downloader2.post_records == 0
+    assert _count_ledger(sync_engine, source_id) == 1
+
+
+@pytest.mark.asyncio
+async def test_recovery_bypasses_gate_and_re_records(source_id, sync_engine, tmp_path):
+    """Recovery bypasses the seen-ledger, so a media-less post is re-recorded
+    (matches recovery semantics for media)."""
+    client = _FakeClient([(None, [("ptext", [])])])
+    ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path))
+    ing.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
+            url="https://patreon.com/ingest", mode="tick")
+
+    client2 = _FakeClient([(None, [("ptext", [])])])
+    dl2 = _FakeDownloader(tmp_path)
+    ing2 = _ingester(sync_engine, tmp_path, client2, dl2)
+    result = ing2.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
+                      url="https://patreon.com/ingest", mode="recovery")
+    assert len(result.post_record_paths) == 1
+    assert dl2.post_records == 1

tests/test_sidecar_import.py

@@ -8,6 +8,7 @@ from PIL import Image
 from sqlalchemy import func, select
 
 from backend.app.models import (
+    Artist,
     ImageProvenance,
     ImageRecord,
     ImportSettings,
@@ -164,8 +165,6 @@ def test_no_artist_anywhere_skips_provenance(importer, import_layout):
 
 
 def test_sidecar_artist_used_when_no_folder_artist(importer, import_layout):
-    from backend.app.models import Artist
-
     import_root, _ = import_layout
     m = import_root / "e.jpg"                 # root → no folder artist
     _split(m, "v")
@@ -183,3 +182,47 @@ def test_sidecar_artist_used_when_no_folder_artist(importer, import_layout):
     assert importer.session.execute(
         select(func.count()).select_from(Source)
     ).scalar_one() == 0
+
+
+def test_upsert_post_record_creates_media_less_post(importer, import_layout):
+    """A post-only sidecar (no media) upserts a Post WITH its body — text posts
+    are captured even when they have nothing to download."""
+    import_root, _ = import_layout
+    artist = Artist(name="Alice", slug="alice")
+    importer.session.add(artist)
+    importer.session.flush()
+    sc = import_root / "Alice" / "_post.json"
+    sc.parent.mkdir(parents=True, exist_ok=True)
+    sc.write_text(json.dumps({
+        "category": "patreon", "id": 777,
+        "url": "https://patreon.com/posts/777", "title": "Text only",
+        "content": "<p>links below: mega</p>",
+        "published_at": "2023-08-01T00:00:00Z",
+    }))
+    assert importer.upsert_post_record(sc, artist=artist) is True
+    post = importer.session.execute(select(Post)).scalar_one()
+    assert post.external_post_id == "777"
+    assert post.post_title == "Text only"
+    assert post.description == "<p>links below: mega</p>"
+    assert post.post_url == "https://patreon.com/posts/777"
+    assert post.post_date is not None
+
+
+def test_upsert_post_record_idempotent_no_double(importer, import_layout):
+    """Re-running upsert_post_record UPDATES the same Post (keyed on
+    external_post_id) — never doubles."""
+    import_root, _ = import_layout
+    artist = Artist(name="Bob", slug="bob")
+    importer.session.add(artist)
+    importer.session.flush()
+    sc = import_root / "Bob" / "_post.json"
+    sc.parent.mkdir(parents=True, exist_ok=True)
+    sc.write_text(json.dumps({
+        "category": "patreon", "id": 888, "title": "T",
+        "content": "<p>body</p>", "url": "https://patreon.com/posts/888",
+    }))
+    assert importer.upsert_post_record(sc, artist=artist) is True
+    assert importer.upsert_post_record(sc, artist=artist) is True
+    assert importer.session.execute(
+        select(func.count()).select_from(Post)
+    ).scalar_one() == 1