feat(patreon): capture media-less/text-only posts (post-only records)
Today the ingest core does `if not media: continue`, so a post with no downloadable media (a pure-text post — which often holds the ONLY copy of an external mega/gdrive/pixeldrain link) never upserts a Post. Now the native ingester emits a post-only sidecar (`_post.json`) for every media-less post, gated through the seen-ledger via a synthetic `post:<id>` key so the body is detail-fetched + recorded ONCE (not re-walked every tick); recovery bypasses the gate. Phase 3 imports these via Importer.upsert_post_record, keyed on external_post_id so it UPDATES the same Post a media import would create — never doubles, never clobbers a populated body with an empty one. - gallery_dl.py: DownloadResult.post_record_paths (default []; gallery-dl path unaffected — all constructions are keyword). - ingest_core.py: media-less branch (optional client/downloader seams via getattr; stub clients in tests skip it as before). - patreon_client.py: post_record_key(post). patreon_downloader.py: write_post_record + _write_sidecar_data refactor (shared serializer). - importer.py: upsert_post_record. download_service.py: phase-3 import loop. - tests: client/downloader/ingester (gate + recovery)/importer (no-double). Slice 0b of milestone #64. Refs FC #830. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -68,6 +68,11 @@ class _FakeClient:
|
||||
def post_meta(self, post):
|
||||
return {"title": post.get("id"), "date": None}
|
||||
|
||||
@staticmethod
|
||||
def post_record_key(post):
|
||||
pid = str(post.get("id") or "")
|
||||
return (f"post:{pid}", pid) if pid else None
|
||||
|
||||
|
||||
class _FakeDownloader:
|
||||
"""Stub PatreonDownloader. Honors the injected is_seen (tier-1); media in
|
||||
@@ -80,6 +85,7 @@ class _FakeDownloader:
|
||||
self.quarantine = set(quarantine or ())
|
||||
self.error = set(error or ())
|
||||
self.download_calls = 0
|
||||
self.post_records = 0
|
||||
|
||||
def download_post(self, post, media_items, artist_slug, *, is_seen,
|
||||
should_stop=lambda: False):
|
||||
@@ -106,6 +112,12 @@ class _FakeDownloader:
|
||||
outcomes.append(MediaOutcome(media=m, status="downloaded", path=p, error=None))
|
||||
return outcomes
|
||||
|
||||
def write_post_record(self, post, artist_slug):
|
||||
self.post_records += 1
|
||||
p = self.tmp_path / f"{post.get('id')}__post.json"
|
||||
p.write_text("{}")
|
||||
return p
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def source_id(db):
|
||||
@@ -743,3 +755,55 @@ async def test_verify_patreon_credential_delegates_to_client(monkeypatch):
|
||||
ok, msg = await mod.verify_patreon_credential("https://patreon.com/x", None, {})
|
||||
assert ok is True
|
||||
assert msg == "ok:123"
|
||||
|
||||
|
||||
# --- media-less (text-only) post capture ----------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tick_captures_media_less_post_once(source_id, sync_engine, tmp_path):
|
||||
"""A post with NO downloadable media still gets a post-only record, gated by
|
||||
the seen-ledger (synthetic post key) so a second walk doesn't re-record it."""
|
||||
client = _FakeClient([(None, [("ptext", [])])])
|
||||
downloader = _FakeDownloader(tmp_path)
|
||||
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
||||
|
||||
result = ing.run(
|
||||
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="tick",
|
||||
)
|
||||
assert result.success is True
|
||||
assert len(result.post_record_paths) == 1
|
||||
assert downloader.post_records == 1
|
||||
# The synthetic `post:ptext` key was marked seen (gates re-capture).
|
||||
assert _count_ledger(sync_engine, source_id) == 1
|
||||
|
||||
# Second walk: already recorded → gated, no re-write, no new ledger row.
|
||||
client2 = _FakeClient([(None, [("ptext", [])])])
|
||||
downloader2 = _FakeDownloader(tmp_path)
|
||||
ing2 = _ingester(sync_engine, tmp_path, client2, downloader2)
|
||||
result2 = ing2.run(
|
||||
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="tick",
|
||||
)
|
||||
assert result2.post_record_paths == []
|
||||
assert downloader2.post_records == 0
|
||||
assert _count_ledger(sync_engine, source_id) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recovery_bypasses_gate_and_re_records(source_id, sync_engine, tmp_path):
|
||||
"""Recovery bypasses the seen-ledger, so a media-less post is re-recorded
|
||||
(matches recovery semantics for media)."""
|
||||
client = _FakeClient([(None, [("ptext", [])])])
|
||||
ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path))
|
||||
ing.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="tick")
|
||||
|
||||
client2 = _FakeClient([(None, [("ptext", [])])])
|
||||
dl2 = _FakeDownloader(tmp_path)
|
||||
ing2 = _ingester(sync_engine, tmp_path, client2, dl2)
|
||||
result = ing2.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="recovery")
|
||||
assert len(result.post_record_paths) == 1
|
||||
assert dl2.post_records == 1
|
||||
|
||||
Reference in New Issue
Block a user