feat(patreon): capture media-less/text-only posts (post-only records)
Today the ingest core does `if not media: continue`, so a post with no downloadable media (a pure-text post — which often holds the ONLY copy of an external mega/gdrive/pixeldrain link) never upserts a Post. Now the native ingester emits a post-only sidecar (`_post.json`) for every media-less post, gated through the seen-ledger via a synthetic `post:<id>` key so the body is detail-fetched + recorded ONCE (not re-walked every tick); recovery bypasses the gate. Phase 3 imports these via Importer.upsert_post_record, keyed on external_post_id so it UPDATES the same Post a media import would create — never doubles, never clobbers a populated body with an empty one. - gallery_dl.py: DownloadResult.post_record_paths (default []; gallery-dl path unaffected — all constructions are keyword). - ingest_core.py: media-less branch (optional client/downloader seams via getattr; stub clients in tests skip it as before). - patreon_client.py: post_record_key(post). patreon_downloader.py: write_post_record + _write_sidecar_data refactor (shared serializer). - importer.py: upsert_post_record. download_service.py: phase-3 import loop. - tests: client/downloader/ingester (gate + recovery)/importer (no-double). Slice 0b of milestone #64. Refs FC #830. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -499,6 +499,12 @@ class PatreonDownloader:
|
||||
registers no derive_post_url, so `url` is trusted as the permalink — we
|
||||
pass the post's attributes.url.
|
||||
"""
|
||||
return self._write_sidecar_data(post, media_path.with_suffix(".json"))
|
||||
|
||||
def _write_sidecar_data(self, post: dict, sidecar_path: Path) -> Path:
|
||||
"""Serialize the post's metadata to `sidecar_path`. Shared by the
|
||||
per-media sidecar (next to each file) and the post-only sidecar
|
||||
(`write_post_record`, for media-less posts)."""
|
||||
attrs = post.get("attributes") or {}
|
||||
title = attrs.get("title")
|
||||
content = attrs.get("content")
|
||||
@@ -524,6 +530,18 @@ class PatreonDownloader:
|
||||
"published_at": published if isinstance(published, str) else None,
|
||||
"url": url if isinstance(url, str) else None,
|
||||
}
|
||||
sidecar_path = media_path.with_suffix(".json")
|
||||
sidecar_path.write_text(json.dumps(data, indent=2))
|
||||
return sidecar_path
|
||||
|
||||
def write_post_record(self, post: dict, artist_slug: str) -> Path | None:
|
||||
"""Write a post-ONLY sidecar (no media file) for a media-less post, so
|
||||
the importer can still upsert the Post + its body — text posts often hold
|
||||
the only copy of an external <a href> link. Named `_post.json`: the
|
||||
leading underscore keeps it from colliding with a media sidecar
|
||||
(`<NN>_<stem>.json`) and from being resolved as some media file's sidecar
|
||||
by find_sidecar. Returns the path, or None when the post has no id."""
|
||||
if not str(post.get("id") or ""):
|
||||
return None
|
||||
post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post)
|
||||
post_dir.mkdir(parents=True, exist_ok=True)
|
||||
return self._write_sidecar_data(post, post_dir / "_post.json")
|
||||
|
||||
Reference in New Issue
Block a user