feat(patreon): capture media-less/text-only posts (post-only records)
Today the ingest core does `if not media: continue`, so a post with no downloadable media (a pure-text post — which often holds the ONLY copy of an external mega/gdrive/pixeldrain link) never upserts a Post. Now the native ingester emits a post-only sidecar (`_post.json`) for every media-less post, gated through the seen-ledger via a synthetic `post:<id>` key so the body is detail-fetched + recorded ONCE (not re-walked every tick); recovery bypasses the gate. Phase 3 imports these via Importer.upsert_post_record, keyed on external_post_id so it UPDATES the same Post a media import would create — never doubles, never clobbers a populated body with an empty one. - gallery_dl.py: DownloadResult.post_record_paths (default []; gallery-dl path unaffected — all constructions are keyword). - ingest_core.py: media-less branch (optional client/downloader seams via getattr; stub clients in tests skip it as before). - patreon_client.py: post_record_key(post). patreon_downloader.py: write_post_record + _write_sidecar_data refactor (shared serializer). - importer.py: upsert_post_record. download_service.py: phase-3 import loop. - tests: client/downloader/ingester (gate + recovery)/importer (no-double). Slice 0b of milestone #64. Refs FC #830. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -710,6 +710,66 @@ class Importer:
|
||||
self.session.commit()
|
||||
return ImportResult(status="refreshed", image_id=existing.id)
|
||||
|
||||
def upsert_post_record(
|
||||
self, sidecar: Path, *, artist: Artist | None = None,
|
||||
source: Source | None = None,
|
||||
) -> bool:
|
||||
"""Upsert the Post for a post-ONLY sidecar (a media-less post), so the
|
||||
artist archive includes text posts (their body + external links).
|
||||
|
||||
Reuses `_find_or_create_post` (keyed on external_post_id) so it UPDATES
|
||||
the SAME Post a media import would create — never doubles. Fields are
|
||||
FILLED, never clobbered with empty: parse_sidecar yields None for empty
|
||||
values, and a None field is left untouched (an empty feed body never
|
||||
wipes a populated one). Returns True if a Post was upserted, False if the
|
||||
sidecar was unusable (parse failure / no artist)."""
|
||||
try:
|
||||
data = json.loads(sidecar.read_text("utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("sidecar JSON is not an object")
|
||||
except Exception as exc:
|
||||
log.warning("post-record sidecar parse failed for %s: %s", sidecar, exc)
|
||||
return False
|
||||
|
||||
sd = parse_sidecar(data)
|
||||
|
||||
if artist is None:
|
||||
name = self._sidecar_artist_name(data)
|
||||
artist = self._upsert_artist(name) if name else None
|
||||
if artist is None:
|
||||
log.warning("post-record sidecar %s has no artist; skipping", sidecar)
|
||||
return False
|
||||
|
||||
if source is not None:
|
||||
src = source
|
||||
else:
|
||||
platform = sd.platform or "unknown"
|
||||
src = self._lookup_source_for_sidecar(
|
||||
artist_id=artist.id, platform=platform,
|
||||
)
|
||||
|
||||
epid = sd.external_post_id or sidecar.stem
|
||||
post = self._find_or_create_post(
|
||||
source_id=src.id if src else None,
|
||||
external_post_id=epid,
|
||||
artist_id=artist.id,
|
||||
)
|
||||
if post.artist_id is None:
|
||||
post.artist_id = artist.id
|
||||
if sd.post_url is not None:
|
||||
post.post_url = sd.post_url
|
||||
if sd.post_title is not None:
|
||||
post.post_title = sd.post_title
|
||||
if sd.post_date is not None:
|
||||
post.post_date = sd.post_date
|
||||
if sd.description is not None:
|
||||
post.description = sd.description
|
||||
if sd.attachment_count is not None:
|
||||
post.attachment_count = sd.attachment_count
|
||||
post.raw_metadata = sd.raw
|
||||
self.session.commit()
|
||||
return True
|
||||
|
||||
def attach_in_place(
|
||||
self,
|
||||
path: Path,
|
||||
|
||||
Reference in New Issue
Block a user