feat(patreon): capture media-less/text-only posts (post-only records)
Today the ingest core does `if not media: continue`, so a post with no downloadable media (a pure-text post — which often holds the ONLY copy of an external mega/gdrive/pixeldrain link) never upserts a Post. Now the native ingester emits a post-only sidecar (`_post.json`) for every media-less post, gated through the seen-ledger via a synthetic `post:<id>` key so the body is detail-fetched + recorded ONCE (not re-walked every tick); recovery bypasses the gate. Phase 3 imports these via Importer.upsert_post_record, keyed on external_post_id so it UPDATES the same Post a media import would create — never doubles, never clobbers a populated body with an empty one. - gallery_dl.py: DownloadResult.post_record_paths (default []; gallery-dl path unaffected — all constructions are keyword). - ingest_core.py: media-less branch (optional client/downloader seams via getattr; stub clients in tests skip it as before). - patreon_client.py: post_record_key(post). patreon_downloader.py: write_post_record + _write_sidecar_data refactor (shared serializer). - importer.py: upsert_post_record. download_service.py: phase-3 import loop. - tests: client/downloader/ingester (gate + recovery)/importer (no-double). Slice 0b of milestone #64. Refs FC #830. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -8,6 +8,7 @@ from PIL import Image
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import (
|
||||
Artist,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
ImportSettings,
|
||||
@@ -164,8 +165,6 @@ def test_no_artist_anywhere_skips_provenance(importer, import_layout):
|
||||
|
||||
|
||||
def test_sidecar_artist_used_when_no_folder_artist(importer, import_layout):
|
||||
from backend.app.models import Artist
|
||||
|
||||
import_root, _ = import_layout
|
||||
m = import_root / "e.jpg" # root → no folder artist
|
||||
_split(m, "v")
|
||||
@@ -183,3 +182,47 @@ def test_sidecar_artist_used_when_no_folder_artist(importer, import_layout):
|
||||
assert importer.session.execute(
|
||||
select(func.count()).select_from(Source)
|
||||
).scalar_one() == 0
|
||||
|
||||
|
||||
def test_upsert_post_record_creates_media_less_post(importer, import_layout):
|
||||
"""A post-only sidecar (no media) upserts a Post WITH its body — text posts
|
||||
are captured even when they have nothing to download."""
|
||||
import_root, _ = import_layout
|
||||
artist = Artist(name="Alice", slug="alice")
|
||||
importer.session.add(artist)
|
||||
importer.session.flush()
|
||||
sc = import_root / "Alice" / "_post.json"
|
||||
sc.parent.mkdir(parents=True, exist_ok=True)
|
||||
sc.write_text(json.dumps({
|
||||
"category": "patreon", "id": 777,
|
||||
"url": "https://patreon.com/posts/777", "title": "Text only",
|
||||
"content": "<p>links below: mega</p>",
|
||||
"published_at": "2023-08-01T00:00:00Z",
|
||||
}))
|
||||
assert importer.upsert_post_record(sc, artist=artist) is True
|
||||
post = importer.session.execute(select(Post)).scalar_one()
|
||||
assert post.external_post_id == "777"
|
||||
assert post.post_title == "Text only"
|
||||
assert post.description == "<p>links below: mega</p>"
|
||||
assert post.post_url == "https://patreon.com/posts/777"
|
||||
assert post.post_date is not None
|
||||
|
||||
|
||||
def test_upsert_post_record_idempotent_no_double(importer, import_layout):
|
||||
"""Re-running upsert_post_record UPDATES the same Post (keyed on
|
||||
external_post_id) — never doubles."""
|
||||
import_root, _ = import_layout
|
||||
artist = Artist(name="Bob", slug="bob")
|
||||
importer.session.add(artist)
|
||||
importer.session.flush()
|
||||
sc = import_root / "Bob" / "_post.json"
|
||||
sc.parent.mkdir(parents=True, exist_ok=True)
|
||||
sc.write_text(json.dumps({
|
||||
"category": "patreon", "id": 888, "title": "T",
|
||||
"content": "<p>body</p>", "url": "https://patreon.com/posts/888",
|
||||
}))
|
||||
assert importer.upsert_post_record(sc, artist=artist) is True
|
||||
assert importer.upsert_post_record(sc, artist=artist) is True
|
||||
assert importer.session.execute(
|
||||
select(func.count()).select_from(Post)
|
||||
).scalar_one() == 1
|
||||
|
||||
Reference in New Issue
Block a user