refactor(ingest): post-first — post-record is the sole body writer on the native path (#856)
Milestone #67 step 2. On the native core ingester the Post becomes the single authoritative record for body/links/metadata, captured once per post by the post-record; the per-media import only links image provenance + localization. Before: every per-media sidecar carried the full post body, so a post with N images wrote the body N+1 times (post-record + N media) — redundant on disk and a divergence risk (#753). gallery-dl is unchanged (its sidecar is still the only body source). - patreon_downloader: the per-media sidecar is now minimal — {category, id, source_url} only, no body. `_write_sidecar_data(minimal=True)` skips the body resolution + detail-fetch (the post-record, written first in the walk, already did it). Body no longer duplicated next to each image. - importer: new per-instance `post_first` flag (Importer is per-task). When set, `_apply_sidecar` still writes source_filehash + provenance + primary_post_id but SKIPS `_apply_post_fields` (the post-record owns body/links/raw_metadata, so applying a body-less sidecar would clobber raw_metadata + re-sync links off empty data). Default False keeps gallery-dl writing post fields. - download_service: `_phase3_persist` sets importer.post_first = uses_native_ingester(platform) — the future-proof seam, so a platform migrating onto the native core flips to post-first automatically (step 3). Media imports before post-records but both unify on external_post_id, so the post ends with its body either way. Tests: per-media sidecar is minimal + never hits the detail fetcher; attach post_first=True links provenance/localization but writes no post body/title; post_first=False (gallery-dl) still applies them. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -231,6 +231,78 @@ def test_attach_in_place_duplicate_attachment_links_both_posts(importer, db_sync
|
||||
assert set(linked) == {"301", "302"} # one row per post, both present
|
||||
|
||||
|
||||
def test_attach_in_place_post_first_skips_post_fields(importer, db_sync):
|
||||
"""Post-first (#856, milestone #67): with post_first set (native path), the
|
||||
per-media import links provenance + localization (source_filehash) but does
|
||||
NOT write the post body/title — the post-record owns those. Guards the import
|
||||
side even if a body sneaks into the sidecar."""
|
||||
from backend.app.models import ImageProvenance, Post
|
||||
|
||||
importer.post_first = True
|
||||
images_root = importer.images_root
|
||||
artist = Artist(name="Pat", slug="pat")
|
||||
db_sync.add(artist)
|
||||
db_sync.flush()
|
||||
|
||||
target = images_root / "pat" / "patreon" / "post1" / "img.jpg"
|
||||
_make_jpg(target)
|
||||
cdn = "https://cdn.patreon.com/" + "a" * 32 + "/x.jpg"
|
||||
target.with_suffix(target.suffix + ".json").write_text(
|
||||
'{"category": "patreon", "id": "900", "title": "Should NOT apply", '
|
||||
'"content": "<p>body should NOT apply</p>", '
|
||||
f'"source_url": "{cdn}"}}'
|
||||
)
|
||||
|
||||
result = importer.attach_in_place(target, artist=artist)
|
||||
assert result.status == "imported"
|
||||
|
||||
# Image-specific work STILL happens: localization filehash + provenance link.
|
||||
rec = db_sync.execute(
|
||||
select(ImageRecord).where(ImageRecord.id == result.image_id)
|
||||
).scalar_one()
|
||||
assert rec.source_filehash == "a" * 32
|
||||
assert rec.primary_post_id is not None
|
||||
|
||||
post = db_sync.get(Post, rec.primary_post_id)
|
||||
assert post.external_post_id == "900"
|
||||
# Post-first: body/title NOT written by the media path (post-record owns them).
|
||||
assert post.description is None
|
||||
assert post.post_title is None
|
||||
# Provenance link exists regardless of post_first.
|
||||
prov = db_sync.execute(
|
||||
select(ImageProvenance).where(
|
||||
ImageProvenance.image_record_id == result.image_id
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(prov) == 1
|
||||
|
||||
|
||||
def test_attach_in_place_post_first_false_applies_post_fields(importer, db_sync):
|
||||
"""Default (gallery-dl path): the sidecar IS the body source, so post fields
|
||||
ARE applied. Guards that the post_first flag actually gates this — not a no-op."""
|
||||
from backend.app.models import Post
|
||||
|
||||
assert importer.post_first is False # default
|
||||
images_root = importer.images_root
|
||||
artist = Artist(name="Gal", slug="gal")
|
||||
db_sync.add(artist)
|
||||
db_sync.flush()
|
||||
|
||||
target = images_root / "gal" / "patreon" / "post1" / "img.jpg"
|
||||
_make_jpg(target)
|
||||
target.with_suffix(target.suffix + ".json").write_text(
|
||||
'{"category": "patreon", "id": "901", "title": "Applies", '
|
||||
'"content": "<p>body applies</p>"}'
|
||||
)
|
||||
|
||||
result = importer.attach_in_place(target, artist=artist)
|
||||
assert result.status == "imported"
|
||||
rec = db_sync.get(ImageRecord, result.image_id)
|
||||
post = db_sync.get(Post, rec.primary_post_id)
|
||||
assert post.description == "<p>body applies</p>"
|
||||
assert post.post_title == "Applies"
|
||||
|
||||
|
||||
def test_attach_in_place_invalid_image_returns_skipped(importer):
|
||||
images_root = importer.images_root
|
||||
bad = images_root / "fred" / "bad.jpg"
|
||||
|
||||
Reference in New Issue
Block a user