fix(posts): link duplicate items to every post + prune bare shells
The native Patreon backfill flooded the feed with bare 'Post <id>' shells (1589 for Anduo). Root cause: PostAttachment.sha256 was GLOBALLY unique, so a non-art file reused across posts only ever linked to the first one, and _capture_attachment created the Post before that dedup check — leaving later posts with no image and no attachment. Duplicate IMAGES had the mirror gap: attach_in_place returned duplicate_hash/duplicate_phash before _apply_sidecar, so the second post got no provenance row, and the feed only rendered via primary_post_id (one post per image). Operator requirement: a duplicate item must show on EVERY post it appears in. Unify the fix as link-not-suppress: - importer: on duplicate_hash / duplicate_phash(larger_exists), append an image_provenance row for the new post (keep primary on the first). Both the download path (attach_in_place) and the filesystem path (_import_media). - post_feed_service: render thumbnails by image_provenance UNION primary_post_id, so a cross-posted image shows on every post (and legacy primary-only images still show). - PostAttachment: per-post uniqueness — drop UNIQUE(sha256), add partial UNIQUE(post_id, sha256) + partial UNIQUE(sha256) WHERE post_id IS NULL (migration 0043); _capture_attachment dedups per-(post,sha) over the shared sha-addressed blob, so no post is left bare. - cleanup: new prune-bare-posts maintenance action (cleanup_service _bare_post_conditions shared by preview/count/delete per preview/apply parity; admin endpoint; PostMaintenanceCard). Deletes posts with zero image links (primary or provenance) AND zero attachments. Run after the feed fix so a hidden provenance link spares the post instead of deleting it. Tests: dup image shows on both posts; dup attachment shows on both posts; feed renders provenance-linked duplicates; prune-bare delete-path == preview. Operator redeploys (migration 0043) then runs the prune to clear the shells. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -8,7 +8,15 @@ re-read ORM attributes after a service mutates and re-fetches.
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import Artist, ImageRecord, Tag, TagKind
|
||||
from backend.app.models import (
|
||||
Artist,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
Post,
|
||||
PostAttachment,
|
||||
Tag,
|
||||
TagKind,
|
||||
)
|
||||
from backend.app.models.series_chapter import SeriesChapter
|
||||
from backend.app.models.series_page import SeriesPage
|
||||
from backend.app.models.tag import image_tag
|
||||
@@ -399,6 +407,39 @@ def test_prune_unused_tags_commit_spares_fandom_and_chaptered_series(
|
||||
assert "GenuinelyUnused" not in surviving
|
||||
|
||||
|
||||
def test_prune_bare_posts_spares_linked_and_deletes_bare(db_sync, tmp_path):
|
||||
# The LIVE delete (not just the preview) shares _bare_post_conditions, so a
|
||||
# post is spared if it has ANY link — a primary image, a provenance (cross-
|
||||
# posted/duplicate) image, or an attachment — and only the truly-bare shell
|
||||
# is deleted (the empty-post flood, operator-flagged 2026-06-08).
|
||||
a = _make_artist(db_sync, slug="pb")
|
||||
img = _make_image(db_sync, artist=a, path=str(tmp_path / "x.jpg"), sha256="a" * 64)
|
||||
|
||||
bare = Post(artist_id=a.id, external_post_id="bare1")
|
||||
has_primary = Post(artist_id=a.id, external_post_id="prim")
|
||||
has_prov = Post(artist_id=a.id, external_post_id="prov")
|
||||
has_att = Post(artist_id=a.id, external_post_id="att")
|
||||
db_sync.add_all([bare, has_primary, has_prov, has_att])
|
||||
db_sync.flush()
|
||||
|
||||
img.primary_post_id = has_primary.id
|
||||
db_sync.add(ImageProvenance(image_record_id=img.id, post_id=has_prov.id))
|
||||
db_sync.add(PostAttachment(
|
||||
post_id=has_att.id, artist_id=a.id, sha256="b" * 64,
|
||||
path="/store/b", original_filename="f.pdf", ext=".pdf", size_bytes=1,
|
||||
))
|
||||
db_sync.commit()
|
||||
|
||||
# dry-run count agrees with the delete: only the 1 truly-bare post.
|
||||
assert cleanup_service.prune_bare_posts(db_sync, dry_run=True)["count"] == 1
|
||||
result = cleanup_service.prune_bare_posts(db_sync, dry_run=False)
|
||||
assert result["deleted"] == 1
|
||||
|
||||
surviving = db_sync.execute(select(Post.external_post_id)).scalars().all()
|
||||
assert "bare1" not in surviving
|
||||
assert set(surviving) == {"prim", "prov", "att"}
|
||||
|
||||
|
||||
# --- reset_content_tagging ------------------------------------------
|
||||
|
||||
|
||||
|
||||
@@ -150,6 +150,87 @@ def test_attach_in_place_phash_smaller_skips_without_modifying_disk(
|
||||
assert smaller.exists()
|
||||
|
||||
|
||||
def test_attach_in_place_duplicate_image_links_both_posts(importer, db_sync):
|
||||
"""A byte-identical image appearing in a SECOND post must link to BOTH posts
|
||||
(enrich-on-duplicate) so it shows on every post — not be silently dropped,
|
||||
which left the second post a bare shell. Operator-flagged 2026-06-08."""
|
||||
from backend.app.models import ImageProvenance, Post
|
||||
|
||||
images_root = importer.images_root
|
||||
artist = Artist(name="Hana", slug="hana")
|
||||
db_sync.add(artist)
|
||||
db_sync.flush()
|
||||
|
||||
first = images_root / "hana" / "patreon" / "post_a" / "img.jpg"
|
||||
bytes_ = _make_jpg(first, color=(120, 40, 90))
|
||||
first.with_suffix(first.suffix + ".json").write_text(
|
||||
'{"category": "patreon", "id": "111", "title": "Post 111"}'
|
||||
)
|
||||
r1 = importer.attach_in_place(first, artist=artist)
|
||||
assert r1.status == "imported"
|
||||
|
||||
second = images_root / "hana" / "patreon" / "post_b" / "img.jpg"
|
||||
second.parent.mkdir(parents=True, exist_ok=True)
|
||||
second.write_bytes(bytes_) # identical bytes → sha256 duplicate
|
||||
second.with_suffix(second.suffix + ".json").write_text(
|
||||
'{"category": "patreon", "id": "222", "title": "Post 222"}'
|
||||
)
|
||||
r2 = importer.attach_in_place(second, artist=artist)
|
||||
assert r2.status == "skipped"
|
||||
assert r2.skip_reason.value == "duplicate_hash"
|
||||
assert r2.image_id == r1.image_id
|
||||
|
||||
linked = db_sync.execute(
|
||||
select(Post.external_post_id)
|
||||
.join(ImageProvenance, ImageProvenance.post_id == Post.id)
|
||||
.where(ImageProvenance.image_record_id == r1.image_id)
|
||||
).scalars().all()
|
||||
assert set(linked) == {"111", "222"} # shows on BOTH posts
|
||||
|
||||
primary = db_sync.execute(
|
||||
select(Post.external_post_id)
|
||||
.join(ImageRecord, ImageRecord.primary_post_id == Post.id)
|
||||
.where(ImageRecord.id == r1.image_id)
|
||||
).scalar_one()
|
||||
assert primary == "111" # primary stays on the first post
|
||||
|
||||
|
||||
def test_attach_in_place_duplicate_attachment_links_both_posts(importer, db_sync):
|
||||
"""The same non-art file attached to two posts gets a row on EACH post over a
|
||||
single sha-addressed blob — neither post is left bare. Operator-flagged
|
||||
2026-06-08 (1589 empty Anduo shells from globally-unique attachment sha)."""
|
||||
from backend.app.models import Post, PostAttachment
|
||||
|
||||
images_root = importer.images_root
|
||||
artist = Artist(name="Iris", slug="iris")
|
||||
db_sync.add(artist)
|
||||
db_sync.flush()
|
||||
|
||||
payload = b"the same readme creator attaches to every post"
|
||||
a = images_root / "iris" / "patreon" / "post_a" / "readme.txt"
|
||||
a.parent.mkdir(parents=True, exist_ok=True)
|
||||
a.write_bytes(payload)
|
||||
a.with_suffix(a.suffix + ".json").write_text(
|
||||
'{"category": "patreon", "id": "301", "title": "Post 301"}'
|
||||
)
|
||||
assert importer.attach_in_place(a, artist=artist).status == "attached"
|
||||
|
||||
b = images_root / "iris" / "patreon" / "post_b" / "readme.txt"
|
||||
b.parent.mkdir(parents=True, exist_ok=True)
|
||||
b.write_bytes(payload)
|
||||
b.with_suffix(b.suffix + ".json").write_text(
|
||||
'{"category": "patreon", "id": "302", "title": "Post 302"}'
|
||||
)
|
||||
assert importer.attach_in_place(b, artist=artist).status == "attached"
|
||||
|
||||
linked = db_sync.execute(
|
||||
select(Post.external_post_id)
|
||||
.join(PostAttachment, PostAttachment.post_id == Post.id)
|
||||
.where(PostAttachment.original_filename == "readme.txt")
|
||||
).scalars().all()
|
||||
assert set(linked) == {"301", "302"} # one row per post, both present
|
||||
|
||||
|
||||
def test_attach_in_place_invalid_image_returns_skipped(importer):
|
||||
images_root = importer.images_root
|
||||
bad = images_root / "fred" / "bad.jpg"
|
||||
|
||||
@@ -11,6 +11,7 @@ from sqlalchemy import select
|
||||
|
||||
from backend.app.models import (
|
||||
Artist,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
Post,
|
||||
PostAttachment,
|
||||
@@ -283,6 +284,42 @@ async def test_scroll_thumbnails_show_first_six_with_overflow_count(db):
|
||||
assert t["mime"] == "image/jpeg"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_thumbnails_include_provenance_linked_duplicates(db):
|
||||
"""A cross-posted/duplicate image is linked to a second post via
|
||||
image_provenance (its primary_post_id stays on the first post). The feed
|
||||
renders by provenance UNION primary, so the image shows on BOTH posts —
|
||||
not just its primary. Operator requirement 2026-06-08."""
|
||||
artist = await _seed_artist(db, "alice-prov")
|
||||
src = await _seed_source(db, artist.id, "patreon", "https://p/alice-prov")
|
||||
primary_post = await _seed_post(
|
||||
db, src.id, external_id="PRIMARY", post_date=datetime.now(UTC),
|
||||
)
|
||||
other_post = await _seed_post(
|
||||
db, src.id, external_id="OTHER",
|
||||
post_date=datetime.now(UTC) + timedelta(minutes=1),
|
||||
)
|
||||
img = ImageRecord(
|
||||
path="/images/dup.jpg", sha256="d" * 64, size_bytes=10,
|
||||
mime="image/jpeg", width=10, height=10, origin="downloaded",
|
||||
primary_post_id=primary_post.id, artist_id=artist.id,
|
||||
)
|
||||
db.add(img)
|
||||
await db.flush()
|
||||
# The duplicate links to the OTHER post via provenance only.
|
||||
db.add(ImageProvenance(image_record_id=img.id, post_id=other_post.id))
|
||||
await db.commit()
|
||||
|
||||
page = await PostFeedService(db).scroll(cursor=None, limit=10)
|
||||
by_ext = {it["external_post_id"]: it for it in page["items"]}
|
||||
assert len(by_ext["PRIMARY"]["thumbnails"]) == 1
|
||||
assert len(by_ext["OTHER"]["thumbnails"]) == 1 # shown via provenance
|
||||
assert (
|
||||
by_ext["PRIMARY"]["thumbnails"][0]["image_id"]
|
||||
== by_ext["OTHER"]["thumbnails"][0]["image_id"]
|
||||
)
|
||||
|
||||
|
||||
# --- Attachments composition ---------------------------------------------
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user