diff --git a/alembic/versions/0022_source_per_artist_platform.py b/alembic/versions/0022_source_per_artist_platform.py index eaee97f..2b2853e 100644 --- a/alembic/versions/0022_source_per_artist_platform.py +++ b/alembic/versions/0022_source_per_artist_platform.py @@ -16,12 +16,21 @@ filesystem importer was misusing Source as a per-post key. Migration steps per (artist_id, platform) group with >1 Source: 1. Pick canonical — prefer a URL NOT matching '/posts/$' (real campaign URL like /cw/Atole); else min(id). - 2. Reparent Posts and ImageProvenance off the other Sources onto canonical. - 3. Merge any Posts that collide on (canonical_source_id, external_post_id) - by repointing ImageProvenance + ImageRecord.primary_post_id to the - earliest Post, dedupe ImageProvenance, delete the loser Posts. - 4. Delete the orphan Source rows. - 5. If the canonical Source's URL still looks like a per-post URL (no + 2. PRE-merge any Posts under non-canonical sources whose + external_post_id ALREADY exists under the canonical source. (Same + gallery-dl post imported via two different sidecar paths can plant + two Post rows with identical external_post_id under different + Sources for the same artist.) Repoint ImageProvenance + + ImageRecord.primary_post_id to the canonical-side Post, dedupe + ImageProvenance against alembic 0021's uq, then delete the + non-canonical-side Post. This MUST happen before step 3 — Postgres + fires uq_post_source_external_id row-by-row during the bulk UPDATE + and the merge-after-reparent ordering 500s on first collision + (operator-hit during v26.05.26.1 deploy, 2026-05-26). + 3. Reparent remaining Posts onto canonical (no collisions possible now). + 4. Reparent ImageProvenance.source_id off the non-canonical sources. + 5. Delete the orphan Source rows. + 6. If the canonical Source's URL still looks like a per-post URL (no campaign URL existed among candidates), rewrite it to 'sidecar::' so the artist detail page shows something readable. @@ -74,57 +83,44 @@ def upgrade() -> None: if not other_ids: continue - # Reparent Posts off the other Sources. - conn.execute( + # STEP 2: PRE-merge colliding Posts BEFORE the bulk reparent. + # Find pairs (keep, drop) where: + # keep = a Post under the canonical source + # drop = a Post under one of the non-canonical sources whose + # external_post_id equals keep.external_post_id + # If we let the bulk UPDATE try to repoint drop onto canonical, + # uq_post_source_external_id fires row-by-row and aborts the + # whole migration. + colliding = conn.execute( text(""" - UPDATE post SET source_id = :canonical - WHERE source_id = ANY(:others) + SELECT keep.id AS keep_id, drop_.id AS drop_id + FROM post AS keep + JOIN post AS drop_ + ON drop_.external_post_id = keep.external_post_id + AND drop_.id != keep.id + WHERE keep.source_id = :canonical + AND drop_.source_id = ANY(:others) """), {"canonical": canonical_id, "others": other_ids}, - ) - # Reparent ImageProvenance.source_id similarly (denormalized FK). - conn.execute( - text(""" - UPDATE image_provenance SET source_id = :canonical - WHERE source_id = ANY(:others) - """), - {"canonical": canonical_id, "others": other_ids}, - ) - - # Merge any Posts colliding on (canonical, external_post_id) after - # reparent. In practice within a single (artist, platform) group - # these should be rare — each gallery-dl post has a unique id — but - # safe to handle. - post_collisions = conn.execute( - text(""" - SELECT external_post_id, ARRAY_AGG(id ORDER BY id) AS ids - FROM post - WHERE source_id = :canonical - GROUP BY external_post_id - HAVING COUNT(*) > 1 - """), - {"canonical": canonical_id}, ).fetchall() - for _epid, post_ids in post_collisions: - keep = post_ids[0] - drops = post_ids[1:] + for keep_id, drop_id in colliding: conn.execute( text(""" UPDATE image_provenance SET post_id = :keep - WHERE post_id = ANY(:drops) + WHERE post_id = :drop_ """), - {"keep": keep, "drops": drops}, + {"keep": keep_id, "drop_": drop_id}, ) conn.execute( text(""" UPDATE image_record SET primary_post_id = :keep - WHERE primary_post_id = ANY(:drops) + WHERE primary_post_id = :drop_ """), - {"keep": keep, "drops": drops}, + {"keep": keep_id, "drop_": drop_id}, ) - # Repointed provenance rows may now collide on - # uq_image_provenance_image_post (alembic 0021). Same dedupe - # pattern: keep min(id) per (image_record_id, post_id). + # Repointed provenance may now collide on + # uq_image_provenance_image_post (alembic 0021). Dedupe: + # keep min(id) per (image_record_id, post_id). conn.execute(text(""" DELETE FROM image_provenance ip1 USING image_provenance ip2 @@ -133,11 +129,32 @@ def upgrade() -> None: AND ip1.id > ip2.id """)) conn.execute( - text("DELETE FROM post WHERE id = ANY(:drops)"), - {"drops": drops}, + text("DELETE FROM post WHERE id = :drop_"), + {"drop_": drop_id}, ) - # Drop the orphan Sources. + # STEP 3: Bulk reparent the remaining Posts off the other + # Sources. After step 2, no collisions on + # (canonical, external_post_id) are possible. + conn.execute( + text(""" + UPDATE post SET source_id = :canonical + WHERE source_id = ANY(:others) + """), + {"canonical": canonical_id, "others": other_ids}, + ) + + # STEP 4: Reparent ImageProvenance.source_id (denormalized FK). + # No UNIQUE on source_id; safe bulk update. + conn.execute( + text(""" + UPDATE image_provenance SET source_id = :canonical + WHERE source_id = ANY(:others) + """), + {"canonical": canonical_id, "others": other_ids}, + ) + + # STEP 5: Drop the orphan Sources. conn.execute( text("DELETE FROM source WHERE id = ANY(:others)"), {"others": other_ids},