"""source-collapse: one Source per (artist, platform) — consolidate junk per-post Sources Revision ID: 0022 Revises: 0021 Create Date: 2026-05-26 Closes the operator-flagged 2026-05-26 issue where the filesystem importer called _find_or_create_source(url=sd.post_url), creating one Source row per imported post URL. Operator's Atole artist had 406 Source rows where there should have been 1 (the /cw/Atole subscription Source). Source represents a subscription feed (one per artist+platform — the gallery-dl URL polled by the FC-3 downloader). Posts hang off it. The filesystem importer was misusing Source as a per-post key. Migration steps per (artist_id, platform) group with >1 Source: 1. Pick canonical — prefer a URL NOT matching '/posts/$' (real campaign URL like /cw/Atole); else min(id). 2. PRE-merge any Posts under non-canonical sources whose external_post_id ALREADY exists under the canonical source. (Same gallery-dl post imported via two different sidecar paths can plant two Post rows with identical external_post_id under different Sources for the same artist.) Repoint ImageProvenance + ImageRecord.primary_post_id to the canonical-side Post, dedupe ImageProvenance against alembic 0021's uq, then delete the non-canonical-side Post. This MUST happen before step 3 — Postgres fires uq_post_source_external_id row-by-row during the bulk UPDATE and the merge-after-reparent ordering 500s on first collision (operator-hit during v26.05.26.1 deploy, 2026-05-26). 3. Reparent remaining Posts onto canonical (no collisions possible now). 4. Reparent ImageProvenance.source_id off the non-canonical sources. 5. Delete the orphan Source rows. 6. If the canonical Source's URL still looks like a per-post URL (no campaign URL existed among candidates), rewrite it to 'sidecar::' so the artist detail page shows something readable. """ from typing import Sequence, Union from alembic import op from sqlalchemy import text revision: str = "0022" down_revision: Union[str, None] = "0021" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None _POST_URL_RE = r"/posts/[^/]+$" def upgrade() -> None: conn = op.get_bind() # Find (artist_id, platform) groups with > 1 Source row. groups = conn.execute(text(""" SELECT artist_id, platform FROM source GROUP BY artist_id, platform HAVING COUNT(*) > 1 """)).fetchall() for artist_id, platform in groups: rows = conn.execute( text(""" SELECT id, url FROM source WHERE artist_id = :a AND platform = :p ORDER BY id ASC """), {"a": artist_id, "p": platform}, ).fetchall() # Canonical: first row whose URL doesn't look like a per-post URL; # else min(id). canonical_id = None for sid, url in rows: if not _matches_post_url(url): canonical_id = sid break if canonical_id is None: canonical_id = rows[0][0] other_ids = [sid for sid, _ in rows if sid != canonical_id] if not other_ids: continue # STEP 2: PRE-merge colliding Posts BEFORE the bulk reparent. # Find pairs (keep, drop) where: # keep = a Post under the canonical source # drop = a Post under one of the non-canonical sources whose # external_post_id equals keep.external_post_id # If we let the bulk UPDATE try to repoint drop onto canonical, # uq_post_source_external_id fires row-by-row and aborts the # whole migration. colliding = conn.execute( text(""" SELECT keep.id AS keep_id, drop_.id AS drop_id FROM post AS keep JOIN post AS drop_ ON drop_.external_post_id = keep.external_post_id AND drop_.id != keep.id WHERE keep.source_id = :canonical AND drop_.source_id = ANY(:others) """), {"canonical": canonical_id, "others": other_ids}, ).fetchall() for keep_id, drop_id in colliding: conn.execute( text(""" UPDATE image_provenance SET post_id = :keep WHERE post_id = :drop_ """), {"keep": keep_id, "drop_": drop_id}, ) conn.execute( text(""" UPDATE image_record SET primary_post_id = :keep WHERE primary_post_id = :drop_ """), {"keep": keep_id, "drop_": drop_id}, ) # Repointed provenance may now collide on # uq_image_provenance_image_post (alembic 0021). Dedupe: # keep min(id) per (image_record_id, post_id). conn.execute(text(""" DELETE FROM image_provenance ip1 USING image_provenance ip2 WHERE ip1.image_record_id = ip2.image_record_id AND ip1.post_id = ip2.post_id AND ip1.id > ip2.id """)) conn.execute( text("DELETE FROM post WHERE id = :drop_"), {"drop_": drop_id}, ) # STEP 3: Bulk reparent the remaining Posts off the other # Sources. After step 2, no collisions on # (canonical, external_post_id) are possible. conn.execute( text(""" UPDATE post SET source_id = :canonical WHERE source_id = ANY(:others) """), {"canonical": canonical_id, "others": other_ids}, ) # STEP 4: Reparent ImageProvenance.source_id (denormalized FK). # No UNIQUE on source_id; safe bulk update. conn.execute( text(""" UPDATE image_provenance SET source_id = :canonical WHERE source_id = ANY(:others) """), {"canonical": canonical_id, "others": other_ids}, ) # STEP 5: Drop the orphan Sources. conn.execute( text("DELETE FROM source WHERE id = ANY(:others)"), {"others": other_ids}, ) # If the canonical's URL still looks per-post (no campaign URL # existed among the candidates), rewrite to a synthetic anchor so # the artist detail page renders something readable. canonical_url = conn.execute( text("SELECT url FROM source WHERE id = :id"), {"id": canonical_id}, ).scalar_one() if _matches_post_url(canonical_url): slug = conn.execute( text("SELECT slug FROM artist WHERE id = :id"), {"id": artist_id}, ).scalar_one() conn.execute( text(""" UPDATE source SET url = :new_url, enabled = false WHERE id = :id """), { "id": canonical_id, "new_url": f"sidecar:{platform}:{slug}", }, ) def downgrade() -> None: # Lossy migration — orphan Sources deleted, Posts reparented, Posts # merged. No safe downgrade. If you need to roll back the schema # invariant, fork from 0021 and re-run filesystem imports. pass def _matches_post_url(url: str) -> bool: """True if url ends with /posts/ (gallery-dl-style per-post URL).""" import re return bool(re.search(_POST_URL_RE, url or ""))