fix(importer): Source = one per (artist, platform), not one per post — filesystem importer's sidecar paths now reuse the artist's existing subscription Source (or create one synthetic anchor with enabled=False) instead of fabricating a new Source per post URL. Alembic 0022 consolidates existing per-post Sources to canonical (prefers campaign URL; falls back to sidecar:<platform>:<slug>) and re-parents Posts + ImageProvenance, merging Post collisions.
Operator-flagged 2026-05-26: Atole artist detail page showed 406 Sources where 1 was right. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,181 @@
|
||||
"""source-collapse: one Source per (artist, platform) — consolidate junk per-post Sources
|
||||
|
||||
Revision ID: 0022
|
||||
Revises: 0021
|
||||
Create Date: 2026-05-26
|
||||
|
||||
Closes the operator-flagged 2026-05-26 issue where the filesystem importer
|
||||
called _find_or_create_source(url=sd.post_url), creating one Source row per
|
||||
imported post URL. Operator's Atole artist had 406 Source rows where there
|
||||
should have been 1 (the /cw/Atole subscription Source).
|
||||
|
||||
Source represents a subscription feed (one per artist+platform — the
|
||||
gallery-dl URL polled by the FC-3 downloader). Posts hang off it. The
|
||||
filesystem importer was misusing Source as a per-post key.
|
||||
|
||||
Migration steps per (artist_id, platform) group with >1 Source:
|
||||
1. Pick canonical — prefer a URL NOT matching '/posts/<id>$' (real
|
||||
campaign URL like /cw/Atole); else min(id).
|
||||
2. Reparent Posts and ImageProvenance off the other Sources onto canonical.
|
||||
3. Merge any Posts that collide on (canonical_source_id, external_post_id)
|
||||
by repointing ImageProvenance + ImageRecord.primary_post_id to the
|
||||
earliest Post, dedupe ImageProvenance, delete the loser Posts.
|
||||
4. Delete the orphan Source rows.
|
||||
5. If the canonical Source's URL still looks like a per-post URL (no
|
||||
campaign URL existed among candidates), rewrite it to
|
||||
'sidecar:<platform>:<artist_slug>' so the artist detail page shows
|
||||
something readable.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
revision: str = "0022"
|
||||
down_revision: Union[str, None] = "0021"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
_POST_URL_RE = r"/posts/[^/]+$"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
# Find (artist_id, platform) groups with > 1 Source row.
|
||||
groups = conn.execute(text("""
|
||||
SELECT artist_id, platform
|
||||
FROM source
|
||||
GROUP BY artist_id, platform
|
||||
HAVING COUNT(*) > 1
|
||||
""")).fetchall()
|
||||
|
||||
for artist_id, platform in groups:
|
||||
rows = conn.execute(
|
||||
text("""
|
||||
SELECT id, url FROM source
|
||||
WHERE artist_id = :a AND platform = :p
|
||||
ORDER BY id ASC
|
||||
"""),
|
||||
{"a": artist_id, "p": platform},
|
||||
).fetchall()
|
||||
|
||||
# Canonical: first row whose URL doesn't look like a per-post URL;
|
||||
# else min(id).
|
||||
canonical_id = None
|
||||
for sid, url in rows:
|
||||
if not _matches_post_url(url):
|
||||
canonical_id = sid
|
||||
break
|
||||
if canonical_id is None:
|
||||
canonical_id = rows[0][0]
|
||||
|
||||
other_ids = [sid for sid, _ in rows if sid != canonical_id]
|
||||
if not other_ids:
|
||||
continue
|
||||
|
||||
# Reparent Posts off the other Sources.
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE post SET source_id = :canonical
|
||||
WHERE source_id = ANY(:others)
|
||||
"""),
|
||||
{"canonical": canonical_id, "others": other_ids},
|
||||
)
|
||||
# Reparent ImageProvenance.source_id similarly (denormalized FK).
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE image_provenance SET source_id = :canonical
|
||||
WHERE source_id = ANY(:others)
|
||||
"""),
|
||||
{"canonical": canonical_id, "others": other_ids},
|
||||
)
|
||||
|
||||
# Merge any Posts colliding on (canonical, external_post_id) after
|
||||
# reparent. In practice within a single (artist, platform) group
|
||||
# these should be rare — each gallery-dl post has a unique id — but
|
||||
# safe to handle.
|
||||
post_collisions = conn.execute(
|
||||
text("""
|
||||
SELECT external_post_id, ARRAY_AGG(id ORDER BY id) AS ids
|
||||
FROM post
|
||||
WHERE source_id = :canonical
|
||||
GROUP BY external_post_id
|
||||
HAVING COUNT(*) > 1
|
||||
"""),
|
||||
{"canonical": canonical_id},
|
||||
).fetchall()
|
||||
for _epid, post_ids in post_collisions:
|
||||
keep = post_ids[0]
|
||||
drops = post_ids[1:]
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE image_provenance SET post_id = :keep
|
||||
WHERE post_id = ANY(:drops)
|
||||
"""),
|
||||
{"keep": keep, "drops": drops},
|
||||
)
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE image_record SET primary_post_id = :keep
|
||||
WHERE primary_post_id = ANY(:drops)
|
||||
"""),
|
||||
{"keep": keep, "drops": drops},
|
||||
)
|
||||
# Repointed provenance rows may now collide on
|
||||
# uq_image_provenance_image_post (alembic 0021). Same dedupe
|
||||
# pattern: keep min(id) per (image_record_id, post_id).
|
||||
conn.execute(text("""
|
||||
DELETE FROM image_provenance ip1
|
||||
USING image_provenance ip2
|
||||
WHERE ip1.image_record_id = ip2.image_record_id
|
||||
AND ip1.post_id = ip2.post_id
|
||||
AND ip1.id > ip2.id
|
||||
"""))
|
||||
conn.execute(
|
||||
text("DELETE FROM post WHERE id = ANY(:drops)"),
|
||||
{"drops": drops},
|
||||
)
|
||||
|
||||
# Drop the orphan Sources.
|
||||
conn.execute(
|
||||
text("DELETE FROM source WHERE id = ANY(:others)"),
|
||||
{"others": other_ids},
|
||||
)
|
||||
|
||||
# If the canonical's URL still looks per-post (no campaign URL
|
||||
# existed among the candidates), rewrite to a synthetic anchor so
|
||||
# the artist detail page renders something readable.
|
||||
canonical_url = conn.execute(
|
||||
text("SELECT url FROM source WHERE id = :id"),
|
||||
{"id": canonical_id},
|
||||
).scalar_one()
|
||||
if _matches_post_url(canonical_url):
|
||||
slug = conn.execute(
|
||||
text("SELECT slug FROM artist WHERE id = :id"),
|
||||
{"id": artist_id},
|
||||
).scalar_one()
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE source
|
||||
SET url = :new_url, enabled = false
|
||||
WHERE id = :id
|
||||
"""),
|
||||
{
|
||||
"id": canonical_id,
|
||||
"new_url": f"sidecar:{platform}:{slug}",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Lossy migration — orphan Sources deleted, Posts reparented, Posts
|
||||
# merged. No safe downgrade. If you need to roll back the schema
|
||||
# invariant, fork from 0021 and re-run filesystem imports.
|
||||
pass
|
||||
|
||||
|
||||
def _matches_post_url(url: str) -> bool:
|
||||
"""True if url ends with /posts/<token> (gallery-dl-style per-post URL)."""
|
||||
import re
|
||||
return bool(re.search(_POST_URL_RE, url or ""))
|
||||
Reference in New Issue
Block a user