feat(migration): bare tag names; fandom_id authoritative
Three-phase Alembic migration j26042101 strips 'kind:' prefix from every non-user tag, extracts character '(Fandom)' suffix into fandom_id with auto-merge on duplicates, and swaps UNIQUE(name) for four partial indices keyed on (name, kind) with special handling for the (character, fandom_id) combination. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,277 @@
|
||||
"""Strip kind: prefix from tag names, extract character (Fandom) suffix into
|
||||
fandom_id, and swap UNIQUE(name) for four partial unique indices keyed on
|
||||
(name, kind) with special handling for characters.
|
||||
|
||||
Revision ID: j26042101
|
||||
Revises: i26041901
|
||||
Create Date: 2026-04-21
|
||||
|
||||
Three phases run in one transaction:
|
||||
|
||||
Phase 1 — Strip kind prefix from tag.name for every non-user tag.
|
||||
'character:Ruby Rose (RWBY)' -> 'Ruby Rose (RWBY)'
|
||||
'artist:Eric Canete' -> 'Eric Canete'
|
||||
'post:patreon:eric:12345' -> 'patreon:eric:12345'
|
||||
User tags (kind='user', already bare) are untouched.
|
||||
|
||||
Phase 2 — For every kind='character' tag whose name still matches
|
||||
'Name (Fandom)' after phase 1, ensure the fandom tag exists,
|
||||
update the character's fandom_id, and strip the suffix from its name.
|
||||
True duplicates (same bare_name + same fandom_id) auto-merge with
|
||||
image_tags reassigned and the duplicate row deleted.
|
||||
|
||||
Phase 3 — Drop UNIQUE(tag.name); create four partial unique indices:
|
||||
tag_character_with_fandom_uniq UNIQUE(name, fandom_id) WHERE kind='character' AND fandom_id IS NOT NULL
|
||||
tag_character_null_fandom_uniq UNIQUE(name) WHERE kind='character' AND fandom_id IS NULL
|
||||
tag_other_kinds_uniq UNIQUE(name, kind) WHERE kind != 'character'
|
||||
tag_kind_idx BTREE(kind)
|
||||
|
||||
Also cascades renames into tag_reference_embedding.tag_name (keyed by name
|
||||
string, same pattern as h26041901 / i26041901).
|
||||
|
||||
Downgrade raises — merges are destructive; roll back via the
|
||||
/tmp/imagerepo_pre_refactor_backup.sql dump captured in pre-flight.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import text
|
||||
|
||||
|
||||
revision = 'j26042101'
|
||||
down_revision = 'i26041901'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
# Same kinds app-side (app/utils/tag_prefix.KNOWN_KINDS), with colon suffix.
|
||||
# 'user' intentionally excluded — those tags have always been stored bare.
|
||||
KNOWN_PREFIXES = (
|
||||
'character:', 'fandom:', 'artist:', 'series:',
|
||||
'post:', 'archive:', 'meta:',
|
||||
)
|
||||
|
||||
FANDOM_SUFFIX_RE = re.compile(r'^(.+?) \(([^()]+)\)$')
|
||||
|
||||
|
||||
def _strip_prefix(name: str) -> tuple[str, str | None]:
|
||||
"""Return (stripped_name, prefix_without_colon). No prefix -> (name, None)."""
|
||||
for prefix in KNOWN_PREFIXES:
|
||||
if name.startswith(prefix):
|
||||
return name[len(prefix):], prefix.rstrip(':')
|
||||
return name, None
|
||||
|
||||
|
||||
def _reassign_image_tags(conn, from_id: int, to_id: int) -> None:
|
||||
"""Move every image_tags row pointing at from_id over to to_id, dedup via
|
||||
ON CONFLICT."""
|
||||
conn.execute(
|
||||
text("""
|
||||
INSERT INTO image_tags (image_id, tag_id)
|
||||
SELECT DISTINCT image_id, :to_id
|
||||
FROM image_tags
|
||||
WHERE tag_id = :from_id
|
||||
ON CONFLICT DO NOTHING
|
||||
"""),
|
||||
{'from_id': from_id, 'to_id': to_id},
|
||||
)
|
||||
conn.execute(
|
||||
text("DELETE FROM image_tags WHERE tag_id = :from_id"),
|
||||
{'from_id': from_id},
|
||||
)
|
||||
|
||||
|
||||
def _rename_tag_reference_embedding(conn, old_name: str, new_name: str) -> None:
|
||||
"""Cascade a tag rename into tag_reference_embedding.tag_name, which is a
|
||||
PK string. Copies the loser's rows onto the winner's name where the winner
|
||||
has no row for that model_version; deletes the rest."""
|
||||
if old_name == new_name:
|
||||
return
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE tag_reference_embedding
|
||||
SET tag_name = :new_name
|
||||
WHERE tag_name = :old_name
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM tag_reference_embedding w
|
||||
WHERE w.tag_name = :new_name
|
||||
AND w.model_version = tag_reference_embedding.model_version
|
||||
)
|
||||
"""),
|
||||
{'old_name': old_name, 'new_name': new_name},
|
||||
)
|
||||
conn.execute(
|
||||
text("DELETE FROM tag_reference_embedding WHERE tag_name = :old_name"),
|
||||
{'old_name': old_name},
|
||||
)
|
||||
|
||||
|
||||
def upgrade():
|
||||
conn = op.get_bind()
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Phase 1 — strip kind: prefix from every non-user tag's name
|
||||
# -------------------------------------------------------------------------
|
||||
phase1_renames = 0
|
||||
rows = conn.execute(
|
||||
text("SELECT id, name, kind FROM tag ORDER BY id")
|
||||
).fetchall()
|
||||
|
||||
for tag_id, old_name, kind in rows:
|
||||
new_name, stripped_prefix = _strip_prefix(old_name)
|
||||
if new_name == old_name:
|
||||
continue
|
||||
# Sanity: the stripped prefix should match the kind column. If not,
|
||||
# the data is inconsistent; log and trust the kind column.
|
||||
if stripped_prefix != kind:
|
||||
print(
|
||||
f"phase1 warning: tag {tag_id} name prefix={stripped_prefix!r} "
|
||||
f"but kind column={kind!r}; trusting kind"
|
||||
)
|
||||
conn.execute(
|
||||
text("UPDATE tag SET name = :new_name WHERE id = :id"),
|
||||
{'new_name': new_name, 'id': tag_id},
|
||||
)
|
||||
_rename_tag_reference_embedding(conn, old_name, new_name)
|
||||
phase1_renames += 1
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Phase 2 — extract (Fandom) suffix from character names into fandom_id
|
||||
# -------------------------------------------------------------------------
|
||||
phase2_extracted = 0
|
||||
phase2_merged = 0
|
||||
phase2_orphans: list[tuple[int, str]] = []
|
||||
phase2_malformed: list[tuple[int, str]] = []
|
||||
|
||||
char_rows = conn.execute(
|
||||
text("SELECT id, name, fandom_id FROM tag WHERE kind = 'character' ORDER BY id")
|
||||
).fetchall()
|
||||
|
||||
for tag_id, char_name, existing_fandom_id in char_rows:
|
||||
m = FANDOM_SUFFIX_RE.match(char_name)
|
||||
if not m:
|
||||
# No suffix. Check if a suffixed sibling exists — if so, this is
|
||||
# an orphan worth reporting so the user can merge manually.
|
||||
sibling = conn.execute(
|
||||
text("""
|
||||
SELECT id FROM tag
|
||||
WHERE kind = 'character'
|
||||
AND name LIKE :pattern
|
||||
AND id != :tag_id
|
||||
"""),
|
||||
{'pattern': f"{char_name} (%)", 'tag_id': tag_id},
|
||||
).fetchone()
|
||||
if sibling:
|
||||
phase2_orphans.append((tag_id, char_name))
|
||||
continue
|
||||
|
||||
bare_name, fandom_name = m.group(1), m.group(2)
|
||||
bare_name = bare_name.strip()
|
||||
fandom_name = fandom_name.strip()
|
||||
if not bare_name or not fandom_name:
|
||||
phase2_malformed.append((tag_id, char_name))
|
||||
continue
|
||||
|
||||
# Ensure the fandom tag exists. Phase 1 has already stripped any
|
||||
# 'fandom:' prefix, so we look up by bare name.
|
||||
fandom_row = conn.execute(
|
||||
text("SELECT id FROM tag WHERE kind = 'fandom' AND name = :name"),
|
||||
{'name': fandom_name},
|
||||
).fetchone()
|
||||
if fandom_row is None:
|
||||
result = conn.execute(
|
||||
text("INSERT INTO tag (name, kind) VALUES (:name, 'fandom') RETURNING id"),
|
||||
{'name': fandom_name},
|
||||
)
|
||||
new_fandom_id = result.scalar_one()
|
||||
else:
|
||||
new_fandom_id = fandom_row[0]
|
||||
|
||||
# Collision check: is there already a (name=bare_name, fandom_id=new_fandom_id, kind='character')?
|
||||
collision = conn.execute(
|
||||
text("""
|
||||
SELECT id, name FROM tag
|
||||
WHERE kind = 'character'
|
||||
AND name = :bare
|
||||
AND fandom_id = :fid
|
||||
AND id != :tag_id
|
||||
"""),
|
||||
{'bare': bare_name, 'fid': new_fandom_id, 'tag_id': tag_id},
|
||||
).fetchone()
|
||||
|
||||
if collision:
|
||||
# Auto-merge. Reassign image_tags, cascade tag_reference_embedding
|
||||
# rename (from current tag's existing name into the collision's
|
||||
# name), delete current tag.
|
||||
collision_id, collision_name = collision
|
||||
_reassign_image_tags(conn, from_id=tag_id, to_id=collision_id)
|
||||
_rename_tag_reference_embedding(conn, old_name=char_name, new_name=collision_name)
|
||||
conn.execute(
|
||||
text("DELETE FROM tag WHERE id = :id"),
|
||||
{'id': tag_id},
|
||||
)
|
||||
phase2_merged += 1
|
||||
continue
|
||||
|
||||
# Happy path: update name + fandom_id, cascade rename in tag_reference_embedding.
|
||||
conn.execute(
|
||||
text("UPDATE tag SET name = :bare, fandom_id = :fid WHERE id = :id"),
|
||||
{'bare': bare_name, 'fid': new_fandom_id, 'id': tag_id},
|
||||
)
|
||||
_rename_tag_reference_embedding(conn, old_name=char_name, new_name=bare_name)
|
||||
phase2_extracted += 1
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Phase 3 — drop old unique, create partial indices
|
||||
# -------------------------------------------------------------------------
|
||||
# The unique index on tag.name was created as part of the initial table
|
||||
# definition; its name in Postgres is typically tag_name_key (SQLAlchemy's
|
||||
# default for UniqueConstraint on column tag.name). If your DB has a
|
||||
# different name, adjust here. The drop + create is atomic with the rest
|
||||
# of the migration.
|
||||
op.drop_constraint('tag_name_key', 'tag', type_='unique')
|
||||
|
||||
op.create_index(
|
||||
'tag_character_with_fandom_uniq', 'tag', ['name', 'fandom_id'],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("kind = 'character' AND fandom_id IS NOT NULL"),
|
||||
)
|
||||
op.create_index(
|
||||
'tag_character_null_fandom_uniq', 'tag', ['name'],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("kind = 'character' AND fandom_id IS NULL"),
|
||||
)
|
||||
op.create_index(
|
||||
'tag_other_kinds_uniq', 'tag', ['name', 'kind'],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("kind != 'character'"),
|
||||
)
|
||||
op.create_index('tag_kind_idx', 'tag', ['kind'])
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Report
|
||||
# -------------------------------------------------------------------------
|
||||
print("=" * 72)
|
||||
print(f"j26042101: Phase 1 prefix-strips: {phase1_renames}")
|
||||
print(f"j26042101: Phase 2 character extractions: {phase2_extracted}")
|
||||
print(f"j26042101: Phase 2 auto-merged duplicates: {phase2_merged}")
|
||||
print(f"j26042101: Phase 2 bare-name orphans (review in /tags): {len(phase2_orphans)}")
|
||||
for tid, nm in phase2_orphans:
|
||||
print(f" - tag {tid} '{nm}' coexists with a suffixed sibling")
|
||||
print(f"j26042101: Phase 2 malformed names skipped: {len(phase2_malformed)}")
|
||||
for tid, nm in phase2_malformed:
|
||||
print(f" - tag {tid} '{nm}'")
|
||||
print("=" * 72)
|
||||
|
||||
|
||||
def downgrade():
|
||||
raise NotImplementedError(
|
||||
"j26042101 strips kind prefixes and extracts character fandoms. "
|
||||
"Both phases are destructive (auto-merges delete tag rows; partial "
|
||||
"indices differ from the old unique constraint). Restore from the "
|
||||
"pre-migration backup at /tmp/imagerepo_pre_refactor_backup.sql "
|
||||
"if you need to roll back."
|
||||
)
|
||||
Reference in New Issue
Block a user