From c7e2605a1036580071bd936f0993fe5acf0b5941 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 22 Apr 2026 09:39:36 -0400 Subject: [PATCH] feat(migration): bare tag names; fandom_id authoritative Three-phase Alembic migration j26042101 strips 'kind:' prefix from every non-user tag, extracts character '(Fandom)' suffix into fandom_id with auto-merge on duplicates, and swaps UNIQUE(name) for four partial indices keyed on (name, kind) with special handling for the (character, fandom_id) combination. Co-Authored-By: Claude Opus 4.7 --- ...e_tag_names_and_fandom_id_authoritative.py | 277 ++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 migrations/versions/j26042101_bare_tag_names_and_fandom_id_authoritative.py diff --git a/migrations/versions/j26042101_bare_tag_names_and_fandom_id_authoritative.py b/migrations/versions/j26042101_bare_tag_names_and_fandom_id_authoritative.py new file mode 100644 index 0000000..025d141 --- /dev/null +++ b/migrations/versions/j26042101_bare_tag_names_and_fandom_id_authoritative.py @@ -0,0 +1,277 @@ +"""Strip kind: prefix from tag names, extract character (Fandom) suffix into +fandom_id, and swap UNIQUE(name) for four partial unique indices keyed on +(name, kind) with special handling for characters. + +Revision ID: j26042101 +Revises: i26041901 +Create Date: 2026-04-21 + +Three phases run in one transaction: + + Phase 1 — Strip kind prefix from tag.name for every non-user tag. + 'character:Ruby Rose (RWBY)' -> 'Ruby Rose (RWBY)' + 'artist:Eric Canete' -> 'Eric Canete' + 'post:patreon:eric:12345' -> 'patreon:eric:12345' + User tags (kind='user', already bare) are untouched. + + Phase 2 — For every kind='character' tag whose name still matches + 'Name (Fandom)' after phase 1, ensure the fandom tag exists, + update the character's fandom_id, and strip the suffix from its name. + True duplicates (same bare_name + same fandom_id) auto-merge with + image_tags reassigned and the duplicate row deleted. + + Phase 3 — Drop UNIQUE(tag.name); create four partial unique indices: + tag_character_with_fandom_uniq UNIQUE(name, fandom_id) WHERE kind='character' AND fandom_id IS NOT NULL + tag_character_null_fandom_uniq UNIQUE(name) WHERE kind='character' AND fandom_id IS NULL + tag_other_kinds_uniq UNIQUE(name, kind) WHERE kind != 'character' + tag_kind_idx BTREE(kind) + +Also cascades renames into tag_reference_embedding.tag_name (keyed by name +string, same pattern as h26041901 / i26041901). + +Downgrade raises — merges are destructive; roll back via the +/tmp/imagerepo_pre_refactor_backup.sql dump captured in pre-flight. +""" +from __future__ import annotations + +import re + +from alembic import op +import sqlalchemy as sa +from sqlalchemy import text + + +revision = 'j26042101' +down_revision = 'i26041901' +branch_labels = None +depends_on = None + + +# Same kinds app-side (app/utils/tag_prefix.KNOWN_KINDS), with colon suffix. +# 'user' intentionally excluded — those tags have always been stored bare. +KNOWN_PREFIXES = ( + 'character:', 'fandom:', 'artist:', 'series:', + 'post:', 'archive:', 'meta:', +) + +FANDOM_SUFFIX_RE = re.compile(r'^(.+?) \(([^()]+)\)$') + + +def _strip_prefix(name: str) -> tuple[str, str | None]: + """Return (stripped_name, prefix_without_colon). No prefix -> (name, None).""" + for prefix in KNOWN_PREFIXES: + if name.startswith(prefix): + return name[len(prefix):], prefix.rstrip(':') + return name, None + + +def _reassign_image_tags(conn, from_id: int, to_id: int) -> None: + """Move every image_tags row pointing at from_id over to to_id, dedup via + ON CONFLICT.""" + conn.execute( + text(""" + INSERT INTO image_tags (image_id, tag_id) + SELECT DISTINCT image_id, :to_id + FROM image_tags + WHERE tag_id = :from_id + ON CONFLICT DO NOTHING + """), + {'from_id': from_id, 'to_id': to_id}, + ) + conn.execute( + text("DELETE FROM image_tags WHERE tag_id = :from_id"), + {'from_id': from_id}, + ) + + +def _rename_tag_reference_embedding(conn, old_name: str, new_name: str) -> None: + """Cascade a tag rename into tag_reference_embedding.tag_name, which is a + PK string. Copies the loser's rows onto the winner's name where the winner + has no row for that model_version; deletes the rest.""" + if old_name == new_name: + return + conn.execute( + text(""" + UPDATE tag_reference_embedding + SET tag_name = :new_name + WHERE tag_name = :old_name + AND NOT EXISTS ( + SELECT 1 FROM tag_reference_embedding w + WHERE w.tag_name = :new_name + AND w.model_version = tag_reference_embedding.model_version + ) + """), + {'old_name': old_name, 'new_name': new_name}, + ) + conn.execute( + text("DELETE FROM tag_reference_embedding WHERE tag_name = :old_name"), + {'old_name': old_name}, + ) + + +def upgrade(): + conn = op.get_bind() + + # ------------------------------------------------------------------------- + # Phase 1 — strip kind: prefix from every non-user tag's name + # ------------------------------------------------------------------------- + phase1_renames = 0 + rows = conn.execute( + text("SELECT id, name, kind FROM tag ORDER BY id") + ).fetchall() + + for tag_id, old_name, kind in rows: + new_name, stripped_prefix = _strip_prefix(old_name) + if new_name == old_name: + continue + # Sanity: the stripped prefix should match the kind column. If not, + # the data is inconsistent; log and trust the kind column. + if stripped_prefix != kind: + print( + f"phase1 warning: tag {tag_id} name prefix={stripped_prefix!r} " + f"but kind column={kind!r}; trusting kind" + ) + conn.execute( + text("UPDATE tag SET name = :new_name WHERE id = :id"), + {'new_name': new_name, 'id': tag_id}, + ) + _rename_tag_reference_embedding(conn, old_name, new_name) + phase1_renames += 1 + + # ------------------------------------------------------------------------- + # Phase 2 — extract (Fandom) suffix from character names into fandom_id + # ------------------------------------------------------------------------- + phase2_extracted = 0 + phase2_merged = 0 + phase2_orphans: list[tuple[int, str]] = [] + phase2_malformed: list[tuple[int, str]] = [] + + char_rows = conn.execute( + text("SELECT id, name, fandom_id FROM tag WHERE kind = 'character' ORDER BY id") + ).fetchall() + + for tag_id, char_name, existing_fandom_id in char_rows: + m = FANDOM_SUFFIX_RE.match(char_name) + if not m: + # No suffix. Check if a suffixed sibling exists — if so, this is + # an orphan worth reporting so the user can merge manually. + sibling = conn.execute( + text(""" + SELECT id FROM tag + WHERE kind = 'character' + AND name LIKE :pattern + AND id != :tag_id + """), + {'pattern': f"{char_name} (%)", 'tag_id': tag_id}, + ).fetchone() + if sibling: + phase2_orphans.append((tag_id, char_name)) + continue + + bare_name, fandom_name = m.group(1), m.group(2) + bare_name = bare_name.strip() + fandom_name = fandom_name.strip() + if not bare_name or not fandom_name: + phase2_malformed.append((tag_id, char_name)) + continue + + # Ensure the fandom tag exists. Phase 1 has already stripped any + # 'fandom:' prefix, so we look up by bare name. + fandom_row = conn.execute( + text("SELECT id FROM tag WHERE kind = 'fandom' AND name = :name"), + {'name': fandom_name}, + ).fetchone() + if fandom_row is None: + result = conn.execute( + text("INSERT INTO tag (name, kind) VALUES (:name, 'fandom') RETURNING id"), + {'name': fandom_name}, + ) + new_fandom_id = result.scalar_one() + else: + new_fandom_id = fandom_row[0] + + # Collision check: is there already a (name=bare_name, fandom_id=new_fandom_id, kind='character')? + collision = conn.execute( + text(""" + SELECT id, name FROM tag + WHERE kind = 'character' + AND name = :bare + AND fandom_id = :fid + AND id != :tag_id + """), + {'bare': bare_name, 'fid': new_fandom_id, 'tag_id': tag_id}, + ).fetchone() + + if collision: + # Auto-merge. Reassign image_tags, cascade tag_reference_embedding + # rename (from current tag's existing name into the collision's + # name), delete current tag. + collision_id, collision_name = collision + _reassign_image_tags(conn, from_id=tag_id, to_id=collision_id) + _rename_tag_reference_embedding(conn, old_name=char_name, new_name=collision_name) + conn.execute( + text("DELETE FROM tag WHERE id = :id"), + {'id': tag_id}, + ) + phase2_merged += 1 + continue + + # Happy path: update name + fandom_id, cascade rename in tag_reference_embedding. + conn.execute( + text("UPDATE tag SET name = :bare, fandom_id = :fid WHERE id = :id"), + {'bare': bare_name, 'fid': new_fandom_id, 'id': tag_id}, + ) + _rename_tag_reference_embedding(conn, old_name=char_name, new_name=bare_name) + phase2_extracted += 1 + + # ------------------------------------------------------------------------- + # Phase 3 — drop old unique, create partial indices + # ------------------------------------------------------------------------- + # The unique index on tag.name was created as part of the initial table + # definition; its name in Postgres is typically tag_name_key (SQLAlchemy's + # default for UniqueConstraint on column tag.name). If your DB has a + # different name, adjust here. The drop + create is atomic with the rest + # of the migration. + op.drop_constraint('tag_name_key', 'tag', type_='unique') + + op.create_index( + 'tag_character_with_fandom_uniq', 'tag', ['name', 'fandom_id'], + unique=True, + postgresql_where=sa.text("kind = 'character' AND fandom_id IS NOT NULL"), + ) + op.create_index( + 'tag_character_null_fandom_uniq', 'tag', ['name'], + unique=True, + postgresql_where=sa.text("kind = 'character' AND fandom_id IS NULL"), + ) + op.create_index( + 'tag_other_kinds_uniq', 'tag', ['name', 'kind'], + unique=True, + postgresql_where=sa.text("kind != 'character'"), + ) + op.create_index('tag_kind_idx', 'tag', ['kind']) + + # ------------------------------------------------------------------------- + # Report + # ------------------------------------------------------------------------- + print("=" * 72) + print(f"j26042101: Phase 1 prefix-strips: {phase1_renames}") + print(f"j26042101: Phase 2 character extractions: {phase2_extracted}") + print(f"j26042101: Phase 2 auto-merged duplicates: {phase2_merged}") + print(f"j26042101: Phase 2 bare-name orphans (review in /tags): {len(phase2_orphans)}") + for tid, nm in phase2_orphans: + print(f" - tag {tid} '{nm}' coexists with a suffixed sibling") + print(f"j26042101: Phase 2 malformed names skipped: {len(phase2_malformed)}") + for tid, nm in phase2_malformed: + print(f" - tag {tid} '{nm}'") + print("=" * 72) + + +def downgrade(): + raise NotImplementedError( + "j26042101 strips kind prefixes and extracts character fandoms. " + "Both phases are destructive (auto-merges delete tag rows; partial " + "indices differ from the old unique constraint). Restore from the " + "pre-migration backup at /tmp/imagerepo_pre_refactor_backup.sql " + "if you need to roll back." + )