feat(tags): retro-normalize existing tags to Title Case + merge case-collisions (plan #714)
Follow-up to #701: new tags are saved canonical, but the back-catalog keeps whatever casing it was created with. This adds a maintenance action that Title-Cases every existing tag (collapsing whitespace) and merges case/whitespace-variant duplicates into one. Backend: - tag_service.normalize_existing_tags(session, *, dry_run): groups all tags by (kind, coalesce(fandom_id,-1), canonical_name). Per group it picks a survivor (prefer an already-canonical member → no rename/self-alias; else the best-connected tag → fewest FK repoints; else lowest id), merges the variants INTO it via the tested TagService._do_merge (image_tag/allowlist/embedding/ aliases/series_page repoints + protective ML aliases), then renames the survivor to canonical. Losers are deleted before the rename so there's no transient unique-index clash; commits per group and isolates failures per group. Idempotent — an already-canonical lone tag is a no-op. - normalize_tags_task (maintenance queue, asyncio.run + per-task NullPool async engine, soft 1800/hard 2400) — recovery/timeout/duration covered by FC-3i. - POST /api/admin/tags/normalize: dry_run=true returns a projection inline (group/collision/rename counts + sample); dry_run=false enqueues the task. Frontend: a "Standardize tag casing" section in TagMaintenanceCard (Cleanup tab) — preview → apply (polls the activity dashboard to terminal status), behind a back-up-first warning. admin store gains normalizeTags(). Tests: tests/test_tag_normalize.py — dry-run counts, live merge + image-tag dedup/repoint, idempotency, same-name-different-fandom and -different-kind kept separate, ML-known loser keeps a protective alias. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,189 @@
|
||||
"""#714: retro-normalize existing tags to the #701 canonical (Title Case +
|
||||
collapsed whitespace) and merge case/whitespace-variant duplicates.
|
||||
|
||||
normalize_existing_tags commits per group, so case-variant duplicates are
|
||||
seeded with DIRECT inserts (find_or_create would case-insensitively dedup
|
||||
them, which is exactly the pre-#701 state we're cleaning up), and assertions
|
||||
re-query via fresh selects rather than touching post-commit ORM entities.
|
||||
"""
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import Tag, TagKind, image_tag
|
||||
from backend.app.models.tag_alias import TagAlias
|
||||
from backend.app.services.tag_service import (
|
||||
TagService,
|
||||
normalize_existing_tags,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
_IMG_SEQ = 0
|
||||
|
||||
|
||||
async def _img(db):
|
||||
"""Minimal valid ImageRecord; returns its id."""
|
||||
global _IMG_SEQ
|
||||
_IMG_SEQ += 1
|
||||
from backend.app.models import ImageRecord
|
||||
|
||||
rec = ImageRecord(
|
||||
path=f"/tmp/fc_norm_{_IMG_SEQ}.png",
|
||||
sha256=f"{_IMG_SEQ:064d}",
|
||||
size_bytes=1,
|
||||
mime="image/png",
|
||||
origin="uploaded",
|
||||
)
|
||||
db.add(rec)
|
||||
await db.flush()
|
||||
return rec.id
|
||||
|
||||
|
||||
async def _raw_tag(db, name, kind, fandom_id=None):
|
||||
"""Insert a Tag row verbatim — bypasses find_or_create's case-insensitive
|
||||
dedup so we can plant the case-variant duplicates #714 exists to fix."""
|
||||
t = Tag(name=name, kind=kind, fandom_id=fandom_id)
|
||||
db.add(t)
|
||||
await db.flush()
|
||||
return t
|
||||
|
||||
|
||||
async def _link(db, image_id, tag_id, source="manual"):
|
||||
await db.execute(
|
||||
image_tag.insert().values(
|
||||
image_record_id=image_id, tag_id=tag_id, source=source
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _count_named(db, name, kind):
|
||||
return await db.scalar(
|
||||
select(func.count())
|
||||
.select_from(Tag)
|
||||
.where(Tag.name == name)
|
||||
.where(Tag.kind == kind)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dry_run_projection_counts(db):
|
||||
# Collision group with an already-canonical member (→ no rename, 1 merge).
|
||||
await _raw_tag(db, "hatsune miku", TagKind.character)
|
||||
await _raw_tag(db, "Hatsune Miku", TagKind.character)
|
||||
# Lone mis-cased + extra whitespace (→ rename, no merge).
|
||||
await _raw_tag(db, "looking at viewer", TagKind.general)
|
||||
# Already canonical (→ no change).
|
||||
await _raw_tag(db, "Already Canonical", TagKind.general)
|
||||
|
||||
proj = await normalize_existing_tags(db, dry_run=True)
|
||||
|
||||
assert proj["collisions"] == 1
|
||||
assert proj["tags_to_merge"] == 1
|
||||
assert proj["tags_to_rename"] == 1 # only the whitespace group renames
|
||||
assert proj["total_changes"] == 2
|
||||
# Nothing mutated by a dry run.
|
||||
assert await _count_named(db, "hatsune miku", TagKind.character) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_merges_case_variants_and_repoints_images(db):
|
||||
a = await _raw_tag(db, "hatsune miku", TagKind.character)
|
||||
b = await _raw_tag(db, "HATSUNE MIKU", TagKind.character)
|
||||
i1, i2, i3 = await _img(db), await _img(db), await _img(db)
|
||||
await _link(db, i1, a.id)
|
||||
await _link(db, i2, a.id)
|
||||
await _link(db, i2, b.id) # shared → must dedup on the survivor
|
||||
await _link(db, i3, b.id)
|
||||
|
||||
summary = await normalize_existing_tags(db, dry_run=False)
|
||||
|
||||
assert summary["errors"] == 0
|
||||
assert summary["merged"] == 1
|
||||
assert summary["renamed"] == 1
|
||||
assert summary["groups_processed"] == 1
|
||||
|
||||
# Exactly one canonical character tag survives; the variant is gone.
|
||||
assert await _count_named(db, "Hatsune Miku", TagKind.character) == 1
|
||||
total_chars = await db.scalar(
|
||||
select(func.count()).select_from(Tag).where(Tag.kind == TagKind.character)
|
||||
)
|
||||
assert total_chars == 1
|
||||
survivor_id = await db.scalar(
|
||||
select(Tag.id)
|
||||
.where(Tag.name == "Hatsune Miku")
|
||||
.where(Tag.kind == TagKind.character)
|
||||
)
|
||||
# Union of {i1,i2} and {i2,i3} with the shared i2 deduped == 3.
|
||||
assoc = await db.scalar(
|
||||
select(func.count())
|
||||
.select_from(image_tag)
|
||||
.where(image_tag.c.tag_id == survivor_id)
|
||||
)
|
||||
assert assoc == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_idempotent_second_run_is_noop(db):
|
||||
await _raw_tag(db, "kafka", TagKind.character)
|
||||
await _raw_tag(db, "KAFKA", TagKind.character)
|
||||
|
||||
first = await normalize_existing_tags(db, dry_run=False)
|
||||
assert first["groups_processed"] == 1
|
||||
|
||||
proj = await normalize_existing_tags(db, dry_run=True)
|
||||
assert proj["total_changes"] == 0
|
||||
again = await normalize_existing_tags(db, dry_run=False)
|
||||
assert again["groups_processed"] == 0
|
||||
assert again["merged"] == 0
|
||||
assert again["renamed"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_name_different_fandom_not_merged(db):
|
||||
f1 = await _raw_tag(db, "Bleach", TagKind.fandom)
|
||||
f2 = await _raw_tag(db, "Naruto", TagKind.fandom)
|
||||
await _raw_tag(db, "alice", TagKind.character, fandom_id=f1.id)
|
||||
await _raw_tag(db, "Alice", TagKind.character, fandom_id=f2.id)
|
||||
|
||||
summary = await normalize_existing_tags(db, dry_run=False)
|
||||
|
||||
assert summary["merged"] == 0
|
||||
assert summary["renamed"] == 1 # only the f1 "alice" recases
|
||||
# Both characters survive — same canonical name, distinct fandoms.
|
||||
assert await _count_named(db, "Alice", TagKind.character) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_name_different_kind_not_merged(db):
|
||||
await _raw_tag(db, "forge", TagKind.artist)
|
||||
await _raw_tag(db, "Forge", TagKind.general)
|
||||
|
||||
summary = await normalize_existing_tags(db, dry_run=False)
|
||||
|
||||
assert summary["merged"] == 0
|
||||
assert await _count_named(db, "Forge", TagKind.artist) == 1
|
||||
assert await _count_named(db, "Forge", TagKind.general) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ml_known_loser_keeps_protective_alias(db):
|
||||
# Survivor is the already-canonical (manual, ML-unknown) tag; the ML-sourced
|
||||
# variant merges away and must leave an alias so the tagger re-resolves it.
|
||||
canon = await _raw_tag(db, "Nemu", TagKind.general)
|
||||
canon_id = canon.id # capture before the service commits
|
||||
ml = await _raw_tag(db, "nemu", TagKind.general)
|
||||
img = await _img(db)
|
||||
await _link(db, img, ml.id, source="ml_auto")
|
||||
|
||||
summary = await normalize_existing_tags(db, dry_run=False)
|
||||
|
||||
assert summary["merged"] == 1
|
||||
assert summary["aliases_created"] == 1
|
||||
assert await _count_named(db, "Nemu", TagKind.general) == 1
|
||||
alias_rows = await db.scalar(
|
||||
select(func.count())
|
||||
.select_from(TagAlias)
|
||||
.where(TagAlias.alias_string == "nemu")
|
||||
.where(TagAlias.canonical_tag_id == canon_id)
|
||||
)
|
||||
assert alias_rows == 1
|
||||
Reference in New Issue
Block a user