feat(tags): retro-normalize existing tags to Title Case + merge case-collisions (plan #714)

Follow-up to #701: new tags are saved canonical, but the back-catalog keeps
whatever casing it was created with. This adds a maintenance action that
Title-Cases every existing tag (collapsing whitespace) and merges
case/whitespace-variant duplicates into one.

Backend:
- tag_service.normalize_existing_tags(session, *, dry_run): groups all tags by
  (kind, coalesce(fandom_id,-1), canonical_name). Per group it picks a survivor
  (prefer an already-canonical member → no rename/self-alias; else the
  best-connected tag → fewest FK repoints; else lowest id), merges the variants
  INTO it via the tested TagService._do_merge (image_tag/allowlist/embedding/
  aliases/series_page repoints + protective ML aliases), then renames the
  survivor to canonical. Losers are deleted before the rename so there's no
  transient unique-index clash; commits per group and isolates failures per
  group. Idempotent — an already-canonical lone tag is a no-op.
- normalize_tags_task (maintenance queue, asyncio.run + per-task NullPool async
  engine, soft 1800/hard 2400) — recovery/timeout/duration covered by FC-3i.
- POST /api/admin/tags/normalize: dry_run=true returns a projection inline
  (group/collision/rename counts + sample); dry_run=false enqueues the task.

Frontend: a "Standardize tag casing" section in TagMaintenanceCard (Cleanup
tab) — preview → apply (polls the activity dashboard to terminal status),
behind a back-up-first warning. admin store gains normalizeTags().

Tests: tests/test_tag_normalize.py — dry-run counts, live merge + image-tag
dedup/repoint, idempotency, same-name-different-fandom and -different-kind kept
separate, ML-known loser keeps a protective alias.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/app/api/admin.py

@@ -245,6 +245,32 @@ async def tags_reset_content():
     return jsonify(result)
 
 
+@admin_bp.route("/tags/normalize", methods=["POST"])
+async def tags_normalize():
+    """#714: retro-normalize existing tags to the #701 canonical form (Title
+    Case + collapsed whitespace) and merge case/whitespace-variant duplicates.
+
+    dry_run=true (default) returns a projection inline — group/collision/rename
+    counts + a sample of the changes — so the UI shows exactly what'll happen.
+    dry_run=false dispatches the long-running maintenance task (the merge FK
+    repoints can touch many tags); the UI tails the activity dashboard for the
+    summary. Idempotent; back up first (the merges are irreversible)."""
+    from ..services.tag_service import normalize_existing_tags
+
+    body = await request.get_json(silent=True) or {}
+    dry_run = bool(body.get("dry_run", True))
+
+    if dry_run:
+        async with get_session() as session:
+            result = await normalize_existing_tags(session, dry_run=True)
+        return jsonify(result)
+
+    from ..tasks.admin import normalize_tags_task
+
+    async_result = normalize_tags_task.delay()
+    return jsonify({"task_id": async_result.id, "status": "queued"}), 202
+
+
 @admin_bp.route("/maintenance/db-stats", methods=["GET"])
 async def db_stats():
     """Per-table bloat readout (pg_stat_user_tables) for the high-churn tables

backend/app/services/tag_service.py

@@ -1,5 +1,6 @@
 """Tag CRUD + autocomplete + image-tag association."""
 
+import logging
 from collections.abc import Sequence
 from dataclasses import dataclass
 
@@ -12,6 +13,8 @@ from ..models import Tag, TagKind, image_tag
 from ..models.tag_allowlist import TagAllowlist
 from ..models.tag_reference_embedding import TagReferenceEmbedding
 
+log = logging.getLogger(__name__)
+
 
 def normalize_tag_name(name: str) -> str:
     """Canonical tag form (#701): collapse whitespace + Title Case.
@@ -526,6 +529,21 @@ class TagService:
             update(Tag).where(Tag.fandom_id == src).values(fandom_id=tgt)
         )
 
+    async def _image_assoc_counts(self, tag_ids: list[int]) -> dict[int, int]:
+        """image_tag row counts keyed by tag_id, for the survivor heuristic
+        (the best-connected tag in a collision group survives → fewest
+        FK repoints). Tags with zero associations are absent from the map."""
+        if not tag_ids:
+            return {}
+        rows = (
+            await self.session.execute(
+                select(image_tag.c.tag_id, func.count())
+                .where(image_tag.c.tag_id.in_(tag_ids))
+                .group_by(image_tag.c.tag_id)
+            )
+        ).all()
+        return {tid: int(n) for tid, n in rows}
+
     async def _create_protective_aliases(
         self, src_name: str, src_kind: TagKind, tgt: int
     ) -> bool:
@@ -573,3 +591,161 @@ class TagService:
             if res.rowcount:
                 created = True
         return created
+
+
+# ---------------------------------------------------------------------------
+# #714: retro-normalize existing tags to the #701 canonical (Title Case +
+# collapsed whitespace) and merge case/whitespace-variant duplicates.
+# ---------------------------------------------------------------------------
+
+_NORMALIZE_SAMPLE_CAP = 50
+
+
+def _group_existing_tags(
+    rows,
+) -> dict[tuple, list[tuple[int, str]]]:
+    """Group (id, name, kind, fandom_id) rows by their post-normalization
+    identity: (kind, COALESCE(fandom_id, -1), canonical_name). Every tag that
+    would collapse to the same canonical lives in ONE group, so the canonical
+    form is unique within (kind, fandom) once each group is resolved."""
+    groups: dict[tuple, list[tuple[int, str]]] = {}
+    for tag_id, name, kind, fandom_id in rows:
+        canonical = normalize_tag_name(name)
+        key = (kind, fandom_id if fandom_id is not None else -1, canonical)
+        groups.setdefault(key, []).append((tag_id, name))
+    return groups
+
+
+def _group_needs_change(canonical: str, members: list[tuple[int, str]]) -> bool:
+    """A group is already canonical iff it's a single member whose name equals
+    the canonical form. Anything else (a collision, or a lone mis-cased tag)
+    needs work."""
+    if len(members) > 1:
+        return True
+    return members[0][1] != canonical
+
+
+def _best_connected(tag_ids: list[int], counts: dict[int, int]) -> int:
+    """The tag with the most image associations (→ fewest FK repoints when it
+    survives), tie-broken to the lowest id for determinism. Module-level so the
+    key closes over its parameter, not a loop variable (ruff B023)."""
+    return max(tag_ids, key=lambda tid: (counts.get(tid, 0), -tid))
+
+
+async def normalize_existing_tags(
+    session: AsyncSession, *, dry_run: bool = False
+) -> dict:
+    """Convert the back-catalog to the #701 canonical tag form.
+
+    For each (kind, fandom, canonical) group: pick a survivor, merge any
+    case/whitespace-variant siblings INTO it via the tested merge path
+    (TagService._do_merge — FK repoints + protective aliases), then rename the
+    survivor to the canonical form. Idempotent: a group that is already a lone
+    canonical tag is a no-op, so re-running is safe.
+
+    dry_run=True returns a projection (counts + a sample of the changes) with no
+    mutations. Live runs commit per group and isolate failures per group so one
+    bad group can't strand the rest.
+
+    Returns (dry_run):
+      {"groups": N, "collisions": M, "tags_to_merge": K, "tags_to_rename": R,
+       "total_changes": T, "sample": [{"to", "from": [...], "kind", "merge"}]}
+    Returns (live):
+      {"groups_processed", "merged", "renamed", "aliases_created", "errors",
+       "sample": [...]}
+    """
+    rows = (
+        await session.execute(
+            select(Tag.id, Tag.name, Tag.kind, Tag.fandom_id)
+        )
+    ).all()
+    groups = _group_existing_tags(rows)
+
+    # Deterministic sample/ordering: by kind then canonical name.
+    touched = sorted(
+        (
+            (key, members)
+            for key, members in groups.items()
+            if _group_needs_change(key[2], members)
+        ),
+        key=lambda km: (
+            km[0][0].value if hasattr(km[0][0], "value") else str(km[0][0]),
+            km[0][2].lower(),
+        ),
+    )
+
+    sample = [
+        {
+            "to": key[2],
+            "from": [name for _id, name in members],
+            "kind": key[0].value if hasattr(key[0], "value") else str(key[0]),
+            "merge": len(members) > 1,
+        }
+        for key, members in touched[:_NORMALIZE_SAMPLE_CAP]
+    ]
+
+    if dry_run:
+        collisions = sum(1 for key, m in touched if len(m) > 1)
+        tags_to_merge = sum(len(m) - 1 for key, m in touched if len(m) > 1)
+        # A group renames iff the canonical form isn't already one of its
+        # members' exact names (else that member is picked as survivor → no
+        # rename, the rest merge into it).
+        tags_to_rename = sum(
+            1
+            for key, m in touched
+            if key[2] not in {name for _id, name in m}
+        )
+        return {
+            "groups": len(groups),
+            "collisions": collisions,
+            "tags_to_merge": tags_to_merge,
+            "tags_to_rename": tags_to_rename,
+            "total_changes": len(touched),
+            "sample": sample,
+        }
+
+    svc = TagService(session)
+    summary = {
+        "groups_processed": 0,
+        "merged": 0,
+        "renamed": 0,
+        "aliases_created": 0,
+        "errors": 0,
+        "sample": sample,
+    }
+    for key, members in touched:
+        canonical = key[2]
+        names_by_id = {tag_id: name for tag_id, name in members}
+        # Survivor: prefer a member already named canonically (no rename, no
+        # self-alias); else the best-connected (fewest FK repoints); else
+        # lowest id for determinism.
+        survivor_id = next(
+            (tid for tid, name in members if name == canonical), None
+        )
+        if survivor_id is None:
+            counts = await svc._image_assoc_counts(list(names_by_id))
+            survivor_id = _best_connected(list(names_by_id), counts)
+        loser_ids = [tid for tid in names_by_id if tid != survivor_id]
+        try:
+            survivor = await session.get(Tag, survivor_id)
+            for loser_id in loser_ids:
+                loser = await session.get(Tag, loser_id)
+                if loser is None:
+                    continue
+                result = await svc._do_merge(loser, survivor)
+                if result.alias_created:
+                    summary["aliases_created"] += 1
+            if survivor.name != canonical:
+                survivor.name = canonical
+                await session.flush()
+                summary["renamed"] += 1
+            await session.commit()
+            summary["groups_processed"] += 1
+            summary["merged"] += len(loser_ids)
+        except Exception as exc:  # one bad group must not strand the rest
+            await session.rollback()
+            summary["errors"] += 1
+            log.warning(
+                "tag normalize failed for group %r: %s", canonical, exc
+            )
+    return summary

backend/app/tasks/admin.py

@@ -73,3 +73,32 @@ def reextract_archive_attachments_task(self) -> dict:
         return cleanup_service.reextract_archive_attachments(
             session, images_root=IMAGES_ROOT,
         )
+
+
+@celery.task(
+    name="backend.app.tasks.admin.normalize_tags_task",
+    bind=True,
+    autoretry_for=(OperationalError, DBAPIError),
+    retry_backoff=15, retry_backoff_max=180, max_retries=1,
+    soft_time_limit=1800, time_limit=2400,  # 30 min / 40 min
+)
+def normalize_tags_task(self) -> dict:
+    """Wraps tag_service.normalize_existing_tags (#714): Title-Case the
+    back-catalog and merge case/whitespace-variant duplicate tags via the
+    tested async merge path. Runs under its own asyncio loop + per-task async
+    engine (NullPool, disposed when the loop ends), mirroring download_source."""
+    import asyncio
+
+    from ..services.tag_service import normalize_existing_tags
+    from ._async_session import async_session_factory
+
+    async def _run() -> dict:
+        async_factory, async_engine = async_session_factory()
+        try:
+            async with async_factory() as session:
+                # normalize_existing_tags commits per group internally.
+                return await normalize_existing_tags(session, dry_run=False)
+        finally:
+            await async_engine.dispose()
+
+    return asyncio.run(_run())

frontend/src/components/settings/TagMaintenanceCard.vue

@@ -136,6 +136,64 @@
         >Delete {{ resetPreview.count }} content tag(s) +
           {{ resetPreview.applications }} application(s)</v-btn>
       </div>
+
+      <v-divider class="my-5" />
+
+      <!-- #714: standardize existing tags to the canonical Title-Case form
+           and merge case/whitespace-variant duplicates. -->
+      <p class="text-body-2 mb-2">
+        <strong>Standardize tag casing.</strong>
+        New tags are saved Title Case, but older tags keep whatever casing they
+        were created with. This renames every existing tag to
+        <code>Title Case</code> (collapsing extra spaces) and
+        <strong>merges</strong> tags that differ only by case or spacing
+        (e.g. <code>hatsune miku</code> + <code>Hatsune&nbsp;&nbsp;Miku</code>)
+        into one — repointing their images, allowlist entries and series pages.
+      </p>
+      <v-alert type="warning" variant="tonal" density="compact" class="mb-3">
+        The merges are irreversible — back up first
+        (Settings → Maintenance → Backup). Safe to run more than once.
+      </v-alert>
+
+      <v-btn
+        color="accent" variant="flat" rounded="pill"
+        prepend-icon="mdi-magnify"
+        :loading="loadingNormPreview"
+        class="mb-3"
+        @click="onNormPreview"
+      >Preview tag standardization</v-btn>
+
+      <div v-if="normPreview">
+        <p class="text-body-2 mb-2">
+          <strong>{{ normPreview.total_changes }}</strong> tag group(s) to
+          change — <strong>{{ normPreview.tags_to_rename }}</strong> rename(s),
+          <strong>{{ normPreview.collisions }}</strong> collision(s) merging
+          <strong>{{ normPreview.tags_to_merge }}</strong> tag(s) away.
+        </p>
+        <div v-if="normPreview.sample?.length" class="fc-name-grid mb-3">
+          <span
+            v-for="s in normPreview.sample" :key="s.to + s.kind"
+            class="fc-name"
+          >
+            <template v-if="s.merge">{{ s.from.join(' + ') }} → </template>{{ s.to }}
+          </span>
+        </div>
+        <v-btn
+          color="error" variant="flat" rounded="pill"
+          prepend-icon="mdi-format-letter-case"
+          :disabled="!normPreview.total_changes"
+          :loading="normCommitting"
+          @click="onNormCommit"
+        >Standardize {{ normPreview.total_changes }} tag group(s)</v-btn>
+        <span
+          v-if="normResult"
+          class="ml-3 text-caption"
+          :class="normResult === 'ok' ? 'text-success' : 'text-warning'"
+        >
+          <template v-if="normResult === 'ok'">Standardization complete ✓</template>
+          <template v-else>Finished with status: {{ normResult }}</template>
+        </span>
+      </div>
     </v-card-text>
   </v-card>
 </template>
@@ -155,6 +213,10 @@ const kindCommitting = ref(false)
 const resetPreview = ref(null)
 const loadingResetPreview = ref(false)
 const resetCommitting = ref(false)
+const normPreview = ref(null)
+const loadingNormPreview = ref(false)
+const normCommitting = ref(false)
+const normResult = ref(null)
 
 async function onPreview() {
   loadingPreview.value = true
@@ -212,6 +274,33 @@ async function onResetCommit() {
     resetCommitting.value = false
   }
 }
+
+async function onNormPreview() {
+  loadingNormPreview.value = true
+  normResult.value = null
+  try {
+    normPreview.value = await store.normalizeTags({ dryRun: true })
+  } finally {
+    loadingNormPreview.value = false
+  }
+}
+
+async function onNormCommit() {
+  normCommitting.value = true
+  normResult.value = null
+  try {
+    // Long op (FK repoints): enqueue the maintenance task, then tail the
+    // activity dashboard until its row reaches a terminal status. (The
+    // per-run summary dict isn't exposed by /activity/runs, so we surface
+    // the terminal status — the dry-run preview is the detailed view.)
+    const { task_id: taskId } = await store.normalizeTags({ dryRun: false })
+    const row = await store.pollTaskUntilDone(taskId)
+    normResult.value = row?.status || 'ok'
+    normPreview.value = { total_changes: 0, tags_to_rename: 0, collisions: 0, tags_to_merge: 0, sample: [] }
+  } finally {
+    normCommitting.value = false
+  }
+}
 </script>
 
 <style scoped>

frontend/src/stores/admin.js

@@ -142,6 +142,22 @@ export const useAdminStore = defineStore('admin', () => {
     }
   }
 
+  // #714: Title-Case the back-catalog + merge case/whitespace-variant tags.
+  // dry-run returns a projection inline; live returns {task_id} (long op —
+  // FK repoints) the caller polls via pollTaskUntilDone.
+  async function normalizeTags({ dryRun = true } = {}) {
+    lastError.value = null
+    try {
+      return await api.post(
+        '/api/admin/tags/normalize',
+        { body: { dry_run: dryRun } },
+      )
+    } catch (e) {
+      lastError.value = e.message
+      throw e
+    }
+  }
+
   // --- Task progress polling (taps FC-3i activity dashboard) --------
 
   /**
@@ -178,6 +194,7 @@ export const useAdminStore = defineStore('admin', () => {
     pruneUnusedTags,
     purgeLegacyTags,
     resetContentTagging,
+    normalizeTags,
     pollTaskUntilDone,
   }
 })

tests/test_tag_normalize.py

@@ -0,0 +1,189 @@
+"""#714: retro-normalize existing tags to the #701 canonical (Title Case +
+collapsed whitespace) and merge case/whitespace-variant duplicates.
+
+normalize_existing_tags commits per group, so case-variant duplicates are
+seeded with DIRECT inserts (find_or_create would case-insensitively dedup
+them, which is exactly the pre-#701 state we're cleaning up), and assertions
+re-query via fresh selects rather than touching post-commit ORM entities.
+"""
+import pytest
+from sqlalchemy import func, select
+
+from backend.app.models import Tag, TagKind, image_tag
+from backend.app.models.tag_alias import TagAlias
+from backend.app.services.tag_service import (
+    TagService,
+    normalize_existing_tags,
+)
+
+pytestmark = pytest.mark.integration
+
+_IMG_SEQ = 0
+
+
+async def _img(db):
+    """Minimal valid ImageRecord; returns its id."""
+    global _IMG_SEQ
+    _IMG_SEQ += 1
+    from backend.app.models import ImageRecord
+
+    rec = ImageRecord(
+        path=f"/tmp/fc_norm_{_IMG_SEQ}.png",
+        sha256=f"{_IMG_SEQ:064d}",
+        size_bytes=1,
+        mime="image/png",
+        origin="uploaded",
+    )
+    db.add(rec)
+    await db.flush()
+    return rec.id
+
+
+async def _raw_tag(db, name, kind, fandom_id=None):
+    """Insert a Tag row verbatim — bypasses find_or_create's case-insensitive
+    dedup so we can plant the case-variant duplicates #714 exists to fix."""
+    t = Tag(name=name, kind=kind, fandom_id=fandom_id)
+    db.add(t)
+    await db.flush()
+    return t
+
+
+async def _link(db, image_id, tag_id, source="manual"):
+    await db.execute(
+        image_tag.insert().values(
+            image_record_id=image_id, tag_id=tag_id, source=source
+        )
+    )
+
+
+async def _count_named(db, name, kind):
+    return await db.scalar(
+        select(func.count())
+        .select_from(Tag)
+        .where(Tag.name == name)
+        .where(Tag.kind == kind)
+    )
+
+
+@pytest.mark.asyncio
+async def test_dry_run_projection_counts(db):
+    # Collision group with an already-canonical member (→ no rename, 1 merge).
+    await _raw_tag(db, "hatsune miku", TagKind.character)
+    await _raw_tag(db, "Hatsune Miku", TagKind.character)
+    # Lone mis-cased + extra whitespace (→ rename, no merge).
+    await _raw_tag(db, "looking  at   viewer", TagKind.general)
+    # Already canonical (→ no change).
+    await _raw_tag(db, "Already Canonical", TagKind.general)
+
+    proj = await normalize_existing_tags(db, dry_run=True)
+
+    assert proj["collisions"] == 1
+    assert proj["tags_to_merge"] == 1
+    assert proj["tags_to_rename"] == 1  # only the whitespace group renames
+    assert proj["total_changes"] == 2
+    # Nothing mutated by a dry run.
+    assert await _count_named(db, "hatsune miku", TagKind.character) == 1
+
+
+@pytest.mark.asyncio
+async def test_live_merges_case_variants_and_repoints_images(db):
+    a = await _raw_tag(db, "hatsune miku", TagKind.character)
+    b = await _raw_tag(db, "HATSUNE MIKU", TagKind.character)
+    i1, i2, i3 = await _img(db), await _img(db), await _img(db)
+    await _link(db, i1, a.id)
+    await _link(db, i2, a.id)
+    await _link(db, i2, b.id)  # shared → must dedup on the survivor
+    await _link(db, i3, b.id)
+
+    summary = await normalize_existing_tags(db, dry_run=False)
+
+    assert summary["errors"] == 0
+    assert summary["merged"] == 1
+    assert summary["renamed"] == 1
+    assert summary["groups_processed"] == 1
+
+    # Exactly one canonical character tag survives; the variant is gone.
+    assert await _count_named(db, "Hatsune Miku", TagKind.character) == 1
+    total_chars = await db.scalar(
+        select(func.count()).select_from(Tag).where(Tag.kind == TagKind.character)
+    )
+    assert total_chars == 1
+    survivor_id = await db.scalar(
+        select(Tag.id)
+        .where(Tag.name == "Hatsune Miku")
+        .where(Tag.kind == TagKind.character)
+    )
+    # Union of {i1,i2} and {i2,i3} with the shared i2 deduped == 3.
+    assoc = await db.scalar(
+        select(func.count())
+        .select_from(image_tag)
+        .where(image_tag.c.tag_id == survivor_id)
+    )
+    assert assoc == 3
+
+
+@pytest.mark.asyncio
+async def test_idempotent_second_run_is_noop(db):
+    await _raw_tag(db, "kafka", TagKind.character)
+    await _raw_tag(db, "KAFKA", TagKind.character)
+
+    first = await normalize_existing_tags(db, dry_run=False)
+    assert first["groups_processed"] == 1
+
+    proj = await normalize_existing_tags(db, dry_run=True)
+    assert proj["total_changes"] == 0
+    again = await normalize_existing_tags(db, dry_run=False)
+    assert again["groups_processed"] == 0
+    assert again["merged"] == 0
+    assert again["renamed"] == 0
+
+
+@pytest.mark.asyncio
+async def test_same_name_different_fandom_not_merged(db):
+    f1 = await _raw_tag(db, "Bleach", TagKind.fandom)
+    f2 = await _raw_tag(db, "Naruto", TagKind.fandom)
+    await _raw_tag(db, "alice", TagKind.character, fandom_id=f1.id)
+    await _raw_tag(db, "Alice", TagKind.character, fandom_id=f2.id)
+
+    summary = await normalize_existing_tags(db, dry_run=False)
+
+    assert summary["merged"] == 0
+    assert summary["renamed"] == 1  # only the f1 "alice" recases
+    # Both characters survive — same canonical name, distinct fandoms.
+    assert await _count_named(db, "Alice", TagKind.character) == 2
+
+
+@pytest.mark.asyncio
+async def test_same_name_different_kind_not_merged(db):
+    await _raw_tag(db, "forge", TagKind.artist)
+    await _raw_tag(db, "Forge", TagKind.general)
+
+    summary = await normalize_existing_tags(db, dry_run=False)
+
+    assert summary["merged"] == 0
+    assert await _count_named(db, "Forge", TagKind.artist) == 1
+    assert await _count_named(db, "Forge", TagKind.general) == 1
+
+
+@pytest.mark.asyncio
+async def test_ml_known_loser_keeps_protective_alias(db):
+    # Survivor is the already-canonical (manual, ML-unknown) tag; the ML-sourced
+    # variant merges away and must leave an alias so the tagger re-resolves it.
+    canon = await _raw_tag(db, "Nemu", TagKind.general)
+    canon_id = canon.id  # capture before the service commits
+    ml = await _raw_tag(db, "nemu", TagKind.general)
+    img = await _img(db)
+    await _link(db, img, ml.id, source="ml_auto")
+
+    summary = await normalize_existing_tags(db, dry_run=False)
+
+    assert summary["merged"] == 1
+    assert summary["aliases_created"] == 1
+    assert await _count_named(db, "Nemu", TagKind.general) == 1
+    alias_rows = await db.scalar(
+        select(func.count())
+        .select_from(TagAlias)
+        .where(TagAlias.alias_string == "nemu")
+        .where(TagAlias.canonical_tag_id == canon_id)
+    )
+    assert alias_rows == 1