Merge pull request 'feat(ml): normalize Camie suggestion names to human-readable' (#56) from dev into main

backend/app/services/ml/suggestions.py

@@ -4,7 +4,7 @@ threshold-filtered, category-grouped, ranked suggestions for one image.
 
 from dataclasses import dataclass, field
 
-from sqlalchemy import select
+from sqlalchemy import func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from ...models import (
@@ -16,6 +16,7 @@ from ...models import (
 from ...models.tag import image_tag
 from .aliases import AliasService
 from .centroids import CentroidService
+from .tag_name import normalize as normalize_tag_name
 from .tagger import SURFACED_CATEGORIES
 
 
@@ -84,7 +85,12 @@ class SuggestionService:
         )
 
         # --- Camie predictions ---
-        candidates: list[tuple[str, str, float]] = []
+        # candidates carry (raw_name, display_name, category, confidence).
+        # raw_name = the booru-formatted vocab key, kept for alias_map
+        # lookup since alias rows are hand-curated against raw keys.
+        # display_name = normalize_tag_name(raw_name) — what the operator
+        # sees AND what gets written to tag.name on Accept.
+        candidates: list[tuple[str, str, str, float]] = []
         for name, p in predictions.items():
             category = p.get("category", "general")
             if category not in SURFACED_CATEGORIES:
@@ -92,10 +98,14 @@ class SuggestionService:
             conf = float(p.get("confidence", 0.0))
             if conf < self._threshold_for(settings, category):
                 continue
-            candidates.append((name, category, conf))
+            display = normalize_tag_name(name)
+            if display is None:
+                # emoticon / pure-punctuation vocab entry — drop entirely
+                continue
+            candidates.append((name, display, category, conf))
 
         alias_map = await self.aliases.resolve_many(
-            [(n, c) for n, c, _ in candidates]
+            [(raw, c) for raw, _disp, c, _conf in candidates]
         )
 
         merged: dict[object, Suggestion] = {}
@@ -116,8 +126,8 @@ class SuggestionService:
                     creates_new_tag=existing.creates_new_tag,
                 )
 
-        for name, category, conf in candidates:
-            canonical = alias_map.get((name, category))
+        for raw, display, category, conf in candidates:
+            canonical = alias_map.get((raw, category))
             if canonical is not None:
                 if canonical.id in applied or canonical.id in rejected:
                     continue
@@ -133,9 +143,17 @@ class SuggestionService:
                     ),
                 )
             else:
+                # Case-insensitive match on BOTH the raw camie key AND
+                # the normalized form — covers legacy underscore-named
+                # Tag rows accepted before normalization shipped, AND
+                # any tag the operator created with the human form.
                 existing_tag = (
                     await self.session.execute(
-                        select(Tag).where(Tag.name == name)
+                        select(Tag).where(
+                            func.lower(Tag.name).in_(
+                                [raw.lower(), display.lower()]
+                            )
+                        )
                     )
                 ).scalars().first()
                 if existing_tag is not None:
@@ -157,10 +175,10 @@ class SuggestionService:
                     )
                 else:
                     _merge(
-                        f"raw:{name}:{category}",
+                        f"raw:{display}:{category}",
                         Suggestion(
                             canonical_tag_id=None,
-                            display_name=name,
+                            display_name=display,
                             category=category,
                             score=conf,
                             source="tagger",

backend/app/services/ml/tag_name.py

@@ -0,0 +1,62 @@
+"""Camie vocabulary -> human-readable tag-name normalization.
+
+Camie v2's ~57k tag vocabulary is booru-derived and arrives as raw
+strings like `uchiha_sasuke_(naruto)`, `#unicus_(idolmaster)`,
+`1000-nen_ikiteru_(vocaloid)`, or `:/`. We want the operator to see
+"Uchiha Sasuke", "Unicus", "1000-Nen Ikiteru", or to never see the
+emoticon at all — and we want the same clean string to be what lands
+in `tag.name` when the suggestion is accepted, so Accept matches the
+existing-tag convention (`tag_service.find_or_create`).
+
+Rules (operator-approved 2026-06-03):
+  1. Strip leading junk chars (#, ., +, ;, ~, _, whitespace)
+  2. Drop trailing `_(disambiguator)` block(s), iteratively
+  3. Strip wrapping single/double quotes (after disambig removal so
+     `"foo_em_up"_(series)` -> `"foo_em_up"` -> `foo_em_up`)
+  4. Replace remaining `_` with space; collapse runs of whitespace
+  5. Add a space after any `:` (namespace:tag -> namespace: tag)
+  6. Preserve hyphens (booru hyphens often carry meaning)
+  7. Title-case each space-separated word (first character only —
+     apostrophes, digits, hyphens stay)
+  8. If no letters AND no digits remain, return None (drops emoticons
+     like `:/` or `^_^`; preserves bare digit tags like `2005`)
+  9. No surname/givenname swap — no reliable signal in the vocab
+"""
+
+import re
+
+_LEADING_JUNK = re.compile(r"^[#.+;~_\s]+")
+_TRAILING_DISAMBIG = re.compile(r"_\([^)]*\)\s*$")
+_MULTISPACE = re.compile(r"\s+")
+_COLON_NOSPACE = re.compile(r":(?=\S)")
+_HAS_ALPHANUMERIC = re.compile(r"[A-Za-z0-9]")
+
+
+def _strip_wrapping_quotes(s: str) -> str:
+    if len(s) >= 2 and s[0] == s[-1] and s[0] in ('"', "'"):
+        return s[1:-1]
+    return s
+
+
+def _title_word(w: str) -> str:
+    return w[:1].upper() + w[1:] if w else w
+
+
+def normalize(raw: str) -> str | None:
+    """Return the human-readable form of a raw Camie tag, or None if the
+    string is junk (emoticon, empty after stripping)."""
+    if not raw:
+        return None
+    s = _LEADING_JUNK.sub("", raw)
+    while True:
+        new = _TRAILING_DISAMBIG.sub("", s)
+        if new == s:
+            break
+        s = new
+    s = _strip_wrapping_quotes(s)
+    s = s.replace("_", " ")
+    s = _COLON_NOSPACE.sub(": ", s)
+    s = _MULTISPACE.sub(" ", s).strip()
+    if not s or not _HAS_ALPHANUMERIC.search(s):
+        return None
+    return " ".join(_title_word(w) for w in s.split(" "))

tests/test_ml_suggestions.py

@@ -39,8 +39,9 @@ async def test_threshold_filters_low_confidence_general(db):
     await db.flush()
     sl = await SuggestionService(db).for_image(img.id)
     names = [s.display_name for s in sl.by_category.get("general", [])]
-    assert "sword" in names
-    assert "lowconf" not in names
+    # display_name is normalized (tag_name.normalize) before surfacing.
+    assert "Sword" in names
+    assert "Lowconf" not in names
 
 
 @pytest.mark.asyncio
@@ -84,7 +85,9 @@ async def test_raw_tag_creates_new(db):
     await db.flush()
     sl = await SuggestionService(db).for_image(img.id)
     chars = sl.by_category["character"]
-    assert chars[0].display_name == "brand_new_tag"
+    # display_name is the normalized Camie name (underscores -> spaces,
+    # title-cased), not the raw vocab key.
+    assert chars[0].display_name == "Brand New Tag"
     assert chars[0].creates_new_tag is True
     assert chars[0].canonical_tag_id is None
 

tests/test_ml_tag_name.py

@@ -0,0 +1,53 @@
+import pytest
+
+from backend.app.services.ml.tag_name import normalize
+
+
+@pytest.mark.parametrize(
+    "raw, expected",
+    [
+        # Rule 4: underscores -> spaces; rule 7: title case
+        ("light_purple_hair", "Light Purple Hair"),
+        ("no_pants", "No Pants"),
+        ("year_2005", "Year 2005"),
+        # Single-word still title-cased
+        ("sword", "Sword"),
+        # Rule 3: drop trailing _(disambiguator)
+        ("uchiha_sasuke_(naruto)", "Uchiha Sasuke"),
+        ("apple_(fruit)", "Apple"),
+        ("kirby_(series)", "Kirby"),
+        # Repeated trailing disambig blocks
+        ("foo_(bar)_(baz)", "Foo"),
+        # Rule 1: leading junk chars
+        ("#unicus_(idolmaster)", "Unicus"),
+        (".52_gal_(splatoon)", "52 Gal"),
+        ("+_+_smile_(emote)", "Smile"),
+        # Rule 5: space after colon
+        ("nier:automata", "Nier: Automata"),
+        # Already-spaced colon left alone
+        ("nier: automata", "Nier: Automata"),
+        # Rule 6: hyphens preserved
+        ("1000-nen_ikiteru_(vocaloid)", "1000-nen Ikiteru"),
+        ("well-known_face", "Well-known Face"),
+        # Rule 2: wrapping quotes
+        ('"pile_em_up"_(genshin_impact)', "Pile Em Up"),
+        ("'foo_bar'", "Foo Bar"),
+        # Rule 8: emoticons -> None
+        (":/", None),
+        (";)", None),
+        ("+_+", None),
+        ("^_^", None),
+        # Empty / whitespace-only
+        ("", None),
+        ("   ", None),
+        ("___", None),
+        # Apostrophe inside word — preserved, not title-cased
+        ("it's_okay", "It's Okay"),
+        # Digit-only still surfaces (year tags)
+        ("2005", "2005"),
+        # Multi-space collapse
+        ("foo___bar", "Foo Bar"),
+    ],
+)
+def test_normalize(raw, expected):
+    assert normalize(raw) == expected