feat(ml): normalize Camie suggestion names to human-readable
CI / lint (push) Successful in 2s
CI / backend-lint-and-test (push) Failing after 24s
CI / frontend-build (push) Successful in 28s
CI / intimp (push) Successful in 3m57s
CI / intapi (push) Successful in 7m40s
CI / intcore (push) Successful in 8m22s

Camie's booru-style vocab strings (`uchiha_sasuke_(naruto)`,
`#unicus_(idolmaster)`, `1000-nen_ikiteru_(vocaloid)`, `:/`) were
surfacing raw in SuggestionsPanel — and worse, the SAME raw string was
written to tag.name on Accept, polluting the DB with `underscored_lowercase`
names that don't match the operator's "Title Case" tag convention.

Add backend/app/services/ml/tag_name.py with a single normalize()
applying nine rules (strip leading junk #/./+/;/~/_/ws, drop trailing
_(disambiguator) blocks iteratively, strip wrapping quotes, underscores
to spaces, space after colon, title-case each word's first char,
preserve hyphens/apostrophes/digits, drop entries with no letters).

Wire into SuggestionService.for_image:
- raw Camie key kept for alias_map lookup (alias rows are hand-curated
  against raw keys; don't disturb)
- display_name = normalize(raw); None means drop the candidate
- existing-tag lookup widened to case-insensitive match against BOTH
  raw and normalized forms so legacy underscore-named Tag rows accepted
  before this change still surface as "existing" not "+ new"
This commit is contained in:
2026-06-03 13:00:08 -04:00
parent f1860866de
commit a6e8d4b52e
4 changed files with 147 additions and 12 deletions
+27 -9
View File
@@ -4,7 +4,7 @@ threshold-filtered, category-grouped, ranked suggestions for one image.
from dataclasses import dataclass, field from dataclasses import dataclass, field
from sqlalchemy import select from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from ...models import ( from ...models import (
@@ -16,6 +16,7 @@ from ...models import (
from ...models.tag import image_tag from ...models.tag import image_tag
from .aliases import AliasService from .aliases import AliasService
from .centroids import CentroidService from .centroids import CentroidService
from .tag_name import normalize as normalize_tag_name
from .tagger import SURFACED_CATEGORIES from .tagger import SURFACED_CATEGORIES
@@ -84,7 +85,12 @@ class SuggestionService:
) )
# --- Camie predictions --- # --- Camie predictions ---
candidates: list[tuple[str, str, float]] = [] # candidates carry (raw_name, display_name, category, confidence).
# raw_name = the booru-formatted vocab key, kept for alias_map
# lookup since alias rows are hand-curated against raw keys.
# display_name = normalize_tag_name(raw_name) — what the operator
# sees AND what gets written to tag.name on Accept.
candidates: list[tuple[str, str, str, float]] = []
for name, p in predictions.items(): for name, p in predictions.items():
category = p.get("category", "general") category = p.get("category", "general")
if category not in SURFACED_CATEGORIES: if category not in SURFACED_CATEGORIES:
@@ -92,10 +98,14 @@ class SuggestionService:
conf = float(p.get("confidence", 0.0)) conf = float(p.get("confidence", 0.0))
if conf < self._threshold_for(settings, category): if conf < self._threshold_for(settings, category):
continue continue
candidates.append((name, category, conf)) display = normalize_tag_name(name)
if display is None:
# emoticon / pure-punctuation vocab entry — drop entirely
continue
candidates.append((name, display, category, conf))
alias_map = await self.aliases.resolve_many( alias_map = await self.aliases.resolve_many(
[(n, c) for n, c, _ in candidates] [(raw, c) for raw, _disp, c, _conf in candidates]
) )
merged: dict[object, Suggestion] = {} merged: dict[object, Suggestion] = {}
@@ -116,8 +126,8 @@ class SuggestionService:
creates_new_tag=existing.creates_new_tag, creates_new_tag=existing.creates_new_tag,
) )
for name, category, conf in candidates: for raw, display, category, conf in candidates:
canonical = alias_map.get((name, category)) canonical = alias_map.get((raw, category))
if canonical is not None: if canonical is not None:
if canonical.id in applied or canonical.id in rejected: if canonical.id in applied or canonical.id in rejected:
continue continue
@@ -133,9 +143,17 @@ class SuggestionService:
), ),
) )
else: else:
# Case-insensitive match on BOTH the raw camie key AND
# the normalized form — covers legacy underscore-named
# Tag rows accepted before normalization shipped, AND
# any tag the operator created with the human form.
existing_tag = ( existing_tag = (
await self.session.execute( await self.session.execute(
select(Tag).where(Tag.name == name) select(Tag).where(
func.lower(Tag.name).in_(
[raw.lower(), display.lower()]
)
)
) )
).scalars().first() ).scalars().first()
if existing_tag is not None: if existing_tag is not None:
@@ -157,10 +175,10 @@ class SuggestionService:
) )
else: else:
_merge( _merge(
f"raw:{name}:{category}", f"raw:{display}:{category}",
Suggestion( Suggestion(
canonical_tag_id=None, canonical_tag_id=None,
display_name=name, display_name=display,
category=category, category=category,
score=conf, score=conf,
source="tagger", source="tagger",
+61
View File
@@ -0,0 +1,61 @@
"""Camie vocabulary -> human-readable tag-name normalization.
Camie v2's ~57k tag vocabulary is booru-derived and arrives as raw
strings like `uchiha_sasuke_(naruto)`, `#unicus_(idolmaster)`,
`1000-nen_ikiteru_(vocaloid)`, or `:/`. We want the operator to see
"Uchiha Sasuke", "Unicus", "1000-Nen Ikiteru", or to never see the
emoticon at all — and we want the same clean string to be what lands
in `tag.name` when the suggestion is accepted, so Accept matches the
existing-tag convention (`tag_service.find_or_create`).
Rules (operator-approved 2026-06-03):
1. Strip leading junk chars (#, ., +, ;, ~, _, whitespace)
2. Drop trailing `_(disambiguator)` block(s), iteratively
3. Strip wrapping single/double quotes (after disambig removal so
`"foo_em_up"_(series)` -> `"foo_em_up"` -> `foo_em_up`)
4. Replace remaining `_` with space; collapse runs of whitespace
5. Add a space after any `:` (namespace:tag -> namespace: tag)
6. Preserve hyphens (booru hyphens often carry meaning)
7. Title-case each space-separated word (first character only —
apostrophes, digits, hyphens stay)
8. If no letters remain, return None (drop emoticons like `:/`)
9. No surname/givenname swap — no reliable signal in the vocab
"""
import re
_LEADING_JUNK = re.compile(r"^[#.+;~_\s]+")
_TRAILING_DISAMBIG = re.compile(r"_\([^)]*\)\s*$")
_MULTISPACE = re.compile(r"\s+")
_COLON_NOSPACE = re.compile(r":(?=\S)")
_HAS_LETTER = re.compile(r"[A-Za-z]")
def _strip_wrapping_quotes(s: str) -> str:
if len(s) >= 2 and s[0] == s[-1] and s[0] in ('"', "'"):
return s[1:-1]
return s
def _title_word(w: str) -> str:
return w[:1].upper() + w[1:] if w else w
def normalize(raw: str) -> str | None:
"""Return the human-readable form of a raw Camie tag, or None if the
string is junk (emoticon, empty after stripping)."""
if not raw:
return None
s = _LEADING_JUNK.sub("", raw)
while True:
new = _TRAILING_DISAMBIG.sub("", s)
if new == s:
break
s = new
s = _strip_wrapping_quotes(s)
s = s.replace("_", " ")
s = _COLON_NOSPACE.sub(": ", s)
s = _MULTISPACE.sub(" ", s).strip()
if not s or not _HAS_LETTER.search(s):
return None
return " ".join(_title_word(w) for w in s.split(" "))
+6 -3
View File
@@ -39,8 +39,9 @@ async def test_threshold_filters_low_confidence_general(db):
await db.flush() await db.flush()
sl = await SuggestionService(db).for_image(img.id) sl = await SuggestionService(db).for_image(img.id)
names = [s.display_name for s in sl.by_category.get("general", [])] names = [s.display_name for s in sl.by_category.get("general", [])]
assert "sword" in names # display_name is normalized (tag_name.normalize) before surfacing.
assert "lowconf" not in names assert "Sword" in names
assert "Lowconf" not in names
@pytest.mark.asyncio @pytest.mark.asyncio
@@ -84,7 +85,9 @@ async def test_raw_tag_creates_new(db):
await db.flush() await db.flush()
sl = await SuggestionService(db).for_image(img.id) sl = await SuggestionService(db).for_image(img.id)
chars = sl.by_category["character"] chars = sl.by_category["character"]
assert chars[0].display_name == "brand_new_tag" # display_name is the normalized Camie name (underscores -> spaces,
# title-cased), not the raw vocab key.
assert chars[0].display_name == "Brand New Tag"
assert chars[0].creates_new_tag is True assert chars[0].creates_new_tag is True
assert chars[0].canonical_tag_id is None assert chars[0].canonical_tag_id is None
+53
View File
@@ -0,0 +1,53 @@
import pytest
from backend.app.services.ml.tag_name import normalize
@pytest.mark.parametrize(
"raw, expected",
[
# Rule 4: underscores -> spaces; rule 7: title case
("light_purple_hair", "Light Purple Hair"),
("no_pants", "No Pants"),
("year_2005", "Year 2005"),
# Single-word still title-cased
("sword", "Sword"),
# Rule 3: drop trailing _(disambiguator)
("uchiha_sasuke_(naruto)", "Uchiha Sasuke"),
("apple_(fruit)", "Apple"),
("kirby_(series)", "Kirby"),
# Repeated trailing disambig blocks
("foo_(bar)_(baz)", "Foo"),
# Rule 1: leading junk chars
("#unicus_(idolmaster)", "Unicus"),
(".52_gal_(splatoon)", "52 Gal"),
("+_+_smile_(emote)", "Smile"),
# Rule 5: space after colon
("nier:automata", "Nier: Automata"),
# Already-spaced colon left alone
("nier: automata", "Nier: Automata"),
# Rule 6: hyphens preserved
("1000-nen_ikiteru_(vocaloid)", "1000-nen Ikiteru"),
("well-known_face", "Well-known Face"),
# Rule 2: wrapping quotes
('"pile_em_up"_(genshin_impact)', "Pile Em Up"),
("'foo_bar'", "Foo Bar"),
# Rule 8: emoticons -> None
(":/", None),
(";)", None),
("+_+", None),
("^_^", None),
# Empty / whitespace-only
("", None),
(" ", None),
("___", None),
# Apostrophe inside word — preserved, not title-cased
("it's_okay", "It's Okay"),
# Digit-only still surfaces (year tags)
("2005", "2005"),
# Multi-space collapse
("foo___bar", "Foo Bar"),
],
)
def test_normalize(raw, expected):
assert normalize(raw) == expected