feat(ml): normalize Camie suggestion names to human-readable
CI / lint (push) Successful in 2s
CI / backend-lint-and-test (push) Failing after 24s
CI / frontend-build (push) Successful in 28s
CI / intimp (push) Successful in 3m57s
CI / intapi (push) Successful in 7m40s
CI / intcore (push) Successful in 8m22s

Camie's booru-style vocab strings (`uchiha_sasuke_(naruto)`,
`#unicus_(idolmaster)`, `1000-nen_ikiteru_(vocaloid)`, `:/`) were
surfacing raw in SuggestionsPanel — and worse, the SAME raw string was
written to tag.name on Accept, polluting the DB with `underscored_lowercase`
names that don't match the operator's "Title Case" tag convention.

Add backend/app/services/ml/tag_name.py with a single normalize()
applying nine rules (strip leading junk #/./+/;/~/_/ws, drop trailing
_(disambiguator) blocks iteratively, strip wrapping quotes, underscores
to spaces, space after colon, title-case each word's first char,
preserve hyphens/apostrophes/digits, drop entries with no letters).

Wire into SuggestionService.for_image:
- raw Camie key kept for alias_map lookup (alias rows are hand-curated
  against raw keys; don't disturb)
- display_name = normalize(raw); None means drop the candidate
- existing-tag lookup widened to case-insensitive match against BOTH
  raw and normalized forms so legacy underscore-named Tag rows accepted
  before this change still surface as "existing" not "+ new"
This commit is contained in:
2026-06-03 13:00:08 -04:00
parent f1860866de
commit a6e8d4b52e
4 changed files with 147 additions and 12 deletions
+27 -9
View File
@@ -4,7 +4,7 @@ threshold-filtered, category-grouped, ranked suggestions for one image.
from dataclasses import dataclass, field
from sqlalchemy import select
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from ...models import (
@@ -16,6 +16,7 @@ from ...models import (
from ...models.tag import image_tag
from .aliases import AliasService
from .centroids import CentroidService
from .tag_name import normalize as normalize_tag_name
from .tagger import SURFACED_CATEGORIES
@@ -84,7 +85,12 @@ class SuggestionService:
)
# --- Camie predictions ---
candidates: list[tuple[str, str, float]] = []
# candidates carry (raw_name, display_name, category, confidence).
# raw_name = the booru-formatted vocab key, kept for alias_map
# lookup since alias rows are hand-curated against raw keys.
# display_name = normalize_tag_name(raw_name) — what the operator
# sees AND what gets written to tag.name on Accept.
candidates: list[tuple[str, str, str, float]] = []
for name, p in predictions.items():
category = p.get("category", "general")
if category not in SURFACED_CATEGORIES:
@@ -92,10 +98,14 @@ class SuggestionService:
conf = float(p.get("confidence", 0.0))
if conf < self._threshold_for(settings, category):
continue
candidates.append((name, category, conf))
display = normalize_tag_name(name)
if display is None:
# emoticon / pure-punctuation vocab entry — drop entirely
continue
candidates.append((name, display, category, conf))
alias_map = await self.aliases.resolve_many(
[(n, c) for n, c, _ in candidates]
[(raw, c) for raw, _disp, c, _conf in candidates]
)
merged: dict[object, Suggestion] = {}
@@ -116,8 +126,8 @@ class SuggestionService:
creates_new_tag=existing.creates_new_tag,
)
for name, category, conf in candidates:
canonical = alias_map.get((name, category))
for raw, display, category, conf in candidates:
canonical = alias_map.get((raw, category))
if canonical is not None:
if canonical.id in applied or canonical.id in rejected:
continue
@@ -133,9 +143,17 @@ class SuggestionService:
),
)
else:
# Case-insensitive match on BOTH the raw camie key AND
# the normalized form — covers legacy underscore-named
# Tag rows accepted before normalization shipped, AND
# any tag the operator created with the human form.
existing_tag = (
await self.session.execute(
select(Tag).where(Tag.name == name)
select(Tag).where(
func.lower(Tag.name).in_(
[raw.lower(), display.lower()]
)
)
)
).scalars().first()
if existing_tag is not None:
@@ -157,10 +175,10 @@ class SuggestionService:
)
else:
_merge(
f"raw:{name}:{category}",
f"raw:{display}:{category}",
Suggestion(
canonical_tag_id=None,
display_name=name,
display_name=display,
category=category,
score=conf,
source="tagger",
+61
View File
@@ -0,0 +1,61 @@
"""Camie vocabulary -> human-readable tag-name normalization.
Camie v2's ~57k tag vocabulary is booru-derived and arrives as raw
strings like `uchiha_sasuke_(naruto)`, `#unicus_(idolmaster)`,
`1000-nen_ikiteru_(vocaloid)`, or `:/`. We want the operator to see
"Uchiha Sasuke", "Unicus", "1000-Nen Ikiteru", or to never see the
emoticon at all — and we want the same clean string to be what lands
in `tag.name` when the suggestion is accepted, so Accept matches the
existing-tag convention (`tag_service.find_or_create`).
Rules (operator-approved 2026-06-03):
1. Strip leading junk chars (#, ., +, ;, ~, _, whitespace)
2. Drop trailing `_(disambiguator)` block(s), iteratively
3. Strip wrapping single/double quotes (after disambig removal so
`"foo_em_up"_(series)` -> `"foo_em_up"` -> `foo_em_up`)
4. Replace remaining `_` with space; collapse runs of whitespace
5. Add a space after any `:` (namespace:tag -> namespace: tag)
6. Preserve hyphens (booru hyphens often carry meaning)
7. Title-case each space-separated word (first character only —
apostrophes, digits, hyphens stay)
8. If no letters remain, return None (drop emoticons like `:/`)
9. No surname/givenname swap — no reliable signal in the vocab
"""
import re
_LEADING_JUNK = re.compile(r"^[#.+;~_\s]+")
_TRAILING_DISAMBIG = re.compile(r"_\([^)]*\)\s*$")
_MULTISPACE = re.compile(r"\s+")
_COLON_NOSPACE = re.compile(r":(?=\S)")
_HAS_LETTER = re.compile(r"[A-Za-z]")
def _strip_wrapping_quotes(s: str) -> str:
if len(s) >= 2 and s[0] == s[-1] and s[0] in ('"', "'"):
return s[1:-1]
return s
def _title_word(w: str) -> str:
return w[:1].upper() + w[1:] if w else w
def normalize(raw: str) -> str | None:
"""Return the human-readable form of a raw Camie tag, or None if the
string is junk (emoticon, empty after stripping)."""
if not raw:
return None
s = _LEADING_JUNK.sub("", raw)
while True:
new = _TRAILING_DISAMBIG.sub("", s)
if new == s:
break
s = new
s = _strip_wrapping_quotes(s)
s = s.replace("_", " ")
s = _COLON_NOSPACE.sub(": ", s)
s = _MULTISPACE.sub(" ", s).strip()
if not s or not _HAS_LETTER.search(s):
return None
return " ".join(_title_word(w) for w in s.split(" "))
+6 -3
View File
@@ -39,8 +39,9 @@ async def test_threshold_filters_low_confidence_general(db):
await db.flush()
sl = await SuggestionService(db).for_image(img.id)
names = [s.display_name for s in sl.by_category.get("general", [])]
assert "sword" in names
assert "lowconf" not in names
# display_name is normalized (tag_name.normalize) before surfacing.
assert "Sword" in names
assert "Lowconf" not in names
@pytest.mark.asyncio
@@ -84,7 +85,9 @@ async def test_raw_tag_creates_new(db):
await db.flush()
sl = await SuggestionService(db).for_image(img.id)
chars = sl.by_category["character"]
assert chars[0].display_name == "brand_new_tag"
# display_name is the normalized Camie name (underscores -> spaces,
# title-cased), not the raw vocab key.
assert chars[0].display_name == "Brand New Tag"
assert chars[0].creates_new_tag is True
assert chars[0].canonical_tag_id is None
+53
View File
@@ -0,0 +1,53 @@
import pytest
from backend.app.services.ml.tag_name import normalize
@pytest.mark.parametrize(
"raw, expected",
[
# Rule 4: underscores -> spaces; rule 7: title case
("light_purple_hair", "Light Purple Hair"),
("no_pants", "No Pants"),
("year_2005", "Year 2005"),
# Single-word still title-cased
("sword", "Sword"),
# Rule 3: drop trailing _(disambiguator)
("uchiha_sasuke_(naruto)", "Uchiha Sasuke"),
("apple_(fruit)", "Apple"),
("kirby_(series)", "Kirby"),
# Repeated trailing disambig blocks
("foo_(bar)_(baz)", "Foo"),
# Rule 1: leading junk chars
("#unicus_(idolmaster)", "Unicus"),
(".52_gal_(splatoon)", "52 Gal"),
("+_+_smile_(emote)", "Smile"),
# Rule 5: space after colon
("nier:automata", "Nier: Automata"),
# Already-spaced colon left alone
("nier: automata", "Nier: Automata"),
# Rule 6: hyphens preserved
("1000-nen_ikiteru_(vocaloid)", "1000-nen Ikiteru"),
("well-known_face", "Well-known Face"),
# Rule 2: wrapping quotes
('"pile_em_up"_(genshin_impact)', "Pile Em Up"),
("'foo_bar'", "Foo Bar"),
# Rule 8: emoticons -> None
(":/", None),
(";)", None),
("+_+", None),
("^_^", None),
# Empty / whitespace-only
("", None),
(" ", None),
("___", None),
# Apostrophe inside word — preserved, not title-cased
("it's_okay", "It's Okay"),
# Digit-only still surfaces (year tags)
("2005", "2005"),
# Multi-space collapse
("foo___bar", "Foo Bar"),
],
)
def test_normalize(raw, expected):
assert normalize(raw) == expected