feat(ml): normalize Camie suggestion names to human-readable
Camie's booru-style vocab strings (`uchiha_sasuke_(naruto)`, `#unicus_(idolmaster)`, `1000-nen_ikiteru_(vocaloid)`, `:/`) were surfacing raw in SuggestionsPanel — and worse, the SAME raw string was written to tag.name on Accept, polluting the DB with `underscored_lowercase` names that don't match the operator's "Title Case" tag convention. Add backend/app/services/ml/tag_name.py with a single normalize() applying nine rules (strip leading junk #/./+/;/~/_/ws, drop trailing _(disambiguator) blocks iteratively, strip wrapping quotes, underscores to spaces, space after colon, title-case each word's first char, preserve hyphens/apostrophes/digits, drop entries with no letters). Wire into SuggestionService.for_image: - raw Camie key kept for alias_map lookup (alias rows are hand-curated against raw keys; don't disturb) - display_name = normalize(raw); None means drop the candidate - existing-tag lookup widened to case-insensitive match against BOTH raw and normalized forms so legacy underscore-named Tag rows accepted before this change still surface as "existing" not "+ new"
This commit is contained in:
@@ -4,7 +4,7 @@ threshold-filtered, category-grouped, ranked suggestions for one image.
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ...models import (
|
||||
@@ -16,6 +16,7 @@ from ...models import (
|
||||
from ...models.tag import image_tag
|
||||
from .aliases import AliasService
|
||||
from .centroids import CentroidService
|
||||
from .tag_name import normalize as normalize_tag_name
|
||||
from .tagger import SURFACED_CATEGORIES
|
||||
|
||||
|
||||
@@ -84,7 +85,12 @@ class SuggestionService:
|
||||
)
|
||||
|
||||
# --- Camie predictions ---
|
||||
candidates: list[tuple[str, str, float]] = []
|
||||
# candidates carry (raw_name, display_name, category, confidence).
|
||||
# raw_name = the booru-formatted vocab key, kept for alias_map
|
||||
# lookup since alias rows are hand-curated against raw keys.
|
||||
# display_name = normalize_tag_name(raw_name) — what the operator
|
||||
# sees AND what gets written to tag.name on Accept.
|
||||
candidates: list[tuple[str, str, str, float]] = []
|
||||
for name, p in predictions.items():
|
||||
category = p.get("category", "general")
|
||||
if category not in SURFACED_CATEGORIES:
|
||||
@@ -92,10 +98,14 @@ class SuggestionService:
|
||||
conf = float(p.get("confidence", 0.0))
|
||||
if conf < self._threshold_for(settings, category):
|
||||
continue
|
||||
candidates.append((name, category, conf))
|
||||
display = normalize_tag_name(name)
|
||||
if display is None:
|
||||
# emoticon / pure-punctuation vocab entry — drop entirely
|
||||
continue
|
||||
candidates.append((name, display, category, conf))
|
||||
|
||||
alias_map = await self.aliases.resolve_many(
|
||||
[(n, c) for n, c, _ in candidates]
|
||||
[(raw, c) for raw, _disp, c, _conf in candidates]
|
||||
)
|
||||
|
||||
merged: dict[object, Suggestion] = {}
|
||||
@@ -116,8 +126,8 @@ class SuggestionService:
|
||||
creates_new_tag=existing.creates_new_tag,
|
||||
)
|
||||
|
||||
for name, category, conf in candidates:
|
||||
canonical = alias_map.get((name, category))
|
||||
for raw, display, category, conf in candidates:
|
||||
canonical = alias_map.get((raw, category))
|
||||
if canonical is not None:
|
||||
if canonical.id in applied or canonical.id in rejected:
|
||||
continue
|
||||
@@ -133,9 +143,17 @@ class SuggestionService:
|
||||
),
|
||||
)
|
||||
else:
|
||||
# Case-insensitive match on BOTH the raw camie key AND
|
||||
# the normalized form — covers legacy underscore-named
|
||||
# Tag rows accepted before normalization shipped, AND
|
||||
# any tag the operator created with the human form.
|
||||
existing_tag = (
|
||||
await self.session.execute(
|
||||
select(Tag).where(Tag.name == name)
|
||||
select(Tag).where(
|
||||
func.lower(Tag.name).in_(
|
||||
[raw.lower(), display.lower()]
|
||||
)
|
||||
)
|
||||
)
|
||||
).scalars().first()
|
||||
if existing_tag is not None:
|
||||
@@ -157,10 +175,10 @@ class SuggestionService:
|
||||
)
|
||||
else:
|
||||
_merge(
|
||||
f"raw:{name}:{category}",
|
||||
f"raw:{display}:{category}",
|
||||
Suggestion(
|
||||
canonical_tag_id=None,
|
||||
display_name=name,
|
||||
display_name=display,
|
||||
category=category,
|
||||
score=conf,
|
||||
source="tagger",
|
||||
|
||||
Reference in New Issue
Block a user