bvandeusen/FabledCurator
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 35 Wiki Activity

refactor(ml): retire the Camie tagger + allowlist bulk-apply (#1189)
CI / lint (push) Failing after 3s
Details
CI / frontend-build (push) Successful in 18s
Details
CI / backend-lint-and-test (push) Successful in 28s
Details
CI / integration (push) Successful in 3m27s
Details

Browse Source 
Heads + CCIP are the tag source and head auto-apply is the earned propagation.
The Camie tagger ran only to feed the allowlist bulk-apply (its ImagePrediction
rows had no other consumer), and the allowlist was a SECOND, un-earned auto-apply
path firing in parallel with heads on every accept — exactly the un-earned spray
the v2 pivot replaced. Retire both.

Behavior change: accepting a suggestion now applies the tag to THAT image only
(source='ml_accepted', a head-training positive) — it no longer allowlists +
fans the tag across the library via Camie. Propagation is heads' earned
auto-apply. (Loses instant cold-start propagation for booru-vocab tags; that was
un-earned and bypassed the precision gate.)

- tag_and_embed is now EMBED-ONLY (no Camie load/infer, no ImagePrediction
  writes); backfill enqueues it for images with no embedding.
- Removed: services/ml/tagger.py, apply_allowlist_tags + helpers + daily beat +
  every enqueue caller (accept/alias/merge/per-image), api/allowlist.py +
  blueprint, ImagePrediction + TagAllowlist models/tables (migration 0067),
  AllowlistTable.vue + allowlist store, the accept coverage-projection payload.
- AllowlistService gutted to accept/dismiss/undismiss/reject (the rejection store
  the rail still needs); accept returns nothing, API returns {accepted, tag_id}.
- tag merge no longer repoints/triggers the allowlist; _keep_as_alias now keys on
  ML-applied image_tag sources (incl. head_auto) instead of the allowlist.
- UI: MLBackfillCard relabelled to embedding-only; accept toast simplified;
  MaintenancePanel drops the allowlist tile.

Left for a follow-up hygiene pass (now-inert, harmless): the dead settings
columns (tagger_store_floor, tagger_model_version, suggestion_threshold_*,
video_min_tag_frames), image_record.tagger_model_version, MLThresholdSliders
trim, and the Camie model download in download_models.py.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
This commit is contained in:
Bryan Van Deusen
2026-06-30 13:04:31 -04:00
parent 3d77a38a25
commit 485387ff0b
31 changed files with 159 additions and 1710 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/importer.py
-10
View File
@@ -1479,16 +1479,6 @@ class Importer:
existing.siglip_embedding = None
existing.siglip_model_version = None
existing.centroid_scores = None
# #768: predictions also live in the normalized image_prediction table
# now — clear them so a re-imported file re-derives a fresh set.
from sqlalchemy import delete as _delete
from ..models import ImagePrediction as _ImagePrediction
self.session.execute(
_delete(_ImagePrediction).where(
_ImagePrediction.image_record_id == existing.id
)
)
# created_at intentionally preserved; updated_at auto-bumps.
self.session.flush()
self.session.commit()
backend/app/services/ml/allowlist.py
+17 -152
View File
@@ -1,36 +1,20 @@
"""Allowlist semantics: accepting a suggestion adds the canonical tag to
image_tag AND to tag_allowlist; per-image removal/dismiss writes a rejection.
"""Suggestion actions: accept applies the canonical tag to an image (which
feeds head training); dismiss / reject record a per-image rejection.
(The Camie allowlist bulk-apply was retired #1189 — heads + CCIP are the tag
source, and head auto-apply is the earned propagation. Accept no longer
allowlists or fans a tag out across the library.)
"""
from collections.abc import Sequence
from dataclasses import dataclass
from sqlalchemy import and_, delete, distinct, func, or_, select
from sqlalchemy import delete
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from ...models import (
ImagePrediction,
MLSettings,
Tag,
TagAlias,
TagAllowlist,
TagSuggestionRejection,
)
from ...models import TagSuggestionRejection
from ...models.tag import image_tag
from .aliases import AliasService
@dataclass(frozen=True)
class AllowlistRow:
tag_id: int
tag_name: str
tag_kind: str
min_confidence: float
applied_count: int # image_tag rows currently carrying this tag
coverage_count: int # images a sweep WOULD cover at min_confidence
class AllowlistService:
def __init__(self, session: AsyncSession):
self.session = session
@@ -39,21 +23,11 @@ class AllowlistService:
async def _apply_image_tag(self, image_id: int, tag_id: int, source: str):
stmt = insert(image_tag).values(
image_record_id=image_id, tag_id=tag_id, source=source
)
stmt = stmt.on_conflict_do_nothing(
).on_conflict_do_nothing(
index_elements=["image_record_id", "tag_id"]
)
await self.session.execute(stmt)
async def _add_to_allowlist(self, tag_id: int) -> bool:
"""Returns True if newly added (caller should kick off retro-apply)."""
exists = await self.session.get(TagAllowlist, tag_id)
if exists is not None:
return False
self.session.add(TagAllowlist(tag_id=tag_id))
await self.session.flush()
return True
async def _clear_rejection(self, image_id: int, tag_id: int):
await self.session.execute(
delete(TagSuggestionRejection)
@@ -61,12 +35,11 @@ class AllowlistService:
.where(TagSuggestionRejection.tag_id == tag_id)
)
async def accept(self, image_id: int, tag_id: int) -> bool:
"""Accept a suggestion. Returns True if the tag was newly added to
the allowlist (the API layer enqueues apply_allowlist_tags then)."""
async def accept(self, image_id: int, tag_id: int) -> None:
"""Apply the accepted tag to this image (source='ml_accepted', a head
training positive) and clear any prior rejection."""
await self._apply_image_tag(image_id, tag_id, source="ml_accepted")
await self._clear_rejection(image_id, tag_id)
return await self._add_to_allowlist(tag_id)
async def add_alias_and_accept(
self,
@@ -74,17 +47,16 @@ class AllowlistService:
alias_string: str,
alias_category: str,
canonical_tag_id: int,
) -> bool:
) -> None:
await self.aliases.create(
alias_string, alias_category, canonical_tag_id
)
return await self.accept(image_id, canonical_tag_id)
await self.accept(image_id, canonical_tag_id)
async def dismiss(self, image_id: int, tag_id: int) -> None:
stmt = insert(TagSuggestionRejection).values(
image_record_id=image_id, tag_id=tag_id
)
stmt = stmt.on_conflict_do_nothing(
).on_conflict_do_nothing(
index_elements=["image_record_id", "tag_id"]
)
await self.session.execute(stmt)
@@ -96,118 +68,11 @@ class AllowlistService:
await self._clear_rejection(image_id, tag_id)
async def reject_applied_tag(self, image_id: int, tag_id: int) -> None:
"""Operator removed an applied tag from an image. Remove the
image_tag row AND record a rejection so the allowlist won't
re-apply it on the next maintenance sweep."""
"""Operator removed an applied tag from an image. Remove the image_tag
row AND record a rejection so head auto-apply won't re-apply it."""
await self.session.execute(
image_tag.delete()
.where(image_tag.c.image_record_id == image_id)
.where(image_tag.c.tag_id == tag_id)
)
await self.dismiss(image_id, tag_id)
async def _store_floor(self) -> float:
return (
await self.session.execute(
select(MLSettings.tagger_store_floor).where(MLSettings.id == 1)
)
).scalar_one()
async def update_threshold(
self, tag_id: int, min_confidence: float
) -> None:
row = await self.session.get(TagAllowlist, tag_id)
if row is not None:
# An allowlist tag can't auto-apply more permissively than the
# ingest store floor — predictions below tagger_store_floor aren't
# stored, so a lower min_confidence would behave identically to the
# floor. Clamp so the stored threshold matches actual behavior
# (#764).
floor = await self._store_floor()
row.min_confidence = max(min_confidence, floor)
async def remove(self, tag_id: int) -> None:
await self.session.execute(
delete(TagAllowlist).where(TagAllowlist.tag_id == tag_id)
)
async def _coverage_match(self, tag: Tag):
"""The predicate over image_prediction rows that resolve to `tag`,
mirroring tasks.ml._confidence_for_tag's resolution: a prediction whose
raw_name equals the tag name (any category), OR an alias maps
(raw_name, category) -> this tag. Returns a SQLAlchemy boolean clause.
"""
alias_rows = (
await self.session.execute(
select(TagAlias.alias_string, TagAlias.alias_category).where(
TagAlias.canonical_tag_id == tag.id
)
)
).all()
name_clause = ImagePrediction.raw_name == tag.name
alias_clauses = [
and_(
ImagePrediction.raw_name == a,
ImagePrediction.category == c,
)
for a, c in alias_rows
]
return or_(name_clause, *alias_clauses) if alias_clauses else name_clause
async def coverage(self, tag_id: int, threshold: float) -> int:
"""How many distinct images a sweep WOULD cover for this tag at
`threshold`: images with a resolving prediction scoring >= threshold.
The gross candidate pool (NOT minus already-applied/rejected) — it's
the tuning signal for "lower the threshold and ~N more images qualify".
"""
tag = await self.session.get(Tag, tag_id)
if tag is None:
return 0
match = await self._coverage_match(tag)
stmt = select(
func.count(distinct(ImagePrediction.image_record_id))
).where(ImagePrediction.score >= threshold, match)
return (await self.session.execute(stmt)).scalar_one()
async def list_all(self) -> Sequence[AllowlistRow]:
stmt = (
select(
TagAllowlist.tag_id,
Tag.name,
Tag.kind,
TagAllowlist.min_confidence,
)
.join(Tag, Tag.id == TagAllowlist.tag_id)
.order_by(Tag.name.asc())
)
rows = (await self.session.execute(stmt)).all()
tag_ids = [r[0] for r in rows]
# Applied counts in ONE grouped query (vs N per-row counts).
applied: dict[int, int] = {}
if tag_ids:
applied = dict(
(
await self.session.execute(
select(image_tag.c.tag_id, func.count())
.where(image_tag.c.tag_id.in_(tag_ids))
.group_by(image_tag.c.tag_id)
)
).all()
)
result = []
for r in rows:
# Coverage is per-tag (alias set differs); allowlist is small.
cov = await self.coverage(r[0], r[3])
result.append(
AllowlistRow(
tag_id=r[0],
tag_name=r[1],
tag_kind=r[2].value if hasattr(r[2], "value") else str(r[2]),
min_confidence=r[3],
applied_count=applied.get(r[0], 0),
coverage_count=cov,
)
)
return result
backend/app/services/ml/tagger.py
-210
View File
@@ -1,210 +0,0 @@
"""Camie-tagger-v2 ONNX wrapper (CPU).
Single-image at a time. Loaded lazily inside the ml-worker process; NOT
thread-safe — the ml queue worker runs --concurrency=1 per process (scale ML by
running multiple worker replicas, not threads).
v2 layout reference: HuggingFace Camais03/camie-tagger-v2 root has
camie-tagger-v2.onnx (789 MB) + camie-tagger-v2-metadata.json (7.77 MB)
+ config.json. Tags ship as nested JSON, not CSV. Preprocessing and
output handling follow the published onnx_inference.py reference:
ImageNet normalize, NCHW layout, sigmoid on refined logits (output[1]).
"""
import json
import os
from dataclasses import dataclass
from pathlib import Path
import numpy as np
from PIL import Image, ImageFile
# Cap inference threads (see Tagger.load) so each ml-worker replica is a bounded
# core consumer on a shared node — keep N_replicas × this within the cores
# allotted to ML so replicas don't oversubscribe the box / starve the DB.
_INTRA_OP_THREADS = 4
# onnxruntime lives in requirements-ml.txt only — it is NOT installed in the
# lean web image or in CI. Imported lazily inside Tagger.load() so this module
# imports fine without it (the suggestion service imports SURFACED_CATEGORIES
# from here in the web container, and CI collects the pure-logic tests).
# Tolerate minutely-truncated source images (same rationale as IR's wd14.py:
# a few missing bytes at the JPEG EOI shouldn't block tagging the whole image).
ImageFile.LOAD_TRUNCATED_IMAGES = True
MODEL_NAME = os.environ.get("CAMIE_MODEL_NAME", "camie-tagger-v2")
_MODEL_DIR = Path(os.environ.get("ML_MODEL_DIR", "/models")) / "camie"
_MODEL_FILE = f"{MODEL_NAME}.onnx"
_METADATA_FILE = f"{MODEL_NAME}-metadata.json"
# Ingest floor below which predictions aren't stored (keeps the JSON compact).
# DEFAULT/fallback only — the live value is DB-backed
# (ml_settings.tagger_store_floor) and passed into infer() per call by the ml
# task. 0.70: the suggestion path already filters there and the centroid path
# covers lower-confidence preferred tags, so the sub-0.70 tail is redundant
# (it had bloated image_record's TOAST to ~100 GB; plan-task #764).
DEFAULT_STORE_FLOOR = 0.70
# The categories FC-2b surfaces in the UI. Others (meta/rating/year) are
# still stored but the suggestion service filters them out.
# 'artist' retired in FC-2d-vii-c — artist identity is acquisition-derived
# (image_record.artist_id), never ML-inferred. 'copyright' retired
# 2026-06-01 — operator doesn't use the copyright tag-kind; fandom is
# this app's franchise/series concept (per TagsView.vue's doc comment).
# Raw predictions for both categories still get stored at STORE_FLOOR but
# don't surface in suggestions.
SURFACED_CATEGORIES = {"character", "general"}
# ImageNet preprocessing constants (per Camie v2 onnx_inference.py).
_IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
_IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)
# Square-pad color ≈ ImageNet mean × 255 (matches reference inference).
_PAD_COLOR = (124, 116, 104)
@dataclass(frozen=True)
class TagPrediction:
name: str
category: str
confidence: float
class Tagger:
def __init__(self, model_dir: Path | None = None):
self._model_dir = model_dir or _MODEL_DIR
self._session = None # onnxruntime.InferenceSession once load()ed
self._tag_names: list[str] | None = None
self._tag_categories: list[str] | None = None
self._input_name: str | None = None
self._input_size: int = 512
def load(self) -> None:
if self._session is not None:
return
model_path = self._model_dir / _MODEL_FILE
meta_path = self._model_dir / _METADATA_FILE
if not model_path.is_file():
raise RuntimeError(
f"Camie {_MODEL_FILE} missing at {model_path}. "
f"Populate /models via the ml-worker downloader."
)
if not meta_path.is_file():
raise RuntimeError(
f"Camie {_METADATA_FILE} missing at {meta_path}. "
f"Populate /models via the ml-worker downloader."
)
with open(meta_path) as f:
metadata = json.load(f)
# Per Camie v2 onnx_inference.py: idx_to_tag is keyed by str(idx);
# tag_to_category maps tag_name -> category. Project to two parallel
# lists indexed by output position for O(1) lookup in the hot path.
ds = metadata["dataset_info"]
idx_to_tag = ds["tag_mapping"]["idx_to_tag"]
tag_to_category = ds["tag_mapping"]["tag_to_category"]
total = ds["total_tags"]
names: list[str] = []
cats: list[str] = []
for i in range(total):
name = idx_to_tag.get(str(i), f"unknown-{i}")
names.append(name)
cats.append(tag_to_category.get(name, "general"))
# Input size from metadata; fall back to 512 (the v2 default).
self._input_size = int(
metadata.get("model_info", {}).get("img_size", 512)
)
# Lazy import — kept after the file-existence checks so the
# missing-model RuntimeError still fires first in environments
# without onnxruntime (CI / lean web image).
import onnxruntime as ort
# Cap the intra-op thread pool. ONNX Runtime otherwise sizes it to ALL
# host cores, so on a shared node each ml-worker replica would grab every
# core and oversubscribe (and starve the co-located DB/web). Bounding it
# makes each replica a predictable core consumer — run N replicas where
# N × _INTRA_OP_THREADS stays within the cores you allot to ML.
opts = ort.SessionOptions()
opts.intra_op_num_threads = _INTRA_OP_THREADS
session = ort.InferenceSession(
str(model_path), sess_options=opts, providers=["CPUExecutionProvider"],
)
self._input_name = session.get_inputs()[0].name
# Assign sentinels last so a partial load isn't observable.
self._tag_names = names
self._tag_categories = cats
self._session = session
def _preprocess(self, image_path: Path) -> np.ndarray:
img = Image.open(image_path)
# Composite RGBA onto neutral so transparency doesn't bias the model.
if img.mode == "RGBA":
bg = Image.new("RGBA", img.size, (255, 255, 255, 255))
bg.paste(img, mask=img.split()[3])
img = bg.convert("RGB")
elif img.mode != "RGB":
img = img.convert("RGB")
# Pad to square with ImageNet-mean color, then bicubic resize.
w, h = img.size
side = max(w, h)
square = Image.new("RGB", (side, side), _PAD_COLOR)
square.paste(img, ((side - w) // 2, (side - h) // 2))
square = square.resize(
(self._input_size, self._input_size), Image.BICUBIC
)
arr = np.array(square, dtype=np.float32) / 255.0 # HWC, [0,1]
arr = (arr - _IMAGENET_MEAN) / _IMAGENET_STD # ImageNet normalize
arr = arr.transpose(2, 0, 1) # HWC -> CHW
return arr[np.newaxis, :, :, :] # NCHW
def infer(
self, image_path: Path, *, store_floor: float = DEFAULT_STORE_FLOOR,
) -> dict[str, TagPrediction]:
"""Run Camie v2 on one image. Returns {name: TagPrediction} with
confidence >= store_floor (across all categories — the suggestion
service does category filtering later). store_floor is the DB-backed
ml_settings.tagger_store_floor, passed in by the ml task.
v2 emits multiple outputs; we use the refined predictions
(output[1] per onnx_inference.py). Sigmoid is applied to raw
logits to produce [0,1] confidence scores.
"""
self.load()
x = self._preprocess(image_path)
outputs = self._session.run(None, {self._input_name: x})
# Refined predictions if present (v2 emits initial + refined),
# fall back to initial for single-output forks.
logits = outputs[1] if len(outputs) > 1 else outputs[0]
# Squeeze batch dim, apply sigmoid.
probs = 1.0 / (1.0 + np.exp(-logits[0]))
results: dict[str, TagPrediction] = {}
names = self._tag_names
cats = self._tag_categories
for idx, score in enumerate(probs):
conf = float(score)
if conf < store_floor:
continue
if idx >= len(names):
# Output longer than metadata declared — shouldn't happen but
# don't crash the import pipeline if v2 metadata desynchronizes.
continue
results[names[idx]] = TagPrediction(
name=names[idx], category=cats[idx], confidence=conf
)
return results
_default_tagger: Tagger | None = None
def get_tagger() -> Tagger:
"""Process-level singleton so the ONNX session loads once per worker."""
global _default_tagger
if _default_tagger is None:
_default_tagger = Tagger()
return _default_tagger
backend/app/services/tag_service.py
+5 -30
View File
@@ -10,7 +10,6 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.ext.asyncio import AsyncSession
from ..models import HeadMetric, Tag, TagHead, TagKind, image_tag
from ..models.tag_allowlist import TagAllowlist
from .db_helpers import get_or_create
from .tag_query import fandom_join_alias, tag_columns
@@ -303,28 +302,22 @@ class TagService:
async def _keep_as_alias(self, tag_id: int) -> bool:
"""A merged-away tag's old name must survive as an alias iff the ML
pipeline has ever applied it OR could re-emit it (allowlisted) —
otherwise the proactive apply_allowlist_tags worker would silently
regenerate it. Purely-manual, ML-unknown tags are deleted outright (no
DB bloat)."""
pipeline has ever applied it (manual accept or head auto-apply) — so a
re-application or an alias remap resolves the canonical name. Purely-
manual, ML-unknown tags are deleted outright (no DB bloat)."""
is_machine = await self.session.scalar(
select(
exists().where(
and_(
image_tag.c.tag_id == tag_id,
image_tag.c.source.in_(
("ml_auto", "ml_accepted", "auto")
("ml_auto", "ml_accepted", "head_auto", "auto")
),
)
)
)
)
if is_machine:
return True
allowlisted = await self.session.scalar(
select(exists().where(TagAllowlist.tag_id == tag_id))
)
return bool(allowlisted)
return bool(is_machine)
async def rename(self, tag_id: int, new_name: str) -> Tag:
"""Rename a tag. Raises TagMergeConflict if the new name collides
@@ -564,7 +557,6 @@ class TagService:
merged_count = await self._repoint_image_tags(source_id, target_id)
await self._repoint_rejections(source_id, target_id)
await self._repoint_allowlist(source_id, target_id)
await self._repoint_aliases(source_id, target_id)
await self._repoint_fandom_children(
source_id, target_id, source_kind
@@ -630,23 +622,6 @@ class TagService:
.values(tag_id=tgt)
)
async def _repoint_allowlist(self, src: int, tgt: int) -> None:
tgt_has = await self.session.scalar(
select(exists().where(TagAllowlist.tag_id == tgt))
)
if tgt_has:
await self.session.execute(
text("DELETE FROM tag_allowlist WHERE tag_id = :src"),
{"src": src},
)
else:
await self.session.execute(
update(TagAllowlist)
.where(TagAllowlist.tag_id == src)
.values(tag_id=tgt)
)
async def _repoint_aliases(self, src: int, tgt: int) -> None:
from ..models.tag_alias import TagAlias