bvandeusen/FabledCurator
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 33 Wiki Activity

refactor(ml): retire the Camie tagger + allowlist bulk-apply (#1189)
CI / lint (push) Failing after 3s
Details
CI / frontend-build (push) Successful in 18s
Details
CI / backend-lint-and-test (push) Successful in 28s
Details
CI / integration (push) Successful in 3m27s
Details

Browse Source 
Heads + CCIP are the tag source and head auto-apply is the earned propagation.
The Camie tagger ran only to feed the allowlist bulk-apply (its ImagePrediction
rows had no other consumer), and the allowlist was a SECOND, un-earned auto-apply
path firing in parallel with heads on every accept — exactly the un-earned spray
the v2 pivot replaced. Retire both.

Behavior change: accepting a suggestion now applies the tag to THAT image only
(source='ml_accepted', a head-training positive) — it no longer allowlists +
fans the tag across the library via Camie. Propagation is heads' earned
auto-apply. (Loses instant cold-start propagation for booru-vocab tags; that was
un-earned and bypassed the precision gate.)

- tag_and_embed is now EMBED-ONLY (no Camie load/infer, no ImagePrediction
  writes); backfill enqueues it for images with no embedding.
- Removed: services/ml/tagger.py, apply_allowlist_tags + helpers + daily beat +
  every enqueue caller (accept/alias/merge/per-image), api/allowlist.py +
  blueprint, ImagePrediction + TagAllowlist models/tables (migration 0067),
  AllowlistTable.vue + allowlist store, the accept coverage-projection payload.
- AllowlistService gutted to accept/dismiss/undismiss/reject (the rejection store
  the rail still needs); accept returns nothing, API returns {accepted, tag_id}.
- tag merge no longer repoints/triggers the allowlist; _keep_as_alias now keys on
  ML-applied image_tag sources (incl. head_auto) instead of the allowlist.
- UI: MLBackfillCard relabelled to embedding-only; accept toast simplified;
  MaintenancePanel drops the allowlist tile.

Left for a follow-up hygiene pass (now-inert, harmless): the dead settings
columns (tagger_store_floor, tagger_model_version, suggestion_threshold_*,
video_min_tag_frames), image_record.tagger_model_version, MLThresholdSliders
trim, and the Camie model download in download_models.py.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
This commit is contained in:
Bryan Van Deusen
2026-06-30 13:04:31 -04:00
parent 3d77a38a25
commit 485387ff0b
31 changed files with 159 additions and 1710 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/api/__init__.py
-2
View File
@@ -16,7 +16,6 @@ api_bp.add_url_rule("/health", view_func=health.get_health, methods=["GET"])
def all_blueprints() -> list[Blueprint]:
from .admin import admin_bp
from .aliases import aliases_bp
from .allowlist import allowlist_bp
from .artist import artist_bp
from .artists import artists_bp
from .attachments import attachments_bp
@@ -58,7 +57,6 @@ def all_blueprints() -> list[Blueprint]:
cleanup_bp,
import_admin_bp,
suggestions_bp,
allowlist_bp,
aliases_bp,
tag_eval_bp,
heads_bp,
backend/app/api/allowlist.py
-84
View File
@@ -1,84 +0,0 @@
"""Allowlist API: list, adjust threshold, remove."""
from quart import Blueprint, jsonify, request
from ..extensions import get_session
from ..models import TagAllowlist
from ..services.ml.allowlist import AllowlistService
allowlist_bp = Blueprint("allowlist", __name__, url_prefix="/api")
@allowlist_bp.route("/allowlist", methods=["GET"])
async def list_allowlist():
async with get_session() as session:
rows = await AllowlistService(session).list_all()
return jsonify(
[
{
"tag_id": r.tag_id,
"tag_name": r.tag_name,
"tag_kind": r.tag_kind,
"min_confidence": r.min_confidence,
"applied_count": r.applied_count,
"coverage_count": r.coverage_count,
}
for r in rows
]
)
@allowlist_bp.route("/tags/<int:tag_id>/allowlist/coverage", methods=["GET"])
async def coverage(tag_id: int):
"""Live "at threshold T, a sweep would cover ~N images" projection for the
allowlist tuning dashboard. Defaults to the tag's stored threshold."""
raw = request.args.get("threshold")
async with get_session() as session:
svc = AllowlistService(session)
if raw is not None:
try:
threshold = float(raw)
except ValueError:
return jsonify({"error": "threshold must be a float"}), 400
if not (0 < threshold <= 1):
return jsonify({"error": "threshold must be in (0, 1]"}), 400
else:
row = await session.get(TagAllowlist, tag_id)
if row is None:
return jsonify({"error": "not on allowlist"}), 404
threshold = row.min_confidence
count = await svc.coverage(tag_id, threshold)
return jsonify({"count": count, "threshold": threshold})
@allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["GET"])
async def get_one(tag_id: int):
async with get_session() as session:
row = await session.get(TagAllowlist, tag_id)
if row is None:
return jsonify({"error": "not on allowlist"}), 404
return jsonify(
{"min_confidence": row.min_confidence, "added_at": row.added_at.isoformat()}
)
@allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["PATCH"])
async def patch_threshold(tag_id: int):
body = await request.get_json()
if not body or "min_confidence" not in body:
return jsonify({"error": "min_confidence required"}), 400
mc = float(body["min_confidence"])
if not (0 < mc <= 1):
return jsonify({"error": "min_confidence must be in (0, 1]"}), 400
async with get_session() as session:
await AllowlistService(session).update_threshold(tag_id, mc)
await session.commit()
return "", 204
@allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["DELETE"])
async def remove(tag_id: int):
async with get_session() as session:
await AllowlistService(session).remove(tag_id)
await session.commit()
return "", 204
backend/app/api/suggestions.py
+4 -37
View File
@@ -3,31 +3,12 @@
from quart import Blueprint, jsonify, request
from ..extensions import get_session
from ..models import Tag, TagAllowlist
from ..services.ml.allowlist import AllowlistService
from ..services.ml.suggestions import SuggestionService
suggestions_bp = Blueprint("suggestions", __name__, url_prefix="/api")
async def _accept_payload(session, svc, newly_added: bool, tag_id: int) -> dict:
"""Shape the accept/alias response. When accepting newly allowlists a tag,
include the coverage PROJECTION (at the tag's threshold) so the UI can show
a non-blocking "auto-applying to ~N images" toast — the actual apply runs
async via apply_allowlist_tags, so this is an estimate, not a post-hoc
count (#7)."""
payload = {"allowlisted": newly_added}
if newly_added:
tag = await session.get(Tag, tag_id)
row = await session.get(TagAllowlist, tag_id)
payload["tag_id"] = tag_id
payload["tag_name"] = tag.name if tag is not None else None
payload["projected_count"] = await svc.coverage(
tag_id, row.min_confidence if row is not None else 0.90,
)
return payload
@suggestions_bp.route("/images/<int:image_id>/suggestions", methods=["GET"])
async def get_suggestions(image_id: int):
# ?min=<float> overrides the configured per-category thresholds so the typed
@@ -83,15 +64,9 @@ async def accept_suggestion(image_id: int):
return jsonify({"error": "tag_id required"}), 400
tag_id = body["tag_id"]
async with get_session() as session:
svc = AllowlistService(session)
newly_added = await svc.accept(image_id, tag_id)
payload = await _accept_payload(session, svc, newly_added, tag_id)
await AllowlistService(session).accept(image_id, tag_id)
await session.commit()
if newly_added:
from ..tasks.ml import apply_allowlist_tags
apply_allowlist_tags.delay(tag_id=tag_id)
return jsonify(payload)
return jsonify({"accepted": True, "tag_id": tag_id})
@suggestions_bp.route(
@@ -104,22 +79,14 @@ async def alias_suggestion(image_id: int):
return jsonify({"error": f"required: {sorted(required)}"}), 400
canonical_tag_id = body["canonical_tag_id"]
async with get_session() as session:
svc = AllowlistService(session)
newly_added = await svc.add_alias_and_accept(
await AllowlistService(session).add_alias_and_accept(
image_id,
body["alias_string"],
body["alias_category"],
canonical_tag_id,
)
payload = await _accept_payload(
session, svc, newly_added, canonical_tag_id,
)
await session.commit()
if newly_added:
from ..tasks.ml import apply_allowlist_tags
apply_allowlist_tags.delay(tag_id=canonical_tag_id)
return jsonify(payload)
return jsonify({"accepted": True, "tag_id": canonical_tag_id})
@suggestions_bp.route(
backend/app/api/tags.py
+1 -9
View File
@@ -1,13 +1,12 @@
"""Tags API: autocomplete, create, list/add/remove for an image."""
from quart import Blueprint, jsonify, request
from sqlalchemy import exists, select
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.exc import IntegrityError
from ..extensions import get_session
from ..models import Tag, TagKind, TagPositiveConfirmation
from ..models.tag_allowlist import TagAllowlist
from ..services.bulk_tag_service import BulkTagService
from ..services.ml.aliases import AliasService
from ..services.series_match_service import SeriesMatchService
@@ -297,13 +296,6 @@ async def merge_tag(source_id: int):
status = 404 if "not found" in msg else 400
return jsonify({"error": msg}), status
await session.commit()
target_allowlisted = await session.scalar(
select(exists().where(TagAllowlist.tag_id == result.target_id))
)
if target_allowlisted:
from ..tasks.ml import apply_allowlist_tags
apply_allowlist_tags.delay(tag_id=result.target_id)
return jsonify(
{
"target": {
backend/app/celery_app.py
-4
View File
@@ -101,10 +101,6 @@ def make_celery() -> Celery:
"task": "backend.app.tasks.ml.backfill",
"schedule": 86400.0,
},
"apply-allowlist-sweep-daily": {
"task": "backend.app.tasks.ml.apply_allowlist_tags",
"schedule": 86400.0,
},
"train-heads-nightly": {
"task": "backend.app.tasks.ml.scheduled_train_heads",
"schedule": 86400.0, # passive cadence; manual retrain stays available
backend/app/models/__init__.py
-4
View File
@@ -13,7 +13,6 @@ from .head_auto_apply_run import HeadAutoApplyRun
from .head_metric import HeadMetric
from .head_metrics_snapshot import HeadMetricsSnapshot
from .head_training_run import HeadTrainingRun
from .image_prediction import ImagePrediction
from .image_provenance import ImageProvenance
from .image_record import ImageRecord
from .image_region import ImageRegion
@@ -34,7 +33,6 @@ from .subscribestar_failed_media import SubscribeStarFailedMedia
from .subscribestar_seen_media import SubscribeStarSeenMedia
from .tag import Tag, TagKind, image_tag
from .tag_alias import TagAlias
from .tag_allowlist import TagAllowlist
from .tag_eval_run import TagEvalRun
from .tag_head import TagHead
from .tag_positive_confirmation import TagPositiveConfirmation
@@ -59,7 +57,6 @@ __all__ = [
"SeriesPage",
"SeriesSuggestion",
"ImageRecord",
"ImagePrediction",
"ImageProvenance",
"ImageRegion",
"Tag",
@@ -78,7 +75,6 @@ __all__ = [
"HeadMetricsSnapshot",
"HeadTrainingRun",
"TagAlias",
"TagAllowlist",
"TagEvalRun",
"TagHead",
"TagPositiveConfirmation",
backend/app/models/image_prediction.py
-37
View File
@@ -1,37 +0,0 @@
"""ImagePrediction — one row per (image, tagger vocab prediction).
Replaces the image_record.tagger_predictions JSON blob (#768). Storing the
raw Camie/booru vocab name (not a tag_id) preserves the suggestion read
path's semantics: raw_name → canonical Tag resolution happens at read time
via the alias map, and accepting a prediction can CREATE the Tag. The store
floor (ml_settings.tagger_store_floor) is applied at WRITE time, so only
predictions >= the floor land here.
"""
from sqlalchemy import Float, ForeignKey, Index, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from .base import Base
class ImagePrediction(Base):
__tablename__ = "image_prediction"
__table_args__ = (
UniqueConstraint(
"image_record_id", "raw_name", name="image_raw_name",
),
# Per-image read (suggestion build) and the "images with tag X above
# Y" query the JSON blob never allowed.
Index("ix_image_prediction_image", "image_record_id"),
Index("ix_image_prediction_name_score", "raw_name", "score"),
)
id: Mapped[int] = mapped_column(primary_key=True)
image_record_id: Mapped[int] = mapped_column(
ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False,
)
# The raw tagger vocab key (booru form) — NOT a tag_id. Resolved to a
# canonical Tag at read time, exactly as the old JSON keys were.
raw_name: Mapped[str] = mapped_column(String(255), nullable=False)
category: Mapped[str] = mapped_column(String(64), nullable=False)
score: Mapped[float] = mapped_column(Float, nullable=False)
backend/app/models/tag_allowlist.py
-32
View File
@@ -1,32 +0,0 @@
"""TagAllowlist — tags the operator opted-in to auto-apply via maintenance."""
from datetime import datetime
from sqlalchemy import CheckConstraint, DateTime, Float, ForeignKey, func
from sqlalchemy.orm import Mapped, mapped_column
from .base import Base
class TagAllowlist(Base):
__tablename__ = "tag_allowlist"
# Bare name — Base.metadata's naming convention prepends ck_<table>_,
# producing the final ck_tag_allowlist_confidence_range (matches migration 0003).
__table_args__ = (
CheckConstraint(
"min_confidence > 0 AND min_confidence <= 1",
name="confidence_range",
),
)
tag_id: Mapped[int] = mapped_column(
ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
)
# Default auto-apply threshold for a newly-accepted tag. 0.90 (lowered from
# 0.95 on operator evidence 2026-06-07: 0.95 was too strict and skipped
# confident-enough applications). Per-tag value is still tunable in the
# allowlist table; existing rows keep whatever they were stored with.
min_confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.90)
added_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
backend/app/services/importer.py
-10
View File
@@ -1479,16 +1479,6 @@ class Importer:
existing.siglip_embedding = None
existing.siglip_model_version = None
existing.centroid_scores = None
# #768: predictions also live in the normalized image_prediction table
# now — clear them so a re-imported file re-derives a fresh set.
from sqlalchemy import delete as _delete
from ..models import ImagePrediction as _ImagePrediction
self.session.execute(
_delete(_ImagePrediction).where(
_ImagePrediction.image_record_id == existing.id
)
)
# created_at intentionally preserved; updated_at auto-bumps.
self.session.flush()
self.session.commit()
backend/app/services/ml/allowlist.py
+17 -152
View File
@@ -1,36 +1,20 @@
"""Allowlist semantics: accepting a suggestion adds the canonical tag to
image_tag AND to tag_allowlist; per-image removal/dismiss writes a rejection.
"""Suggestion actions: accept applies the canonical tag to an image (which
feeds head training); dismiss / reject record a per-image rejection.
(The Camie allowlist bulk-apply was retired #1189 — heads + CCIP are the tag
source, and head auto-apply is the earned propagation. Accept no longer
allowlists or fans a tag out across the library.)
"""
from collections.abc import Sequence
from dataclasses import dataclass
from sqlalchemy import and_, delete, distinct, func, or_, select
from sqlalchemy import delete
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from ...models import (
ImagePrediction,
MLSettings,
Tag,
TagAlias,
TagAllowlist,
TagSuggestionRejection,
)
from ...models import TagSuggestionRejection
from ...models.tag import image_tag
from .aliases import AliasService
@dataclass(frozen=True)
class AllowlistRow:
tag_id: int
tag_name: str
tag_kind: str
min_confidence: float
applied_count: int # image_tag rows currently carrying this tag
coverage_count: int # images a sweep WOULD cover at min_confidence
class AllowlistService:
def __init__(self, session: AsyncSession):
self.session = session
@@ -39,21 +23,11 @@ class AllowlistService:
async def _apply_image_tag(self, image_id: int, tag_id: int, source: str):
stmt = insert(image_tag).values(
image_record_id=image_id, tag_id=tag_id, source=source
)
stmt = stmt.on_conflict_do_nothing(
).on_conflict_do_nothing(
index_elements=["image_record_id", "tag_id"]
)
await self.session.execute(stmt)
async def _add_to_allowlist(self, tag_id: int) -> bool:
"""Returns True if newly added (caller should kick off retro-apply)."""
exists = await self.session.get(TagAllowlist, tag_id)
if exists is not None:
return False
self.session.add(TagAllowlist(tag_id=tag_id))
await self.session.flush()
return True
async def _clear_rejection(self, image_id: int, tag_id: int):
await self.session.execute(
delete(TagSuggestionRejection)
@@ -61,12 +35,11 @@ class AllowlistService:
.where(TagSuggestionRejection.tag_id == tag_id)
)
async def accept(self, image_id: int, tag_id: int) -> bool:
"""Accept a suggestion. Returns True if the tag was newly added to
the allowlist (the API layer enqueues apply_allowlist_tags then)."""
async def accept(self, image_id: int, tag_id: int) -> None:
"""Apply the accepted tag to this image (source='ml_accepted', a head
training positive) and clear any prior rejection."""
await self._apply_image_tag(image_id, tag_id, source="ml_accepted")
await self._clear_rejection(image_id, tag_id)
return await self._add_to_allowlist(tag_id)
async def add_alias_and_accept(
self,
@@ -74,17 +47,16 @@ class AllowlistService:
alias_string: str,
alias_category: str,
canonical_tag_id: int,
) -> bool:
) -> None:
await self.aliases.create(
alias_string, alias_category, canonical_tag_id
)
return await self.accept(image_id, canonical_tag_id)
await self.accept(image_id, canonical_tag_id)
async def dismiss(self, image_id: int, tag_id: int) -> None:
stmt = insert(TagSuggestionRejection).values(
image_record_id=image_id, tag_id=tag_id
)
stmt = stmt.on_conflict_do_nothing(
).on_conflict_do_nothing(
index_elements=["image_record_id", "tag_id"]
)
await self.session.execute(stmt)
@@ -96,118 +68,11 @@ class AllowlistService:
await self._clear_rejection(image_id, tag_id)
async def reject_applied_tag(self, image_id: int, tag_id: int) -> None:
"""Operator removed an applied tag from an image. Remove the
image_tag row AND record a rejection so the allowlist won't
re-apply it on the next maintenance sweep."""
"""Operator removed an applied tag from an image. Remove the image_tag
row AND record a rejection so head auto-apply won't re-apply it."""
await self.session.execute(
image_tag.delete()
.where(image_tag.c.image_record_id == image_id)
.where(image_tag.c.tag_id == tag_id)
)
await self.dismiss(image_id, tag_id)
async def _store_floor(self) -> float:
return (
await self.session.execute(
select(MLSettings.tagger_store_floor).where(MLSettings.id == 1)
)
).scalar_one()
async def update_threshold(
self, tag_id: int, min_confidence: float
) -> None:
row = await self.session.get(TagAllowlist, tag_id)
if row is not None:
# An allowlist tag can't auto-apply more permissively than the
# ingest store floor — predictions below tagger_store_floor aren't
# stored, so a lower min_confidence would behave identically to the
# floor. Clamp so the stored threshold matches actual behavior
# (#764).
floor = await self._store_floor()
row.min_confidence = max(min_confidence, floor)
async def remove(self, tag_id: int) -> None:
await self.session.execute(
delete(TagAllowlist).where(TagAllowlist.tag_id == tag_id)
)
async def _coverage_match(self, tag: Tag):
"""The predicate over image_prediction rows that resolve to `tag`,
mirroring tasks.ml._confidence_for_tag's resolution: a prediction whose
raw_name equals the tag name (any category), OR an alias maps
(raw_name, category) -> this tag. Returns a SQLAlchemy boolean clause.
"""
alias_rows = (
await self.session.execute(
select(TagAlias.alias_string, TagAlias.alias_category).where(
TagAlias.canonical_tag_id == tag.id
)
)
).all()
name_clause = ImagePrediction.raw_name == tag.name
alias_clauses = [
and_(
ImagePrediction.raw_name == a,
ImagePrediction.category == c,
)
for a, c in alias_rows
]
return or_(name_clause, *alias_clauses) if alias_clauses else name_clause
async def coverage(self, tag_id: int, threshold: float) -> int:
"""How many distinct images a sweep WOULD cover for this tag at
`threshold`: images with a resolving prediction scoring >= threshold.
The gross candidate pool (NOT minus already-applied/rejected) — it's
the tuning signal for "lower the threshold and ~N more images qualify".
"""
tag = await self.session.get(Tag, tag_id)
if tag is None:
return 0
match = await self._coverage_match(tag)
stmt = select(
func.count(distinct(ImagePrediction.image_record_id))
).where(ImagePrediction.score >= threshold, match)
return (await self.session.execute(stmt)).scalar_one()
async def list_all(self) -> Sequence[AllowlistRow]:
stmt = (
select(
TagAllowlist.tag_id,
Tag.name,
Tag.kind,
TagAllowlist.min_confidence,
)
.join(Tag, Tag.id == TagAllowlist.tag_id)
.order_by(Tag.name.asc())
)
rows = (await self.session.execute(stmt)).all()
tag_ids = [r[0] for r in rows]
# Applied counts in ONE grouped query (vs N per-row counts).
applied: dict[int, int] = {}
if tag_ids:
applied = dict(
(
await self.session.execute(
select(image_tag.c.tag_id, func.count())
.where(image_tag.c.tag_id.in_(tag_ids))
.group_by(image_tag.c.tag_id)
)
).all()
)
result = []
for r in rows:
# Coverage is per-tag (alias set differs); allowlist is small.
cov = await self.coverage(r[0], r[3])
result.append(
AllowlistRow(
tag_id=r[0],
tag_name=r[1],
tag_kind=r[2].value if hasattr(r[2], "value") else str(r[2]),
min_confidence=r[3],
applied_count=applied.get(r[0], 0),
coverage_count=cov,
)
)
return result
backend/app/services/ml/tagger.py
-210
View File
@@ -1,210 +0,0 @@
"""Camie-tagger-v2 ONNX wrapper (CPU).
Single-image at a time. Loaded lazily inside the ml-worker process; NOT
thread-safe — the ml queue worker runs --concurrency=1 per process (scale ML by
running multiple worker replicas, not threads).
v2 layout reference: HuggingFace Camais03/camie-tagger-v2 root has
camie-tagger-v2.onnx (789 MB) + camie-tagger-v2-metadata.json (7.77 MB)
+ config.json. Tags ship as nested JSON, not CSV. Preprocessing and
output handling follow the published onnx_inference.py reference:
ImageNet normalize, NCHW layout, sigmoid on refined logits (output[1]).
"""
import json
import os
from dataclasses import dataclass
from pathlib import Path
import numpy as np
from PIL import Image, ImageFile
# Cap inference threads (see Tagger.load) so each ml-worker replica is a bounded
# core consumer on a shared node — keep N_replicas × this within the cores
# allotted to ML so replicas don't oversubscribe the box / starve the DB.
_INTRA_OP_THREADS = 4
# onnxruntime lives in requirements-ml.txt only — it is NOT installed in the
# lean web image or in CI. Imported lazily inside Tagger.load() so this module
# imports fine without it (the suggestion service imports SURFACED_CATEGORIES
# from here in the web container, and CI collects the pure-logic tests).
# Tolerate minutely-truncated source images (same rationale as IR's wd14.py:
# a few missing bytes at the JPEG EOI shouldn't block tagging the whole image).
ImageFile.LOAD_TRUNCATED_IMAGES = True
MODEL_NAME = os.environ.get("CAMIE_MODEL_NAME", "camie-tagger-v2")
_MODEL_DIR = Path(os.environ.get("ML_MODEL_DIR", "/models")) / "camie"
_MODEL_FILE = f"{MODEL_NAME}.onnx"
_METADATA_FILE = f"{MODEL_NAME}-metadata.json"
# Ingest floor below which predictions aren't stored (keeps the JSON compact).
# DEFAULT/fallback only — the live value is DB-backed
# (ml_settings.tagger_store_floor) and passed into infer() per call by the ml
# task. 0.70: the suggestion path already filters there and the centroid path
# covers lower-confidence preferred tags, so the sub-0.70 tail is redundant
# (it had bloated image_record's TOAST to ~100 GB; plan-task #764).
DEFAULT_STORE_FLOOR = 0.70
# The categories FC-2b surfaces in the UI. Others (meta/rating/year) are
# still stored but the suggestion service filters them out.
# 'artist' retired in FC-2d-vii-c — artist identity is acquisition-derived
# (image_record.artist_id), never ML-inferred. 'copyright' retired
# 2026-06-01 — operator doesn't use the copyright tag-kind; fandom is
# this app's franchise/series concept (per TagsView.vue's doc comment).
# Raw predictions for both categories still get stored at STORE_FLOOR but
# don't surface in suggestions.
SURFACED_CATEGORIES = {"character", "general"}
# ImageNet preprocessing constants (per Camie v2 onnx_inference.py).
_IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
_IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)
# Square-pad color ≈ ImageNet mean × 255 (matches reference inference).
_PAD_COLOR = (124, 116, 104)
@dataclass(frozen=True)
class TagPrediction:
name: str
category: str
confidence: float
class Tagger:
def __init__(self, model_dir: Path | None = None):
self._model_dir = model_dir or _MODEL_DIR
self._session = None # onnxruntime.InferenceSession once load()ed
self._tag_names: list[str] | None = None
self._tag_categories: list[str] | None = None
self._input_name: str | None = None
self._input_size: int = 512
def load(self) -> None:
if self._session is not None:
return
model_path = self._model_dir / _MODEL_FILE
meta_path = self._model_dir / _METADATA_FILE
if not model_path.is_file():
raise RuntimeError(
f"Camie {_MODEL_FILE} missing at {model_path}. "
f"Populate /models via the ml-worker downloader."
)
if not meta_path.is_file():
raise RuntimeError(
f"Camie {_METADATA_FILE} missing at {meta_path}. "
f"Populate /models via the ml-worker downloader."
)
with open(meta_path) as f:
metadata = json.load(f)
# Per Camie v2 onnx_inference.py: idx_to_tag is keyed by str(idx);
# tag_to_category maps tag_name -> category. Project to two parallel
# lists indexed by output position for O(1) lookup in the hot path.
ds = metadata["dataset_info"]
idx_to_tag = ds["tag_mapping"]["idx_to_tag"]
tag_to_category = ds["tag_mapping"]["tag_to_category"]
total = ds["total_tags"]
names: list[str] = []
cats: list[str] = []
for i in range(total):
name = idx_to_tag.get(str(i), f"unknown-{i}")
names.append(name)
cats.append(tag_to_category.get(name, "general"))
# Input size from metadata; fall back to 512 (the v2 default).
self._input_size = int(
metadata.get("model_info", {}).get("img_size", 512)
)
# Lazy import — kept after the file-existence checks so the
# missing-model RuntimeError still fires first in environments
# without onnxruntime (CI / lean web image).
import onnxruntime as ort
# Cap the intra-op thread pool. ONNX Runtime otherwise sizes it to ALL
# host cores, so on a shared node each ml-worker replica would grab every
# core and oversubscribe (and starve the co-located DB/web). Bounding it
# makes each replica a predictable core consumer — run N replicas where
# N × _INTRA_OP_THREADS stays within the cores you allot to ML.
opts = ort.SessionOptions()
opts.intra_op_num_threads = _INTRA_OP_THREADS
session = ort.InferenceSession(
str(model_path), sess_options=opts, providers=["CPUExecutionProvider"],
)
self._input_name = session.get_inputs()[0].name
# Assign sentinels last so a partial load isn't observable.
self._tag_names = names
self._tag_categories = cats
self._session = session
def _preprocess(self, image_path: Path) -> np.ndarray:
img = Image.open(image_path)
# Composite RGBA onto neutral so transparency doesn't bias the model.
if img.mode == "RGBA":
bg = Image.new("RGBA", img.size, (255, 255, 255, 255))
bg.paste(img, mask=img.split()[3])
img = bg.convert("RGB")
elif img.mode != "RGB":
img = img.convert("RGB")
# Pad to square with ImageNet-mean color, then bicubic resize.
w, h = img.size
side = max(w, h)
square = Image.new("RGB", (side, side), _PAD_COLOR)
square.paste(img, ((side - w) // 2, (side - h) // 2))
square = square.resize(
(self._input_size, self._input_size), Image.BICUBIC
)
arr = np.array(square, dtype=np.float32) / 255.0 # HWC, [0,1]
arr = (arr - _IMAGENET_MEAN) / _IMAGENET_STD # ImageNet normalize
arr = arr.transpose(2, 0, 1) # HWC -> CHW
return arr[np.newaxis, :, :, :] # NCHW
def infer(
self, image_path: Path, *, store_floor: float = DEFAULT_STORE_FLOOR,
) -> dict[str, TagPrediction]:
"""Run Camie v2 on one image. Returns {name: TagPrediction} with
confidence >= store_floor (across all categories — the suggestion
service does category filtering later). store_floor is the DB-backed
ml_settings.tagger_store_floor, passed in by the ml task.
v2 emits multiple outputs; we use the refined predictions
(output[1] per onnx_inference.py). Sigmoid is applied to raw
logits to produce [0,1] confidence scores.
"""
self.load()
x = self._preprocess(image_path)
outputs = self._session.run(None, {self._input_name: x})
# Refined predictions if present (v2 emits initial + refined),
# fall back to initial for single-output forks.
logits = outputs[1] if len(outputs) > 1 else outputs[0]
# Squeeze batch dim, apply sigmoid.
probs = 1.0 / (1.0 + np.exp(-logits[0]))
results: dict[str, TagPrediction] = {}
names = self._tag_names
cats = self._tag_categories
for idx, score in enumerate(probs):
conf = float(score)
if conf < store_floor:
continue
if idx >= len(names):
# Output longer than metadata declared — shouldn't happen but
# don't crash the import pipeline if v2 metadata desynchronizes.
continue
results[names[idx]] = TagPrediction(
name=names[idx], category=cats[idx], confidence=conf
)
return results
_default_tagger: Tagger | None = None
def get_tagger() -> Tagger:
"""Process-level singleton so the ONNX session loads once per worker."""
global _default_tagger
if _default_tagger is None:
_default_tagger = Tagger()
return _default_tagger
backend/app/services/tag_service.py
+5 -30
View File
@@ -10,7 +10,6 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.ext.asyncio import AsyncSession
from ..models import HeadMetric, Tag, TagHead, TagKind, image_tag
from ..models.tag_allowlist import TagAllowlist
from .db_helpers import get_or_create
from .tag_query import fandom_join_alias, tag_columns
@@ -303,28 +302,22 @@ class TagService:
async def _keep_as_alias(self, tag_id: int) -> bool:
"""A merged-away tag's old name must survive as an alias iff the ML
pipeline has ever applied it OR could re-emit it (allowlisted) —
otherwise the proactive apply_allowlist_tags worker would silently
regenerate it. Purely-manual, ML-unknown tags are deleted outright (no
DB bloat)."""
pipeline has ever applied it (manual accept or head auto-apply) — so a
re-application or an alias remap resolves the canonical name. Purely-
manual, ML-unknown tags are deleted outright (no DB bloat)."""
is_machine = await self.session.scalar(
select(
exists().where(
and_(
image_tag.c.tag_id == tag_id,
image_tag.c.source.in_(
("ml_auto", "ml_accepted", "auto")
("ml_auto", "ml_accepted", "head_auto", "auto")
),
)
)
)
)
if is_machine:
return True
allowlisted = await self.session.scalar(
select(exists().where(TagAllowlist.tag_id == tag_id))
)
return bool(allowlisted)
return bool(is_machine)
async def rename(self, tag_id: int, new_name: str) -> Tag:
"""Rename a tag. Raises TagMergeConflict if the new name collides
@@ -564,7 +557,6 @@ class TagService:
merged_count = await self._repoint_image_tags(source_id, target_id)
await self._repoint_rejections(source_id, target_id)
await self._repoint_allowlist(source_id, target_id)
await self._repoint_aliases(source_id, target_id)
await self._repoint_fandom_children(
source_id, target_id, source_kind
@@ -630,23 +622,6 @@ class TagService:
.values(tag_id=tgt)
)
async def _repoint_allowlist(self, src: int, tgt: int) -> None:
tgt_has = await self.session.scalar(
select(exists().where(TagAllowlist.tag_id == tgt))
)
if tgt_has:
await self.session.execute(
text("DELETE FROM tag_allowlist WHERE tag_id = :src"),
{"src": src},
)
else:
await self.session.execute(
update(TagAllowlist)
.where(TagAllowlist.tag_id == src)
.values(tag_id=tgt)
)
async def _repoint_aliases(self, src: int, tgt: int) -> None:
from ..models.tag_alias import TagAlias
backend/app/tasks/ml.py
+24 -261
View File
@@ -1,20 +1,19 @@
"""ML Celery tasks: per-image inference, backfill discovery, head training,
allowlist auto-apply, model self-heal.
"""ML Celery tasks: per-image embedding, backfill discovery, head training,
model self-heal.
All run on the ml-worker (queue 'ml') except apply_allowlist_tags sweeps which
are 'maintenance' lane. Sync sessions (Celery workers are sync processes), same
pattern as FC-2a tasks.
All run on the ml-worker (queue 'ml'). Sync sessions (Celery workers are sync
processes), same pattern as FC-2a tasks.
"""
import logging
from pathlib import Path
from celery.exceptions import SoftTimeLimitExceeded
from sqlalchemy import delete, select
from sqlalchemy import select
from sqlalchemy.exc import DBAPIError, OperationalError
from ..celery_app import celery
from ..models import ImagePrediction, ImageRecord, MLSettings
from ..models import ImageRecord, MLSettings
from ._sync_engine import sync_session_factory as _sync_session_factory
log = logging.getLogger(__name__)
@@ -46,19 +45,16 @@ def _is_video(path: Path) -> bool:
time_limit=1200, # 20 min hard
)
def tag_and_embed(self, image_id: int) -> dict:
"""Run Camie + SigLIP on one image; store predictions + embedding;
then enqueue per-image allowlist application.
"""Compute + store one image's SigLIP embedding.
Video (#747): sample frames at a fixed cadence (ml_settings
video_frame_interval_seconds, capped at video_max_frames), keep a tag only if
it appears in >= video_min_tag_frames frames and average its confidence over
those frames (mean-pool, not max — kills one-frame noise); mean-pool the
SigLIP embeddings. On no-frames returns status='no_frames' (not an error).
video_frame_interval_seconds, capped at video_max_frames) and mean-pool the
per-frame SigLIP embeddings. On no-frames returns status='no_frames' (not an
error). (Camie tagging was retired #1189 — heads + CCIP are the tag source.)
"""
import time
from ..services.ml.embedder import get_embedder
from ..services.ml.tagger import get_tagger
# Phase + file context, so a timeout/crash names WHICH file and WHERE it
# died instead of a bare SoftTimeLimitExceeded() (operator-flagged 2026-06-08:
@@ -94,15 +90,13 @@ def tag_and_embed(self, image_id: int) -> dict:
return {"status": "file_missing", "image_id": image_id}
phase = "load_models"
tagger = get_tagger()
embedder = get_embedder(settings.embedder_model_name)
if is_vid:
# Layer-3 isolation: ffprobe (a separate process) validates
# the container before we burn ~20 GPU ops sampling frames
# from it. A corrupt video that would crash the frame
# decoder is rejected cleanly here instead of taking down
# the ml-worker. Operator-flagged 2026-05-28.
# the container before we burn GPU ops sampling frames from it.
# A corrupt video that would crash the frame decoder is rejected
# cleanly here instead of taking down the ml-worker.
phase = "video_probe"
from ..utils import safe_probe
vprobe = safe_probe.probe_video(src)
@@ -115,48 +109,23 @@ def tag_and_embed(self, image_id: int) -> dict:
"reason": vprobe.reason,
}
phase = "video_sample_frames"
t0 = time.monotonic()
frames = _sample_video_frames(
src,
interval=settings.video_frame_interval_seconds,
max_frames=settings.video_max_frames,
)
log.info(
"tag_and_embed sampled %d frame(s) in %.1fs: %s",
len(frames), time.monotonic() - t0, ctx,
)
if not frames:
return {"status": "no_frames", "image_id": image_id}
phase = "video_infer"
phase = "video_embed"
import numpy as np
preds = _aggregate_video_predictions(
[tagger.infer(f, store_floor=settings.tagger_store_floor)
for f in frames],
min_frames=settings.video_min_tag_frames,
)
# Mean-pool the per-frame SigLIP embeddings into one vector.
embedding = np.mean(
[embedder.infer(f) for f in frames], axis=0
).astype("float32")
log.info(
"tag_and_embed video aggregated %d tag(s) from %d frame(s) "
"(min_frames=%d): %s",
len(preds), len(frames), settings.video_min_tag_frames, ctx,
)
for f in frames:
f.unlink(missing_ok=True)
else:
phase = "tag"
t0 = time.monotonic()
raw = tagger.infer(src, store_floor=settings.tagger_store_floor)
log.info(
"tag_and_embed tagged in %.1fs (%d tags): %s",
time.monotonic() - t0, len(raw), ctx,
)
preds = {
name: {"category": p.category, "confidence": p.confidence}
for name, p in raw.items()
}
phase = "embed"
t0 = time.monotonic()
embedding = embedder.infer(src)
@@ -166,28 +135,9 @@ def tag_and_embed(self, image_id: int) -> dict:
)
phase = "persist"
record.tagger_model_version = settings.tagger_model_version
record.siglip_embedding = embedding.tolist()
record.siglip_model_version = settings.embedder_model_version
session.add(record)
# Write the normalized image_prediction rows (#768) — the sole home
# for predictions now (image_record.tagger_predictions was dropped in
# migration 0046). Delete-then-insert keeps a re-tag idempotent;
# tagger_store_floor was already applied in tagger.infer, so preds is
# the >=floor set.
session.execute(
delete(ImagePrediction).where(
ImagePrediction.image_record_id == image_id
)
)
session.add_all([
ImagePrediction(
image_record_id=image_id, raw_name=name,
category=p.get("category", "general"),
score=float(p.get("confidence", 0.0)),
)
for name, p in preds.items()
])
session.commit()
except SoftTimeLimitExceeded:
log.error(
@@ -210,11 +160,8 @@ def tag_and_embed(self, image_id: int) -> dict:
)
raise
log.info(
"tag_and_embed ok in %.1fs (%d tags): %s", _elapsed(), len(preds), ctx
)
apply_allowlist_tags.delay(image_id=image_id)
return {"status": "ok", "image_id": image_id, "tags": len(preds)}
log.info("tag_and_embed ok in %.1fs: %s", _elapsed(), ctx)
return {"status": "ok", "image_id": image_id}
def _sample_video_frames(
@@ -273,68 +220,24 @@ def _sample_video_frames(
return out
def _aggregate_video_predictions(per_frame: list[dict], *, min_frames: int) -> dict:
"""Aggregate per-frame {name: TagPrediction} into one prediction set (#747).
A tag is kept only if it appears (≥ the tagger store floor, already applied)
in at least `min_frames` of the sampled frames — because sampling is at a
fixed cadence, that means it was on screen for roughly min_frames×interval
seconds, so a single-frame flicker / scene-transition artifact is dropped
while a genuine scene-local tag in a long video survives. Confidence is the
MEAN over the frames where the tag appears (not max — max re-inflated the
one-frame noise this whole change exists to remove).
`min_frames` is clamped to the number of frames actually sampled so a very
short video (1–2 frames) still tags instead of dropping everything.
"""
n = len(per_frame)
if n == 0:
return {}
threshold = max(1, min(min_frames, n))
agg: dict[str, dict] = {}
for frame_preds in per_frame:
for name, p in frame_preds.items():
cur = agg.get(name)
if cur is None:
agg[name] = {"category": p.category, "sum": p.confidence, "count": 1}
else:
cur["sum"] += p.confidence
cur["count"] += 1
return {
name: {"category": v["category"], "confidence": v["sum"] / v["count"]}
for name, v in agg.items()
if v["count"] >= threshold
}
@celery.task(name="backend.app.tasks.ml.backfill", bind=True)
def backfill(self) -> int:
"""Enqueue tag_and_embed for images missing predictions/embeddings for
the current model versions. Keyset pagination by id ASC (restart-safe).
"""Enqueue tag_and_embed (embed-only) for images with no SigLIP embedding.
Keyset pagination by id ASC (restart-safe).
NB: a siglip MODEL-VERSION mismatch (an operator model swap, #1190) is NOT
re-embedded here — the CPU ml-worker can't churn the library at 384/512px;
the GPU agent owns version re-embeds via the 'embed' job.
"""
SessionLocal = _sync_session_factory()
enqueued = 0
last_id = 0
with SessionLocal() as session:
settings = session.execute(
select(MLSettings).where(MLSettings.id == 1)
).scalar_one()
while True:
rows = session.execute(
select(ImageRecord.id)
.where(ImageRecord.id > last_id)
.where(
(ImageRecord.tagger_model_version.is_(None))
| (
ImageRecord.tagger_model_version
!= settings.tagger_model_version
)
| (ImageRecord.siglip_embedding.is_(None))
# NB: a siglip MODEL-VERSION mismatch (an operator model swap,
# #1190) is intentionally NOT re-embedded here — the CPU
# ml-worker can't churn the whole library at 384/512px. The
# GPU agent owns version re-embeds via the 'embed' job.
)
.where(ImageRecord.siglip_embedding.is_(None))
.order_by(ImageRecord.id.asc())
.limit(500)
).scalars().all()
@@ -347,146 +250,6 @@ def backfill(self) -> int:
return enqueued
@celery.task(
name="backend.app.tasks.ml.apply_allowlist_tags",
bind=True,
# Audit 2026-06-02 — the full-sweep mode (neither tag_id nor image_id)
# is O(images × allowlist) and legitimately runs >5 min on large
# libraries. Cap matches the maintenance queue's recovery threshold.
soft_time_limit=1800, time_limit=2100,
)
def apply_allowlist_tags(self, tag_id: int | None = None,
image_id: int | None = None) -> int:
"""Retroactively apply allowlisted tags.
Modes:
- tag_id only : scan all images for this tag.
- image_id only : scan all allowlisted tags for this image.
- both : just the (image, tag) pair.
- neither : full sweep (daily beat).
Skips: already-applied, rejected (tag_suggestion_rejection), or
confidence below the tag's allowlist min_confidence. Applied with
source='ml_auto'.
"""
from sqlalchemy import and_
from sqlalchemy import select as sa_select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from ..models import TagAllowlist, TagSuggestionRejection
from ..models.tag import image_tag
SessionLocal = _sync_session_factory()
applied = 0
with SessionLocal() as session:
allow_rows = session.execute(
sa_select(TagAllowlist.tag_id, TagAllowlist.min_confidence)
if tag_id is None
else sa_select(
TagAllowlist.tag_id, TagAllowlist.min_confidence
).where(TagAllowlist.tag_id == tag_id)
).all()
allow = {r[0]: r[1] for r in allow_rows}
if not allow:
return 0
# Images that have any predictions (#768: from image_prediction, not
# the old JSON column), optionally narrowed to one image.
img_ids_query = sa_select(ImagePrediction.image_record_id).distinct()
if image_id is not None:
img_ids_query = img_ids_query.where(
ImagePrediction.image_record_id == image_id
)
for (img_id,) in session.execute(img_ids_query).all():
preds = _load_predictions_sync(session, img_id)
for a_tag_id, min_conf in allow.items():
exists = session.execute(
sa_select(image_tag.c.tag_id).where(
and_(
image_tag.c.image_record_id == img_id,
image_tag.c.tag_id == a_tag_id,
)
)
).scalar_one_or_none()
if exists is not None:
continue
rej = session.get(
TagSuggestionRejection, (img_id, a_tag_id)
)
if rej is not None:
continue
from ..models import Tag
tag = session.get(Tag, a_tag_id)
if tag is None:
continue
conf = _confidence_for_tag(session, tag, preds)
if conf is None or conf < min_conf:
continue
stmt = pg_insert(image_tag).values(
image_record_id=img_id,
tag_id=a_tag_id,
source="ml_auto",
)
stmt = stmt.on_conflict_do_nothing(
index_elements=["image_record_id", "tag_id"]
)
session.execute(stmt)
applied += 1
session.commit()
return applied
def _load_predictions_sync(session, image_id: int) -> dict:
"""Predictions for one image from image_prediction (#768), in the
{raw_name: {category, confidence}} shape _confidence_for_tag consumes —
keeps the allowlist resolution logic unchanged."""
from sqlalchemy import select as sa_select
rows = session.execute(
sa_select(
ImagePrediction.raw_name,
ImagePrediction.category,
ImagePrediction.score,
).where(ImagePrediction.image_record_id == image_id)
).all()
return {
r.raw_name: {"category": r.category, "confidence": r.score}
for r in rows
}
def _confidence_for_tag(session, tag, preds: dict) -> float | None:
"""Highest confidence among predictions that resolve to `tag` —
either the prediction name equals the tag name, or an alias maps
(prediction name, category) -> tag.id.
"""
from sqlalchemy import select as sa_select
from ..models import TagAlias
best: float | None = None
direct = preds.get(tag.name)
if direct is not None:
best = float(direct.get("confidence", 0.0))
alias_rows = session.execute(
sa_select(TagAlias.alias_string, TagAlias.alias_category).where(
TagAlias.canonical_tag_id == tag.id
)
).all()
for alias_string, alias_category in alias_rows:
p = preds.get(alias_string)
if p is None:
continue
if p.get("category") != alias_category:
continue
c = float(p.get("confidence", 0.0))
if best is None or c > best:
best = c
return best
@celery.task(
name="backend.app.tasks.ml.tag_eval_run",
bind=True,