diff --git a/alembic/versions/0068_drop_dead_tagger_settings.py b/alembic/versions/0068_drop_dead_tagger_settings.py new file mode 100644 index 0000000..770676d --- /dev/null +++ b/alembic/versions/0068_drop_dead_tagger_settings.py @@ -0,0 +1,80 @@ +"""drop dead tagger/suggestion settings + columns left after Camie retirement (#1199) + +Hygiene follow-up to #1189. These were left inert to bound that change; nothing +reads them now: +- ml_settings: tagger_store_floor + tagger_model_version (only the deleted Camie + tagger used them), suggestion_threshold_character/general (already dead pre- + retirement — scoring uses per-head thresholds), video_min_tag_frames (only the + deleted video-prediction aggregator used it). +- image_record: tagger_model_version (no writer now), centroid_scores (long-dead + JSON cache, no reader). + +Revision ID: 0068 +Revises: 0067 +Create Date: 2026-06-30 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0068" +down_revision: Union[str, None] = "0067" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.drop_column("ml_settings", "suggestion_threshold_character") + op.drop_column("ml_settings", "suggestion_threshold_general") + op.drop_column("ml_settings", "tagger_store_floor") + op.drop_column("ml_settings", "video_min_tag_frames") + op.drop_column("ml_settings", "tagger_model_version") + op.drop_column("image_record", "tagger_model_version") + op.drop_column("image_record", "centroid_scores") + + +def downgrade() -> None: + op.add_column( + "image_record", + sa.Column("centroid_scores", sa.JSON(), nullable=True), + ) + op.add_column( + "image_record", + sa.Column("tagger_model_version", sa.String(length=128), nullable=True), + ) + op.add_column( + "ml_settings", + sa.Column( + "tagger_model_version", sa.String(length=128), nullable=False, + server_default="camie-tagger-v2", + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "video_min_tag_frames", sa.Integer(), nullable=False, + server_default="3", + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "tagger_store_floor", sa.Float(), nullable=False, + server_default="0.7", + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "suggestion_threshold_general", sa.Float(), nullable=False, + server_default="0.7", + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "suggestion_threshold_character", sa.Float(), nullable=False, + server_default="0.7", + ), + ) diff --git a/backend/app/api/ml_admin.py b/backend/app/api/ml_admin.py index cd0cf56..054fa69 100644 --- a/backend/app/api/ml_admin.py +++ b/backend/app/api/ml_admin.py @@ -9,12 +9,8 @@ ml_admin_bp = Blueprint("ml_admin", __name__, url_prefix="/api/ml") _EDITABLE = ( - "suggestion_threshold_character", - "suggestion_threshold_general", - "tagger_store_floor", "video_frame_interval_seconds", "video_max_frames", - "video_min_tag_frames", "head_min_positives", "head_auto_apply_precision", "head_auto_apply_enabled", @@ -37,13 +33,8 @@ async def get_settings(): ).scalar_one() return jsonify( { - "suggestion_threshold_character": s.suggestion_threshold_character, - "suggestion_threshold_general": s.suggestion_threshold_general, - "tagger_store_floor": s.tagger_store_floor, "video_frame_interval_seconds": s.video_frame_interval_seconds, "video_max_frames": s.video_max_frames, - "video_min_tag_frames": s.video_min_tag_frames, - "tagger_model_version": s.tagger_model_version, "embedder_model_version": s.embedder_model_version, "head_min_positives": s.head_min_positives, "head_auto_apply_precision": s.head_auto_apply_precision, @@ -88,31 +79,12 @@ async def patch_settings(): def _validate(p: dict) -> str | None: - """Returns an error string if the proposed settings are invalid, else None. - - Invariant (plan-task #764): the per-category suggestion thresholds can't - drop below tagger_store_floor — nothing below the floor is stored, so a - lower threshold would silently surface nothing in that gap. The UI clamps - the sliders to the floor; this is the server-side backstop. - """ - floor = p["tagger_store_floor"] - if not (0.0 <= floor <= 1.0): - return "tagger_store_floor must be between 0 and 1" - for cat in ("character", "general"): - if p[f"suggestion_threshold_{cat}"] < floor: - return ( - f"suggestion_threshold_{cat} cannot be below tagger_store_floor " - f"({floor}) — predictions below the floor are not stored" - ) - # Video tagging (#747). + """Returns an error string if the proposed settings are invalid, else None.""" + # Video embedding (#747). if p["video_frame_interval_seconds"] <= 0: return "video_frame_interval_seconds must be > 0" if p["video_max_frames"] < 1: return "video_max_frames must be >= 1" - if p["video_min_tag_frames"] < 1: - return "video_min_tag_frames must be >= 1" - if p["video_min_tag_frames"] > p["video_max_frames"]: - return "video_min_tag_frames cannot exceed video_max_frames" # Head training (#114). if int(p["head_min_positives"]) < 1: return "head_min_positives must be >= 1" diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py index b3ba120..fc459dd 100644 --- a/backend/app/models/image_record.py +++ b/backend/app/models/image_record.py @@ -9,7 +9,6 @@ from datetime import datetime from pgvector.sqlalchemy import Vector from sqlalchemy import ( - JSON, BigInteger, DateTime, Enum, @@ -77,19 +76,13 @@ class ImageRecord(Base): ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True ) - # ML fields (populated by FC-2's ml-worker). Per-tag predictions live in the - # normalized image_prediction table (#768) — the tagger_predictions JSON - # column was dropped in migration 0046. tagger_model_version stays as the - # "has this been tagged / is it current?" signal the backfill sweep reads. - tagger_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True) - # 1152 = SigLIP-so400m embedding dim. Swapping models in FC-2 may require - # a column-width migration. + # ML fields (populated by the ml-worker / GPU agent). 1152 = SigLIP-so400m + # embedding dim; siglip_model_version stamps which model produced it (so an + # operator model swap, #1190, can re-embed the stale rows). A different-dim + # model would need a column-width migration. siglip_embedding: Mapped[list[float] | None] = mapped_column(Vector(1152), nullable=True) siglip_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True) - # Centroid score cache (populated post-tagging) - centroid_scores: Mapped[dict | None] = mapped_column(JSON, nullable=True) - created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) diff --git a/backend/app/models/ml_settings.py b/backend/app/models/ml_settings.py index 75db5b2..1d7c729 100644 --- a/backend/app/models/ml_settings.py +++ b/backend/app/models/ml_settings.py @@ -23,39 +23,16 @@ class MLSettings(Base): __table_args__ = (CheckConstraint("id = 1", name="singleton"),) id: Mapped[int] = mapped_column(Integer, primary_key=True) - suggestion_threshold_character: Mapped[float] = mapped_column( - Float, nullable=False, default=0.70 - ) - # Default raised 0.50 → 0.70 on 2026-06-02 — operator-flagged 0.50 - # surfaced too many low-confidence picks; 0.70 keeps the rail - # signal-rich while still surfacing more than the original 0.95 - # which hid almost everything. Operator-tunable via Settings → ML. - suggestion_threshold_general: Mapped[float] = mapped_column( - Float, nullable=False, default=0.70 - ) - # Ingest floor: tagger predictions below this confidence are not stored - # (tagger.Tagger.infer). Default 0.70 — the suggestion path already filters - # there, so the sub-0.70 tail is redundant weight (it had bloated - # image_record's TOAST to ~100 GB; plan-task #764). Operator-tunable via - # Settings → ML; must stay ≤ the suggestion thresholds. - tagger_store_floor: Mapped[float] = mapped_column( - Float, nullable=False, default=0.70 - ) - # Video tagging (#747). Sample one frame every N seconds (fixed CADENCE, not a - # fixed count) so a tag's frame-presence reflects real screen time regardless - # of video length; cap the total so a long video can't explode into hundreds - # of inferences (the cadence stretches past the cap). A tag is kept only if it - # appears in >= video_min_tag_frames sampled frames (≈ that many × interval - # seconds on screen) — duration-independent noise rejection. Operator-tunable. + # Video embedding (#747). Sample one frame every N seconds (fixed CADENCE, not + # a fixed count) so coverage reflects real screen time regardless of length; + # cap the total so a long video can't explode into hundreds of embeds. The + # per-frame SigLIP embeddings are mean-pooled. Operator-tunable. video_frame_interval_seconds: Mapped[float] = mapped_column( Float, nullable=False, default=4.0 ) video_max_frames: Mapped[int] = mapped_column( Integer, nullable=False, default=64 ) - video_min_tag_frames: Mapped[int] = mapped_column( - Integer, nullable=False, default=3 - ) # Tagging-v2 head training (#114). The head is the suggestion source that # LEARNS from the operator's tags (replacing Camie + centroid). A concept # needs >= head_min_positives labelled images before a head is trained; @@ -94,9 +71,6 @@ class MLSettings(Base): ccip_auto_apply_threshold: Mapped[float] = mapped_column( Float, nullable=False, default=0.92 ) - tagger_model_version: Mapped[str] = mapped_column( - String(128), nullable=False, default="camie-tagger-v2" - ) embedder_model_version: Mapped[str] = mapped_column( String(128), nullable=False, default="siglip-so400m-patch14-384" ) diff --git a/backend/app/scripts/download_models.py b/backend/app/scripts/download_models.py index 4b5e6d3..c6ff7a3 100644 --- a/backend/app/scripts/download_models.py +++ b/backend/app/scripts/download_models.py @@ -7,7 +7,6 @@ import sys from pathlib import Path MODEL_ROOT = Path(os.environ.get("ML_MODEL_DIR", "/models")) -CAMIE_REPO = os.environ.get("CAMIE_HF_REPO", "Camais03/camie-tagger-v2") SIGLIP_REPO = os.environ.get( "SIGLIP_HF_REPO", "google/siglip-so400m-patch14-384" ) @@ -24,34 +23,6 @@ def _snapshot(repo_id: str, dest: Path, allow_patterns: list[str] | None) -> Non ) -def ensure_camie() -> None: - """Fetch Camie v2 weights + metadata. - - v2 layout (HuggingFace Camais03/camie-tagger-v2): the ONNX file is - named camie-tagger-v2.onnx (not model.onnx) and tags ship inside - camie-tagger-v2-metadata.json (not selected_tags.csv). Both at root. - The repo also contains app/, game/, training/, images/ subdirs full - of setup/demo files we don't need — allow_patterns scopes the fetch - to just the inference essentials (~790 MB instead of ~2 GB). - """ - dest = MODEL_ROOT / "camie" - model_file = dest / "camie-tagger-v2.onnx" - meta_file = dest / "camie-tagger-v2-metadata.json" - if model_file.is_file() and meta_file.is_file(): - print(f"[download_models] Camie present at {dest}") - return - print(f"[download_models] Fetching {CAMIE_REPO} -> {dest}") - _snapshot( - CAMIE_REPO, dest, - [ - "camie-tagger-v2.onnx", - "camie-tagger-v2-metadata.json", - "config.json", - "config.yaml", - ], - ) - - def ensure_siglip() -> None: dest = MODEL_ROOT / "siglip" if (dest / "config.json").is_file() and any(dest.glob("*.safetensors")): @@ -62,7 +33,6 @@ def ensure_siglip() -> None: def main() -> int: - ensure_camie() ensure_siglip() print("[download_models] Done.") return 0 diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index d5f86d4..3916909 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -395,9 +395,8 @@ def delete_images( def delete_tag(session: Session, *, tag_id: int) -> dict: """Simple DELETE FROM tag WHERE id=?. - Postgres cascades the rest (image_tag, tag_alias, tag_allowlist, - tag_reference_embedding, tag_suggestion_rejection, series_page). - Returns counts BEFORE delete so the caller can surface them. + Postgres cascades the rest (image_tag, tag_alias, tag_suggestion_rejection, + series_page). Returns counts BEFORE delete so the caller can surface them. Raises LookupError if tag_id not found. """ tag = session.get(Tag, tag_id) @@ -742,8 +741,7 @@ def purge_legacy_tags(session: Session, *, dry_run: bool = False) -> dict: artist-kind tags PLUS general tags whose name matches a legacy prefix (source:*). - CASCADE on image_tag / tag_alias / tag_allowlist / - tag_reference_embedding / tag_suggestion_rejection / series_page + CASCADE on image_tag / tag_alias / tag_suggestion_rejection / series_page clears the related rows on the parent DELETE. Returns: @@ -785,23 +783,21 @@ def purge_legacy_tags(session: Session, *, dry_run: bool = False) -> dict: return result -# The Camie-suggestable CONTENT vocabulary. "Reset content tagging" wipes -# these so the operator can re-tag from scratch via auto-suggest. fandom + -# series (and series_page ordering) are deliberately NOT here — they're kept. +# The CONTENT vocabulary. "Reset content tagging" wipes these so the operator +# can re-tag from scratch. fandom + series (and series_page ordering) are +# deliberately NOT here — they're kept. RESETTABLE_TAG_KINDS = ("general", "character") def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict: """Count (dry_run) or DELETE every general + character tag so the operator - can re-tag from scratch via the Camie auto-suggest. + can re-tag from scratch (heads/CCIP repopulate suggestions). - PRESERVED: fandom + series tags and their series_page ordering, plus every - image's image_prediction rows (untouched) so suggestions - repopulate immediately. CASCADE on image_tag / tag_alias / tag_allowlist / - tag_reference_embedding / tag_suggestion_rejection clears each deleted - tag's applications + metadata. Tag.fandom_id is SET NULL, so deleting - character tags never touches the fandom rows. Irreversible except via DB - backup restore. + PRESERVED: fandom + series tags and their series_page ordering. CASCADE on + image_tag / tag_alias / tag_suggestion_rejection clears each deleted tag's + applications + metadata. Tag.fandom_id is SET NULL, so deleting character + tags never touches the fandom rows. Irreversible except via DB backup + restore. Returns: {"by_kind": {"general": N, "character": M}, diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index 7ba8e5c..70bc7c8 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -1475,10 +1475,8 @@ class Importer: existing.duration_seconds = duration # #871: keep the kept copy's duration existing.thumbnail_path = None existing.integrity_status = "unknown" - existing.tagger_model_version = None existing.siglip_embedding = None existing.siglip_model_version = None - existing.centroid_scores = None # created_at intentionally preserved; updated_at auto-bumps. self.session.flush() self.session.commit() diff --git a/frontend/src/components/settings/MLThresholdSliders.vue b/frontend/src/components/settings/MLThresholdSliders.vue index 685009f..49c4a59 100644 --- a/frontend/src/components/settings/MLThresholdSliders.vue +++ b/frontend/src/components/settings/MLThresholdSliders.vue @@ -1,69 +1,30 @@