From 0f35a0c48405abb1764546059d5e36a86e8d646b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 19 Apr 2026 19:50:58 -0400 Subject: [PATCH] feat: ML tag suggestions, character/fandom integrity, underscores, modal polish Consolidated merge of feat/tag-suggestions branch. Original 64-commit history was lost to git-object corruption in a Nextcloud-synced checkout; this single commit captures the equivalent diff. Includes: - pgvector-backed tag suggestion infra (WD14 + SigLIP centroids, ml-worker container, Celery tasks, suggestion service, accept/reject endpoints + modal UI with green/red chip buttons) - Character/fandom integrity: title-case normalization on every write path, fandom-id backfill, maintenance task + settings button, migrations g26041901 + h26041901 to canonicalize legacy rows with case-only duplicate merging - Tag-underscores + modal polish: WD14 name canonicalization at emit + accept + add/bulk-add paths, migration i26041901 for legacy-row rename-or-merge across character/fandom/NULL kinds, suggestion-accept refresh parity via awaited loadTags, persistent chip tint --- Dockerfile.ml | 42 + app/celery_app.py | 8 +- app/main.py | 222 +- app/ml/__init__.py | 1 + app/ml/siglip.py | 58 + app/ml/wd14.py | 110 + app/models.py | 60 + app/services/__init__.py | 1 + app/services/tag_suggestions.py | 235 +++ app/static/js/bulk-select.js | 5 +- app/static/js/view-modal.js | 234 ++- app/static/style.css | 176 ++ app/tasks/import_file.py | 8 + app/tasks/maintenance.py | 66 + app/tasks/ml.py | 239 +++ app/templates/_gallery_modal.html | 14 + app/templates/settings.html | 29 + app/utils/tag_names.py | 49 + docker-compose.yml | 33 +- .../plans/2026-04-18-clickable-post-tag.md | 318 +++ .../plans/2026-04-18-tag-suggestions.md | 1825 +++++++++++++++++ .../2026-04-19-character-tag-integrity.md | 1045 ++++++++++ ...-04-19-tag-underscores-and-modal-polish.md | 1007 +++++++++ ...26-04-19-user-tag-embedding-suggestions.md | 1052 ++++++++++ .../2026-04-18-clickable-post-tag-design.md | 100 + .../2026-04-18-tag-suggestions-integration.md | 252 +++ ...26-04-19-character-tag-integrity-design.md | 177 ++ ...tag-underscores-and-modal-polish-design.md | 269 +++ ...9-user-tag-embedding-suggestions-design.md | 157 ++ .../superpowers/tag-suggestion-system-spec.md | 265 +++ .../0017871d2221_add_tag_suggestions.py | 121 ++ ...41901_rename_character_to_tag_reference.py | 75 + ...01_normalize_character_fandom_tag_names.py | 163 ++ ..._normalize_underscores_and_general_tags.py | 189 ++ requirements-ml.txt | 21 + requirements.txt | 1 + scripts/download_models.py | 45 + 37 files changed, 8642 insertions(+), 30 deletions(-) create mode 100644 Dockerfile.ml create mode 100644 app/ml/__init__.py create mode 100644 app/ml/siglip.py create mode 100644 app/ml/wd14.py create mode 100644 app/services/__init__.py create mode 100644 app/services/tag_suggestions.py create mode 100644 app/tasks/maintenance.py create mode 100644 app/tasks/ml.py create mode 100644 app/utils/tag_names.py create mode 100644 docs/superpowers/plans/2026-04-18-clickable-post-tag.md create mode 100644 docs/superpowers/plans/2026-04-18-tag-suggestions.md create mode 100644 docs/superpowers/plans/2026-04-19-character-tag-integrity.md create mode 100644 docs/superpowers/plans/2026-04-19-tag-underscores-and-modal-polish.md create mode 100644 docs/superpowers/plans/2026-04-19-user-tag-embedding-suggestions.md create mode 100644 docs/superpowers/specs/2026-04-18-clickable-post-tag-design.md create mode 100644 docs/superpowers/specs/2026-04-18-tag-suggestions-integration.md create mode 100644 docs/superpowers/specs/2026-04-19-character-tag-integrity-design.md create mode 100644 docs/superpowers/specs/2026-04-19-tag-underscores-and-modal-polish-design.md create mode 100644 docs/superpowers/specs/2026-04-19-user-tag-embedding-suggestions-design.md create mode 100644 docs/superpowers/tag-suggestion-system-spec.md create mode 100644 migrations/versions/0017871d2221_add_tag_suggestions.py create mode 100644 migrations/versions/g26041901_rename_character_to_tag_reference.py create mode 100644 migrations/versions/h26041901_normalize_character_fandom_tag_names.py create mode 100644 migrations/versions/i26041901_normalize_underscores_and_general_tags.py create mode 100644 requirements-ml.txt create mode 100644 scripts/download_models.py diff --git a/Dockerfile.ml b/Dockerfile.ml new file mode 100644 index 0000000..8abbfd4 --- /dev/null +++ b/Dockerfile.ml @@ -0,0 +1,42 @@ +# syntax=docker/dockerfile:1 +FROM python:3.12-slim AS base + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + ML_MODEL_DIR=/models + +# System deps: Pillow / image handling + psycopg2 build deps (binary wheel usually fine but libpq is needed at runtime on slim) +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgl1 \ + libglib2.0-0 \ + libpq5 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Install PyTorch CPU wheel first from its dedicated index to avoid pulling CUDA libs. +# Floor only — buildkit will pick the latest matching CPU wheel; keep this above the app-deps +# layer so transformers on PyPI doesn't accidentally resolve a mismatched torch. +RUN pip install --no-cache-dir \ + --index-url https://download.pytorch.org/whl/cpu \ + "torch>=2.5" + +# Pre-install only what the model downloader needs, then fetch models. By keeping the model-download +# layer above the full requirements install, a change to requirements-ml.txt won't invalidate the +# (slow, ~2 GB) model download layer. +RUN pip install --no-cache-dir "huggingface_hub>=0.25" +COPY scripts/download_models.py /app/scripts/download_models.py +RUN mkdir -p ${ML_MODEL_DIR} && python /app/scripts/download_models.py + +# Install the rest of the ML requirements. Model files are already on disk from the step above. +COPY requirements-ml.txt /app/requirements-ml.txt +RUN pip install --no-cache-dir -r requirements-ml.txt + +# Copy application code (needed so Celery can import app.tasks.ml and app.ml). +# config.py lives at the repo root and is imported by app/__init__.py via 'config.Config'. +COPY app /app/app +COPY config.py /app/config.py + +# Celery needs these env vars set at runtime via docker-compose. +CMD ["celery", "-A", "app.celery_app:celery", "worker", "--loglevel=info", "-Q", "ml", "--concurrency=1"] diff --git a/app/celery_app.py b/app/celery_app.py index af78f7e..164cbaf 100644 --- a/app/celery_app.py +++ b/app/celery_app.py @@ -37,7 +37,9 @@ def make_celery(app=None): 'app.tasks.scan', 'app.tasks.import_file', 'app.tasks.thumbnail', - 'app.tasks.sidecar' + 'app.tasks.sidecar', + 'app.tasks.ml', + 'app.tasks.maintenance', ] ) @@ -69,11 +71,15 @@ def make_celery(app=None): 'app.tasks.scan.cleanup_old_tasks': {'queue': 'maintenance'}, 'app.tasks.scan.update_system_stats': {'queue': 'maintenance'}, 'app.tasks.scan.update_batch_stats': {'queue': 'maintenance'}, + 'app.tasks.maintenance.sync_character_fandoms': {'queue': 'maintenance'}, # Import tasks - handled by worker (heavy processing) 'app.tasks.import_file.*': {'queue': 'import'}, 'app.tasks.thumbnail.*': {'queue': 'thumbnail'}, 'app.tasks.sidecar.*': {'queue': 'sidecar'}, + + # ML inference tasks - handled by ml-worker + 'app.tasks.ml.*': {'queue': 'ml'}, }, # Task default queue for unrouted tasks diff --git a/app/main.py b/app/main.py index 7f32f09..399a8ad 100644 --- a/app/main.py +++ b/app/main.py @@ -1,6 +1,6 @@ # app/main.py -from flask import Blueprint, render_template, flash, redirect, url_for, abort, send_from_directory, request, jsonify +from flask import Blueprint, render_template, flash, redirect, url_for, abort, send_from_directory, request, jsonify, current_app from sqlalchemy import desc, func, text, extract, and_ from sqlalchemy.orm import joinedload, aliased @@ -9,6 +9,7 @@ from datetime import datetime from app.models import ImageRecord, Tag, ArchiveRecord, image_tags, ImportTask, ImportBatch, AppSettings from app import db +from app.utils.tag_names import normalize_display_name import os import re import shutil @@ -25,8 +26,12 @@ def _parse_character_fandom(name_after_prefix): def _ensure_fandom_tag(fandom_name): - """Find or create a fandom tag by name. Returns the Tag object.""" - full_name = f"fandom:{fandom_name}" + """Find or create a fandom tag by name. Returns the Tag object. + + Normalizes the display name so callers don't have to remember. + """ + normalized = normalize_display_name(fandom_name.strip()) + full_name = f"fandom:{normalized}" fandom_tag = Tag.query.filter_by(name=full_name).first() if not fandom_tag: fandom_tag = Tag(name=full_name, kind="fandom") @@ -601,6 +606,27 @@ def settings(): return render_template('settings.html', active_tab=tab) +@main.post('/settings/maintenance/ml-backfill') +def trigger_ml_backfill(): + from app.tasks.ml import backfill + backfill.apply_async(queue='ml') + return redirect(url_for('main.settings', tab='maintenance')) + + +@main.post('/settings/maintenance/recompute-centroids') +def trigger_recompute_all_centroids(): + from app.tasks.ml import recompute_all_centroids + recompute_all_centroids.apply_async(queue='ml') + return redirect(url_for('main.settings', tab='maintenance')) + + +@main.post('/settings/maintenance/sync-character-fandoms') +def trigger_sync_character_fandoms(): + from app.tasks.maintenance import sync_character_fandoms + sync_character_fandoms.apply_async(queue='maintenance') + return redirect(url_for('main.settings', tab='maintenance')) + + # ---------------------------- # Tag add/remove endpoints # ---------------------------- @@ -677,6 +703,114 @@ def list_tags(image_id): ]) +# --------------------------------------------------------------------------- +# Tag suggestions (ML-backed) +# --------------------------------------------------------------------------- + +@main.get("/image//suggestions") +def get_image_suggestions(image_id): + from app.services.tag_suggestions import get_suggestions + suggestions = get_suggestions(image_id) + return jsonify({'ok': True, 'suggestions': suggestions}) + + +@main.post("/image//suggestions/accept") +def accept_image_suggestion(image_id): + from app.models import SuggestionFeedback + from app.services.tag_suggestions import _WD14_CATEGORY_TO_KIND, _canonicalize_wd14_name + payload = request.get_json(force=True, silent=True) or {} + tag_name = (payload.get('tag_name') or '').strip() + source = payload.get('source') or '' + category = payload.get('category') or '' + confidence = float(payload.get('confidence') or 0.0) + if not tag_name or not source or not category: + return jsonify({'ok': False, 'error': 'missing_fields'}), 400 + + image = ImageRecord.query.get(image_id) + if image is None: + return jsonify({'ok': False, 'error': 'image_not_found'}), 404 + + # Canonicalize before lookup so an already-attached "big hair" matches a + # freshly-POSTed "big_hair" (defensive — the client should already send canonical). + tag_name = _canonicalize_wd14_name(tag_name, category) + + tag = Tag.query.filter_by(name=tag_name).first() + if tag is None: + kind = _WD14_CATEGORY_TO_KIND.get(category) # None for 'general' + tag = Tag(name=tag_name, kind=kind) + db.session.add(tag) + db.session.flush() + + if tag not in image.tags: + image.tags.append(tag) + + db.session.add(SuggestionFeedback( + image_id=image_id, + tag_name=tag_name, + suggestion_source=source, + confidence=confidence, + decision='accepted', + )) + db.session.commit() + + # Schedule centroid recompute if this tag's kind is eligible and delta is exceeded. + from app.services.tag_suggestions import ELIGIBLE_CENTROID_KINDS + if tag.kind in ELIGIBLE_CENTROID_KINDS: + from app.models import TagReferenceEmbedding, TagSuggestionConfig + from app.tasks.ml import recompute_centroid + from app.ml.siglip import MODEL_VERSION as SIGLIP_VER + + delta_cfg = TagSuggestionConfig.query.filter_by(key='centroid_recompute_delta').first() + delta_threshold = int(delta_cfg.value) if delta_cfg else 3 + + current_count = ( + db.session.query(db.func.count(image_tags.c.image_id)) + .filter(image_tags.c.tag_id == tag.id) + .scalar() + ) or 0 + ref = ( + TagReferenceEmbedding.query + .filter_by(tag_name=tag_name, model_version=SIGLIP_VER) + .first() + ) + last_count = ref.reference_count if ref else 0 + if current_count - last_count >= delta_threshold: + try: + recompute_centroid.apply_async(args=[tag_name], queue='ml') + except Exception: + # Don't fail the accept if enqueue fails + pass + + return jsonify({ + 'ok': True, + 'tag': {'id': tag.id, 'name': tag.name, 'kind': tag.kind}, + }) + + +@main.post("/image//suggestions/reject") +def reject_image_suggestion(image_id): + from app.models import SuggestionFeedback + payload = request.get_json(force=True, silent=True) or {} + tag_name = (payload.get('tag_name') or '').strip() + source = payload.get('source') or '' + confidence = float(payload.get('confidence') or 0.0) + if not tag_name or not source: + return jsonify({'ok': False, 'error': 'missing_fields'}), 400 + + if ImageRecord.query.get(image_id) is None: + return jsonify({'ok': False, 'error': 'image_not_found'}), 404 + + db.session.add(SuggestionFeedback( + image_id=image_id, + tag_name=tag_name, + suggestion_source=source, + confidence=confidence, + decision='rejected', + )) + db.session.commit() + return jsonify({'ok': True}) + + @main.post("/image//tags/add") def add_tag(image_id): """ @@ -695,8 +829,11 @@ def add_tag(image_id): kind = name.split(":", 1)[0] tag_name = name else: + # No prefix — treat as a user-typed general tag. Strip underscores so + # WD14-style names pasted into the add box converge with our convention. + from app.utils.tag_names import underscores_to_spaces kind = "user" - tag_name = name + tag_name = underscores_to_spaces(name) img = ImageRecord.query.get_or_404(image_id) tag = Tag.query.filter_by(name=tag_name).first() @@ -704,16 +841,29 @@ def add_tag(image_id): fandom_tag = None if not tag: - tag = Tag(name=tag_name, kind=kind) - # For new character tags, parse and link fandom + # For new character tags, parse + normalize + link fandom if kind == "character": char_part = tag_name.split(":", 1)[1] char_name, fandom_name = _parse_character_fandom(char_part) + norm_char = normalize_display_name(char_name) if fandom_name: fandom_tag = _ensure_fandom_tag(fandom_name) + tag_name = f"character:{norm_char} ({fandom_tag.name.split(':', 1)[1]})" + else: + tag_name = f"character:{norm_char}" + elif kind == "fandom": + fandom_part = tag_name.split(":", 1)[1] + tag_name = f"fandom:{normalize_display_name(fandom_part)}" + # Look up again under the normalized name — an existing row may match. + tag = Tag.query.filter_by(name=tag_name).first() + if not tag: + tag = Tag(name=tag_name, kind=kind) + if kind == "character" and fandom_name: tag.fandom_id = fandom_tag.id - db.session.add(tag) - db.session.flush() + db.session.add(tag) + db.session.flush() + elif kind == "character" and tag.fandom_id: + fandom_tag = Tag.query.get(tag.fandom_id) elif tag.kind == "character" and tag.fandom_id: # Existing character tag with a fandom — grab the fandom tag for auto-apply fandom_tag = Tag.query.get(tag.fandom_id) @@ -786,9 +936,10 @@ def set_tag_fandom(tag_id): fandom_id = request.form.get("fandom_id", type=int) fandom_name = (request.form.get("fandom_name") or "").strip() - # Get the character's base name (strip existing fandom suffix) + # Get the character's base name (strip existing fandom suffix) and normalize it. char_part = tag.name.split(":", 1)[1] if ":" in tag.name else tag.name char_base, _ = _parse_character_fandom(char_part) + char_base = normalize_display_name(char_base) if fandom_id: fandom_tag = Tag.query.get(fandom_id) @@ -796,8 +947,9 @@ def set_tag_fandom(tag_id): return jsonify(ok=False, error="Invalid fandom tag"), 400 fandom_display = fandom_tag.name.split(":", 1)[1] if ":" in fandom_tag.name else fandom_tag.name elif fandom_name: + # _ensure_fandom_tag normalizes internally. fandom_tag = _ensure_fandom_tag(fandom_name) - fandom_display = fandom_name + fandom_display = fandom_tag.name.split(":", 1)[1] else: # Clear fandom tag.fandom_id = None @@ -821,6 +973,33 @@ def set_tag_fandom(tag_id): tag.name = new_name db.session.commit() + # Retroactive backfill: every image already tagged with this character should + # also have the fandom tag attached. Runs in a second transaction so a failed + # backfill doesn't roll back the rename. + try: + db.session.execute( + text(""" + INSERT INTO image_tags (image_id, tag_id) + SELECT it.image_id, :fandom_id + FROM image_tags it + WHERE it.tag_id = :char_id + AND NOT EXISTS ( + SELECT 1 FROM image_tags it2 + WHERE it2.image_id = it.image_id + AND it2.tag_id = :fandom_id + ) + """), + {'char_id': tag.id, 'fandom_id': fandom_tag.id}, + ) + db.session.commit() + except Exception as e: + db.session.rollback() + # Rename already committed; log and continue. Sweep button recovers. + current_app.logger.warning( + "set_tag_fandom backfill failed for tag %s -> fandom %s: %s", + tag.id, fandom_tag.id, e, + ) + return jsonify(ok=True, tag={ "id": tag.id, "name": tag.name, @@ -2112,23 +2291,38 @@ def bulk_add_tag(): if ':' in tag_name: kind = tag_name.split(':', 1)[0] else: + # No prefix — treat as a user-typed general tag. Strip underscores so + # WD14-style names pasted into the bulk box converge with our convention. + from app.utils.tag_names import underscores_to_spaces kind = 'user' + tag_name = underscores_to_spaces(tag_name) # Get or create the tag tag = Tag.query.filter_by(name=tag_name).first() fandom_tag = None if not tag: - tag = Tag(name=tag_name, kind=kind) - # For new character tags, parse and link fandom if kind == 'character': char_part = tag_name.split(':', 1)[1] char_name, fandom_name = _parse_character_fandom(char_part) + norm_char = normalize_display_name(char_name) if fandom_name: fandom_tag = _ensure_fandom_tag(fandom_name) + tag_name = f"character:{norm_char} ({fandom_tag.name.split(':', 1)[1]})" + else: + tag_name = f"character:{norm_char}" + elif kind == 'fandom': + fandom_part = tag_name.split(':', 1)[1] + tag_name = f"fandom:{normalize_display_name(fandom_part)}" + tag = Tag.query.filter_by(name=tag_name).first() + if not tag: + tag = Tag(name=tag_name, kind=kind) + if kind == 'character' and fandom_name: tag.fandom_id = fandom_tag.id - db.session.add(tag) - db.session.flush() + db.session.add(tag) + db.session.flush() + elif tag.kind == 'character' and tag.fandom_id: + fandom_tag = Tag.query.get(tag.fandom_id) elif tag.kind == 'character' and tag.fandom_id: fandom_tag = Tag.query.get(tag.fandom_id) diff --git a/app/ml/__init__.py b/app/ml/__init__.py new file mode 100644 index 0000000..2660dce --- /dev/null +++ b/app/ml/__init__.py @@ -0,0 +1 @@ +"""Machine-learning inference wrappers used by the ml-worker.""" diff --git a/app/ml/siglip.py b/app/ml/siglip.py new file mode 100644 index 0000000..693b28b --- /dev/null +++ b/app/ml/siglip.py @@ -0,0 +1,58 @@ +"""SigLIP SO400M image-embedding wrapper (PyTorch CPU).""" +from __future__ import annotations +import os + +import numpy as np +from PIL import Image + +# Defer torch/transformers imports to lazily-loaded functions to allow +# importing MODEL_VERSION in non-ML-worker contexts (e.g., web container +# centroid-recompute enqueue logic). +_torch = None +_AutoModel = None +_AutoProcessor = None + +MODEL_NAME = os.environ.get('SIGLIP_MODEL_NAME', 'google/siglip-so400m-patch14-384') +MODEL_VERSION = os.environ.get('SIGLIP_MODEL_VERSION', 'siglip-so400m-patch14-384') +# Model files live flat under this directory (written by scripts/download_models.py via +# snapshot_download(local_dir=...)). We point from_pretrained at the local path directly +# so transformers bypasses its HF cache layout and doesn't need network access at load time. +_LOCAL_DIR = os.path.join(os.environ.get('ML_MODEL_DIR', '/models'), 'siglip') + +_model = None +_processor = None + + +# NOT thread-safe. Must run in the ml-worker container with --concurrency=1. +def _load() -> None: + global _model, _processor, _torch, _AutoModel, _AutoProcessor + if _model is not None: + return + # Lazy import torch/transformers so this module can be imported in contexts + # where they're not available (e.g., web container for centroid-recompute enqueue). + import torch + from transformers import AutoModel, AutoProcessor + _torch = torch + _AutoModel = AutoModel + _AutoProcessor = AutoProcessor + + _processor = _AutoProcessor.from_pretrained(_LOCAL_DIR) + _model = _AutoModel.from_pretrained(_LOCAL_DIR) + _model.eval() + + +def infer(image_path: str) -> np.ndarray: + """Return a 1152-dim float32 numpy embedding for the image. + + SigLIP uses a MAP-pooled vision head — the pooled output is the retrieval-ready + embedding the model was trained to produce. `get_image_features` on transformers + >= 4.45 returns a BaseModelOutputWithPooling, so pull `.pooler_output` explicitly + rather than relying on the first-field fallback from indexing. + """ + _load() + img = Image.open(image_path).convert('RGB') + with _torch.no_grad(): + inputs = _processor(images=img, return_tensors='pt') + out = _model.get_image_features(**inputs) + pooled = out.pooler_output if hasattr(out, 'pooler_output') else out # (1, 1152) + return pooled[0].numpy().astype(np.float32) diff --git a/app/ml/wd14.py b/app/ml/wd14.py new file mode 100644 index 0000000..640e0cb --- /dev/null +++ b/app/ml/wd14.py @@ -0,0 +1,110 @@ +"""WD14 EVA02-Large tagger (ONNX CPU inference).""" +from __future__ import annotations +import csv +import os +from typing import Iterable + +import numpy as np +import onnxruntime as ort +from PIL import Image + +MODEL_VERSION = os.environ.get('WD14_MODEL_VERSION', 'wd-eva02-large-tagger-v3') +_MODEL_DIR = os.environ.get('ML_MODEL_DIR', '/models') +_WD14_DIR = os.path.join(_MODEL_DIR, 'wd14') +_MODEL_PATH = os.path.join(_WD14_DIR, 'model.onnx') +_TAGS_PATH = os.path.join(_WD14_DIR, 'selected_tags.csv') + +# WD14 selected_tags.csv uses Danbooru category ids: +# 0=general, 1=artist, 3=copyright, 4=character, 5=meta, 9=rating +_CATEGORY_MAP = {0: 'general', 1: 'artist', 3: 'copyright', 4: 'character', 5: 'meta', 9: 'rating'} + +_session: ort.InferenceSession | None = None +_tag_meta: list[dict] | None = None +_input_name: str | None = None +_output_name: str | None = None +_input_size: int = 448 + + +# NOT thread-safe. Must run in the ml-worker container with --concurrency=1. +def _load() -> None: + global _session, _tag_meta, _input_name, _output_name, _input_size + if _session is not None: + return + + if not os.path.isfile(_MODEL_PATH): + raise RuntimeError( + f"WD14 model file missing at {_MODEL_PATH}. " + f"Populate the /models volume via the ml-worker downloader." + ) + if not os.path.isfile(_TAGS_PATH): + raise RuntimeError( + f"WD14 selected_tags.csv missing at {_TAGS_PATH}. " + f"Populate the /models volume via the ml-worker downloader." + ) + + tag_meta: list[dict] = [] + with open(_TAGS_PATH, newline='') as f: + reader = csv.DictReader(f) + for row in reader: + tag_meta.append({ + 'name': row['name'], + 'category': _CATEGORY_MAP.get(int(row['category']), 'unknown'), + }) + + session = ort.InferenceSession( + _MODEL_PATH, + providers=['CPUExecutionProvider'], + ) + _input_name = session.get_inputs()[0].name + _output_name = session.get_outputs()[0].name + # Input shape is usually [batch, H, W, 3] NHWC; pick the spatial dim + input_shape = session.get_inputs()[0].shape + for dim in input_shape: + if isinstance(dim, int) and dim > 1: + _input_size = dim + break + # Assign sentinels last so a partially-loaded state can't be observed. + _tag_meta = tag_meta + _session = session + + +def _preprocess(image_path: str) -> np.ndarray: + img = Image.open(image_path) + if img.mode != 'RGBA': + img = img.convert('RGBA') + # Composite onto white background so transparency doesn't bias the model + bg = Image.new('RGBA', img.size, (255, 255, 255, 255)) + bg.paste(img, mask=img.split()[3] if img.mode == 'RGBA' else None) + img = bg.convert('RGB') + + w, h = img.size + side = max(w, h) + square = Image.new('RGB', (side, side), (255, 255, 255)) + square.paste(img, ((side - w) // 2, (side - h) // 2)) + square = square.resize((_input_size, _input_size), Image.BICUBIC) + + arr = np.array(square, dtype=np.float32) + # WD14 was trained on BGR + arr = arr[:, :, ::-1] + return arr[np.newaxis, :, :, :] # NHWC + + +def infer(image_path: str) -> list[dict]: + """Run WD14 on one image. Returns a list of {name, category, confidence}.""" + _load() + x = _preprocess(image_path) + out = _session.run([_output_name], {_input_name: x})[0][0] + results: list[dict] = [] + for idx, score in enumerate(out): + meta = _tag_meta[idx] + results.append({ + 'name': meta['name'], + 'category': meta['category'], + 'confidence': float(score), + }) + return results + + +def infer_filtered(image_path: str, min_any: float = 0.05) -> list[dict]: + """Same as infer() but drops tags below a floor to keep DB rows reasonable.""" + return [r for r in infer(image_path) if r['confidence'] >= min_any] diff --git a/app/models.py b/app/models.py index 70fe09f..88117bc 100644 --- a/app/models.py +++ b/app/models.py @@ -1,3 +1,5 @@ +from pgvector.sqlalchemy import Vector + from . import db # tag to object relationship table @@ -218,3 +220,61 @@ class ImportTask(db.Model): db.Index('ix_import_task_status_type', 'status', 'task_type'), db.Index('ix_import_task_batch_status', 'batch_id', 'status'), ) + + +class ImageTagPrediction(db.Model): + __tablename__ = "image_tag_prediction" + id = db.Column(db.Integer, primary_key=True) + image_id = db.Column(db.Integer, db.ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False) + tag_name = db.Column(db.Text, nullable=False) + tag_category = db.Column(db.Text, nullable=False) # general/character/copyright/rating/meta + confidence = db.Column(db.Float, nullable=False) + model_version = db.Column(db.Text, nullable=False) + created_at = db.Column(db.DateTime(timezone=True), server_default=db.func.now(), nullable=False) + + __table_args__ = ( + db.Index('idx_tag_predictions_image', 'image_id'), + db.Index('idx_tag_predictions_tag', 'tag_name'), + db.Index('idx_tag_predictions_model', 'model_version'), + ) + + +class ImageEmbedding(db.Model): + __tablename__ = "image_embedding" + image_id = db.Column(db.Integer, db.ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True) + model_version = db.Column(db.Text, primary_key=True) + embedding = db.Column(Vector(1152), nullable=False) + created_at = db.Column(db.DateTime(timezone=True), server_default=db.func.now(), nullable=False) + + +class TagReferenceEmbedding(db.Model): + __tablename__ = "tag_reference_embedding" + tag_name = db.Column(db.Text, primary_key=True) + model_version = db.Column(db.Text, primary_key=True) + tag_kind = db.Column(db.Text, nullable=True) + centroid = db.Column(Vector(1152), nullable=False) + reference_count = db.Column(db.Integer, nullable=False) + computed_at = db.Column(db.DateTime(timezone=True), server_default=db.func.now(), nullable=False) + + +class SuggestionFeedback(db.Model): + __tablename__ = "suggestion_feedback" + id = db.Column(db.Integer, primary_key=True) + image_id = db.Column(db.Integer, db.ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False) + tag_name = db.Column(db.Text, nullable=False) + suggestion_source = db.Column(db.Text, nullable=False) # wd14 | embedding_similarity + confidence = db.Column(db.Float, nullable=False) + decision = db.Column(db.Text, nullable=False) # accepted | rejected + decided_at = db.Column(db.DateTime(timezone=True), server_default=db.func.now(), nullable=False) + + __table_args__ = ( + db.Index('idx_feedback_image', 'image_id'), + db.Index('idx_feedback_tag', 'tag_name'), + ) + + +class TagSuggestionConfig(db.Model): + __tablename__ = "tag_suggestion_config" + key = db.Column(db.Text, primary_key=True) + value = db.Column(db.Text, nullable=False) + description = db.Column(db.Text, nullable=True) diff --git a/app/services/__init__.py b/app/services/__init__.py new file mode 100644 index 0000000..3660859 --- /dev/null +++ b/app/services/__init__.py @@ -0,0 +1 @@ +"""Read-path service modules called from Flask routes.""" diff --git a/app/services/tag_suggestions.py b/app/services/tag_suggestions.py new file mode 100644 index 0000000..84d7fdb --- /dev/null +++ b/app/services/tag_suggestions.py @@ -0,0 +1,235 @@ +"""Compute merged tag suggestions for an image on demand. + +Sources: + - WD14 predictions filtered by per-category thresholds + - Embedding similarity vs. character centroids (for character tags only) + +The result is grouped by category and annotated with whether each tag already +exists in the Tag table (the modal dims disallowed auto-creations). +""" +from __future__ import annotations +from typing import Iterable + +from sqlalchemy import select + +from app import db +from app.models import ( + ImageTagPrediction, + ImageEmbedding, + TagReferenceEmbedding, + TagSuggestionConfig, + ImageRecord, + Tag, + image_tags, +) + +# Tag kinds that receive embedding-similarity suggestions. None = general/topic tags. +# Adding 'artist' or 'series' here enables them with no other code changes. +ELIGIBLE_CENTROID_KINDS = ('character', 'fandom', None) + +# Maps the WD14 prediction category ('character', 'copyright') to the kind used +# in the Tag table. 'general' maps to None and is handled separately at accept +# time. Lives here (not in main) so the canonicalization helper below can use it +# without a circular import. +_WD14_CATEGORY_TO_KIND = { + 'character': 'character', + 'copyright': 'fandom', +} + + +def _canonicalize_wd14_name(raw: str, category: str) -> str: + """Rewrite a raw WD14 tag name to the canonical form ImageRepo persists. + + - 'character' / 'copyright': title-case the display portion. + - everything else ('general', 'meta', unknown): underscore-to-space only, + preserving the user's casing for general topics. + """ + from app.utils.tag_names import normalize_display_name, underscores_to_spaces + kind = _WD14_CATEGORY_TO_KIND.get(category) + if ':' in raw: + prefix, rest = raw.split(':', 1) + transform = normalize_display_name if kind in ('character', 'fandom') else underscores_to_spaces + return f"{prefix}:{transform(rest)}" + return normalize_display_name(raw) if kind in ('character', 'fandom') else underscores_to_spaces(raw) + + +# Config keys with safe defaults (used if the row is missing from tag_suggestion_config). +_DEFAULTS = { + 'threshold_general': 0.35, + 'threshold_character_wd14': 0.75, + 'threshold_copyright': 0.5, + 'threshold_meta': 0.5, + 'threshold_embedding': 0.85, + 'min_reference_images': 5.0, + 'wd14_model_version': 'wd-eva02-large-tagger-v3', + 'siglip_model_version': 'siglip-so400m-patch14-384', +} + + +def _config() -> dict: + rows = TagSuggestionConfig.query.all() + cfg = dict(_DEFAULTS) + for r in rows: + try: + cfg[r.key] = float(r.value) + except ValueError: + cfg[r.key] = r.value + return cfg + + +def _category_threshold(category: str, cfg: dict) -> float: + return { + 'general': cfg['threshold_general'], + 'character': cfg['threshold_character_wd14'], + 'copyright': cfg['threshold_copyright'], + 'meta': cfg['threshold_meta'], + }.get(category, 1.01) # unknown/rating → effectively disabled (not surfaced) + + +def _existing_tag_names_for_image(image_id: int) -> set[str]: + """Names of tags already attached to this image. + + Names are returned as-stored; since every write path canonicalizes, any + row in the DB is already in canonical form. WD14 output is canonicalized + before lookup, so equality works. + """ + rows = ( + db.session.query(Tag.name) + .join(image_tags, image_tags.c.tag_id == Tag.id) + .filter(image_tags.c.image_id == image_id) + .all() + ) + return {row[0] for row in rows} + + +def _existing_tag_names() -> set[str]: + return {row[0] for row in db.session.query(Tag.name).all()} + + +def _wd14_suggestions(image_id: int, cfg: dict, already: set[str]) -> list[dict]: + wd14_ver = cfg.get('wd14_model_version', _DEFAULTS['wd14_model_version']) + preds = ( + ImageTagPrediction.query + .filter_by(image_id=image_id, model_version=wd14_ver) + .all() + ) + out: list[dict] = [] + for p in preds: + threshold = _category_threshold(p.tag_category, cfg) + if p.confidence < threshold: + continue + canonical = _canonicalize_wd14_name(p.tag_name, p.tag_category) + if canonical in already: + continue + out.append({ + 'name': canonical, + 'category': p.tag_category, + 'confidence': p.confidence, + 'source': 'wd14', + }) + return out + + +# Maps tag kind (as stored on Tag and TagReferenceEmbedding) to the display +# category used in the modal UI's grouped suggestions. +_KIND_TO_DISPLAY_CATEGORY = { + 'character': 'character', + 'fandom': 'copyright', + None: 'general', +} + + +def _embedding_tag_suggestions(image_id: int, cfg: dict, already: set[str]) -> list[dict]: + siglip_ver = cfg.get('siglip_model_version', _DEFAULTS['siglip_model_version']) + min_refs = int(cfg.get('min_reference_images', 5)) + threshold = cfg.get('threshold_embedding', 0.85) + + image_emb = ( + ImageEmbedding.query + .filter_by(image_id=image_id, model_version=siglip_ver) + .first() + ) + if image_emb is None: + return [] + + # pgvector cosine distance; similarity = 1 - distance + distance = TagReferenceEmbedding.centroid.cosine_distance(image_emb.embedding) + rows = ( + db.session.query( + TagReferenceEmbedding.tag_name, + TagReferenceEmbedding.tag_kind, + TagReferenceEmbedding.reference_count, + distance.label('distance'), + ) + .filter(TagReferenceEmbedding.model_version == siglip_ver) + .filter(TagReferenceEmbedding.reference_count >= min_refs) + .order_by(distance) + .limit(40) + .all() + ) + out: list[dict] = [] + for tag_name, tag_kind, ref_count, dist in rows: + similarity = 1.0 - float(dist) + if similarity < threshold: + continue + if tag_name in already: + continue + category = _KIND_TO_DISPLAY_CATEGORY.get(tag_kind) + if category is None and tag_kind is not None: + # Unknown kind — skip defensively. (Should never happen because + # recompute_centroid only writes eligible kinds.) + continue + out.append({ + 'name': tag_name, + 'category': category, + 'confidence': similarity, + 'source': 'embedding_similarity', + }) + return out + + +def _merge(wd14: list[dict], embedding: list[dict]) -> list[dict]: + by_name: dict[str, dict] = {} + for s in wd14 + embedding: + existing = by_name.get(s['name']) + if existing is None or s['confidence'] > existing['confidence']: + by_name[s['name']] = s + return list(by_name.values()) + + +def get_suggestions(image_id: int, top_k_per_category: int = 10) -> dict: + """Return suggestions grouped by category for one image. + + Shape: + { + 'character': [{name, confidence, source, exists_in_db}, ...], + 'copyright': [...], + 'general': [...], + } + Ordered by confidence desc within each category. 'meta' and 'rating' are omitted. + """ + if ImageRecord.query.get(image_id) is None: + return {'character': [], 'copyright': [], 'general': []} + + cfg = _config() + already = _existing_tag_names_for_image(image_id) + + merged = _merge( + _wd14_suggestions(image_id, cfg, already), + _embedding_tag_suggestions(image_id, cfg, already), + ) + + existing_all = _existing_tag_names() + + grouped: dict[str, list[dict]] = {'character': [], 'copyright': [], 'general': []} + for s in merged: + if s['category'] not in grouped: + continue # meta / rating / artist / unknown are dropped + s['exists_in_db'] = s['name'] in existing_all + grouped[s['category']].append(s) + + for cat in grouped: + grouped[cat].sort(key=lambda x: x['confidence'], reverse=True) + grouped[cat] = grouped[cat][:top_k_per_category] + + return grouped diff --git a/app/static/js/bulk-select.js b/app/static/js/bulk-select.js index e048a99..bb45fd4 100644 --- a/app/static/js/bulk-select.js +++ b/app/static/js/bulk-select.js @@ -498,8 +498,9 @@ // Tag Refresh for Gallery Items // --------------------------- async function refreshItemTags(imageId) { - // Find the gallery item - const item = document.querySelector(`.img-clickable[data-id="${imageId}"]`); + // Only the gallery page renders tag overlays on thumbnails. Skip showcase items + // (class .masonry-item) so closing the modal doesn't inject a .tag-overlay there. + const item = document.querySelector(`.gallery-item[data-id="${imageId}"]`); if (!item) return; try { diff --git a/app/static/js/view-modal.js b/app/static/js/view-modal.js index 7ce228d..ba333ed 100644 --- a/app/static/js/view-modal.js +++ b/app/static/js/view-modal.js @@ -14,6 +14,11 @@ document.addEventListener('DOMContentLoaded', () => { const tagInput = tagForm ? tagForm.querySelector('input[name="name"]') : null; const tagAutocomplete = document.getElementById('tagAutocomplete'); const tagActionFeedback = document.getElementById('tagActionFeedback'); + const provenanceSection = document.getElementById('modalProvenanceSection'); + const provenanceList = document.getElementById('modalProvenanceList'); + const suggestionsSection = document.getElementById('modalSuggestionsSection'); + const suggestionsList = document.getElementById('modalSuggestionsList'); + const suggestionsRefresh = document.getElementById('suggestionsRefreshBtn'); // Series info elements (when image IS in a series) const seriesInfoEl = document.getElementById('modalSeriesInfo'); @@ -82,6 +87,15 @@ document.addEventListener('DOMContentLoaded', () => { return tagEditor ? tagEditor.dataset.imageId : ''; } + function getProvenanceDisplayName(name, kind) { + // Artist tag names are "artist:name" — strip the prefix. + if (kind === 'artist' && name.startsWith('artist:')) return name.slice(7); + // Post tag names are "post:platform:artist:id" — strip "post:" only; the rest + // is a placeholder replaced by PostMetadata.title or post_id once it loads. + if (kind === 'post' && name.startsWith('post:')) return name.slice(5); + return name; + } + function renderTags(tags) { if (!tagList) return; tagList.innerHTML = ''; @@ -94,12 +108,206 @@ document.addEventListener('DOMContentLoaded', () => { tagList.appendChild(chip); }); } + + const SUGGESTION_CATEGORY_ORDER = ['character', 'copyright', 'general']; + const SUGGESTION_CATEGORY_LABEL = { + character: 'Characters', + copyright: 'Fandoms', + general: 'General', + }; + + function clearSuggestions() { + if (suggestionsList) suggestionsList.innerHTML = ''; + if (suggestionsSection) suggestionsSection.style.display = 'none'; + } + + function buildSuggestionChip(suggestion, imageId) { + const chip = document.createElement('div'); + chip.className = 'suggestion-chip'; + chip.dataset.tagName = suggestion.name; + chip.dataset.source = suggestion.source; + chip.dataset.category = suggestion.category; + chip.dataset.confidence = String(suggestion.confidence); + chip.title = `${suggestion.name} (${suggestion.source})`; + + const pct = Math.round(suggestion.confidence * 100); + chip.innerHTML = ` + + + ${escapeHtml(suggestion.name)} + ${pct}% + + + `; + + const acceptBtn = chip.querySelector('.sugg-accept'); + const rejectBtn = chip.querySelector('.sugg-reject'); + if (acceptBtn) { + acceptBtn.addEventListener('click', () => acceptSuggestion(chip, imageId)); + } + if (rejectBtn) { + rejectBtn.addEventListener('click', () => rejectSuggestion(chip, imageId)); + } + return chip; + } + + function renderSuggestions(grouped, imageId) { + if (!suggestionsSection || !suggestionsList) return; + suggestionsList.innerHTML = ''; + let total = 0; + for (const cat of SUGGESTION_CATEGORY_ORDER) { + const items = (grouped && grouped[cat]) || []; + if (items.length === 0) continue; + total += items.length; + + const group = document.createElement('div'); + group.className = 'suggestions-category'; + group.dataset.category = cat; + + const label = document.createElement('div'); + label.className = 'suggestions-category-label'; + label.textContent = SUGGESTION_CATEGORY_LABEL[cat] || cat; + group.appendChild(label); + + const chips = document.createElement('div'); + chips.className = 'suggestions-chips'; + for (const item of items) { + chips.appendChild(buildSuggestionChip(item, imageId)); + } + group.appendChild(chips); + suggestionsList.appendChild(group); + } + suggestionsSection.style.display = total > 0 ? '' : 'none'; + } + + async function loadSuggestions(imageId) { + if (!imageId || !suggestionsSection) { clearSuggestions(); return; } + try { + const r = await fetch(`/image/${imageId}/suggestions`); + const j = await r.json(); + if (j.ok) renderSuggestions(j.suggestions || {}, imageId); + else clearSuggestions(); + } catch { + clearSuggestions(); + } + } + + function animateChipOut(chip) { + chip.classList.add('leaving'); + setTimeout(() => { chip.remove(); }, 200); + } + + async function acceptSuggestion(chip, imageId) { + if (chip.dataset.disabled === 'true') return; + chip.dataset.disabled = 'true'; + chip.style.pointerEvents = 'none'; + try { + const r = await fetch(`/image/${imageId}/suggestions/accept`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + tag_name: chip.dataset.tagName, + source: chip.dataset.source, + category: chip.dataset.category, + confidence: Number(chip.dataset.confidence) || 0, + }), + }); + const j = await r.json(); + if (!j.ok) { + chip.dataset.disabled = 'false'; + chip.style.pointerEvents = ''; + if (tagActionFeedback) { + tagActionFeedback.textContent = `Couldn't accept: ${j.error || 'error'}`; + } + return; + } + if (typeof loadTags === 'function') await loadTags(imageId); + if (tagActionFeedback) tagActionFeedback.textContent = `Added ${j.tag.name}`; + if (typeof window.refreshItemTags === 'function') window.refreshItemTags(imageId); + animateChipOut(chip); + } catch { + chip.dataset.disabled = 'false'; + chip.style.pointerEvents = ''; + } + } + + async function rejectSuggestion(chip, imageId) { + chip.style.pointerEvents = 'none'; + try { + await fetch(`/image/${imageId}/suggestions/reject`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + tag_name: chip.dataset.tagName, + source: chip.dataset.source, + confidence: Number(chip.dataset.confidence) || 0, + }), + }); + animateChipOut(chip); + if (tagActionFeedback) tagActionFeedback.textContent = `Rejected ${chip.dataset.tagName}`; + } catch { + chip.style.pointerEvents = ''; + } + } + + function renderProvenance(tags) { + if (!provenanceSection || !provenanceList) return; + provenanceList.innerHTML = ''; + if (!tags || tags.length === 0) { + provenanceSection.style.display = 'none'; + return; + } + tags.forEach(t => { + const chip = document.createElement('a'); + chip.className = 'provenance-chip'; + chip.href = `/gallery?tag=${encodeURIComponent(t.name)}`; + chip.title = t.name; + chip.dataset.tagName = t.name; + chip.dataset.kind = t.kind; + const icon = getTagIcon(t.kind); + const placeholder = escapeHtml(getProvenanceDisplayName(t.name, t.kind)); + chip.innerHTML = `${icon} ${placeholder} ↗`; + provenanceList.appendChild(chip); + if (t.kind === 'post') upgradeProvenanceChip(chip, t.name); + }); + provenanceSection.style.display = ''; + } + + async function upgradeProvenanceChip(chip, tagName) { + try { + const r = await fetch(`/api/post-metadata/by-tag-name/${encodeURIComponent(tagName)}`); + const j = await r.json(); + if (!j.ok || !j.metadata) return; + const pm = j.metadata; + if (pm.platform) chip.dataset.platform = pm.platform; + const label = pm.title || pm.post_id; + if (label) { + const labelEl = chip.querySelector('.provenance-label'); + if (labelEl) labelEl.textContent = label; + } + } catch { + // Fetch failed — chip keeps its placeholder label and neutral accent. + } + } async function loadTags(imageId) { - if (!imageId) { renderTags([]); return; } + if (!imageId) { renderTags([]); renderProvenance([]); return; } try { const r = await fetch(`/image/${imageId}/tags`); const j = await r.json(); - if (j.ok) renderTags(j.tags); + if (j.ok) { + const all = j.tags || []; + // Provenance: artist first, then post — non-editable, system-tagged origin info. + const provenanceTags = [ + ...all.filter(t => t.kind === 'artist'), + ...all.filter(t => t.kind === 'post'), + ]; + // Editable list excludes provenance kinds and hidden kinds (source, archive). + const hiddenFromModal = new Set(['artist', 'post', 'source', 'archive']); + const editableTags = all.filter(t => !hiddenFromModal.has(t.kind)); + renderTags(editableTags); + renderProvenance(provenanceTags); + loadSuggestions(imageId); + } } catch { // ignore } @@ -109,7 +317,9 @@ document.addEventListener('DOMContentLoaded', () => { // Gallery item tag refresh (fallback if bulk-select.js not loaded) // --------------------------- async function refreshItemTagsFallback(imageId) { - const item = document.querySelector(`.img-clickable[data-id="${imageId}"]`); + // Only the gallery page renders tag overlays on thumbnails. Showcase thumbnails + // use a tag-count badge instead and must not have .tag-overlay forced onto them. + const item = document.querySelector(`.gallery-item[data-id="${imageId}"]`); if (!item) return; try { @@ -601,15 +811,6 @@ document.addEventListener('DOMContentLoaded', () => { modal.classList.add('active'); updateImage(index); history.replaceState({ modalIndex: index }, '', window.location.href); - // Focus the tag input after modal opens (desktop only - avoids keyboard popup on mobile) - if (tagInput && !isTouchDevice()) { - setTimeout(() => tagInput.focus(), 100); - } - } - - // Detect touch devices to avoid auto-focus keyboard popup - function isTouchDevice() { - return ('ontouchstart' in window) || (navigator.maxTouchPoints > 0); } function closeModal() { @@ -623,6 +824,8 @@ document.addEventListener('DOMContentLoaded', () => { // clear tag UI setEditorImageId(''); renderTags([]); + renderProvenance([]); + clearSuggestions(); hideSeriesInfo(); hideAddToSeries(); history.replaceState({}, '', window.location.pathname + window.location.search); @@ -821,4 +1024,11 @@ document.addEventListener('DOMContentLoaded', () => { location.reload(); } }); + + if (suggestionsRefresh) { + suggestionsRefresh.addEventListener('click', () => { + const currentId = getEditorImageId(); + if (currentId) loadSuggestions(currentId); + }); + } }); diff --git a/app/static/style.css b/app/static/style.css index 42d3a78..cb2c7eb 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -840,6 +840,182 @@ header { color: #ef4444; } +/* Modal provenance section — system-generated, non-editable tags (artist, post) */ +.modal-provenance-section { + border-bottom: none; +} +.modal-provenance-section .provenance-list { + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 0.25rem; +} + +.provenance-chip { + display: inline-flex; + align-items: center; + gap: 0.4rem; + padding: 2px 8px 2px 10px; + border-radius: 999px; + background: transparent; + color: var(--text-dim); + text-decoration: none; + border: 1px solid rgba(255, 255, 255, 0.15); + border-left-width: 2px; + border-left-color: rgba(255, 255, 255, 0.4); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + font: 12px/1.6 system-ui, sans-serif; + max-width: clamp(8rem, 22vw, 14rem); + transition: border-color 0.15s ease, background 0.15s ease, color 0.15s ease; +} + +.provenance-chip:hover { + border-color: rgba(255, 255, 255, 0.35); + background: rgba(255, 255, 255, 0.04); + color: var(--text); +} + +.provenance-chip[data-platform="patreon"] { + border-left-color: #f96854; +} +.provenance-chip[data-platform="subscribestar"] { + border-left-color: #009cde; +} +.provenance-chip[data-platform="hentaifoundry"] { + border-left-color: #c8232c; +} + +.provenance-chip .provenance-arrow { + opacity: 0.6; + font-size: 0.85em; +} + +/* Modal suggestions section — ML-backed chips with accept/reject */ +.modal-suggestions-section .section-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 0.4rem; +} +.modal-suggestions-section .section-title { + font-size: 0.85rem; + font-weight: 600; + color: var(--text-dim); + letter-spacing: 0.04em; + text-transform: uppercase; +} +.modal-suggestions-section .suggestions-refresh { + background: transparent; + border: none; + color: var(--text-dim); + cursor: pointer; + font-size: 0.9rem; + padding: 0 0.25rem; +} +.modal-suggestions-section .suggestions-refresh:hover { + color: var(--text); +} +.suggestions-list { + display: flex; + flex-direction: column; + gap: 0.35rem; +} +.suggestions-category { + display: flex; + flex-direction: column; + gap: 0.2rem; +} +.suggestions-category-label { + font-size: 0.7rem; + color: var(--text-dim); + opacity: 0.65; + letter-spacing: 0.05em; + text-transform: uppercase; +} +.suggestions-chips { + display: flex; + flex-direction: column; + gap: 4px; +} +.suggestion-chip { + display: flex; + align-items: stretch; + gap: 6px; + min-height: 36px; + padding: 0; + border-radius: 6px; + background: rgba(255, 255, 255, 0.02); + color: var(--text-dim); + border: 1px dashed rgba(255, 255, 255, 0.18); + font: 13px/1.3 system-ui, sans-serif; + transition: border-color 0.15s ease, background 0.15s ease, color 0.15s ease, opacity 0.15s ease, transform 0.18s ease; +} +.suggestion-chip:hover { + border-color: rgba(255, 255, 255, 0.35); + background: rgba(255, 255, 255, 0.04); + color: var(--text); +} +.suggestion-chip .sugg-body { + flex: 1; + display: flex; + align-items: center; + gap: 8px; + min-width: 0; + padding: 0 2px; +} +.suggestion-chip .sugg-label { + flex: 1; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.suggestion-chip .sugg-confidence { + font-size: 0.72rem; + opacity: 0.6; + font-variant-numeric: tabular-nums; + flex-shrink: 0; +} +.suggestion-chip .sugg-btn { + flex: 0 0 auto; + width: 32px; + min-height: 32px; + display: inline-flex; + align-items: center; + justify-content: center; + background: transparent; + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 4px; + color: inherit; + cursor: pointer; + padding: 0; + font-size: 0.95rem; + line-height: 1; + margin: 1px; + transition: background 0.12s ease, color 0.12s ease, border-color 0.12s ease; +} +.suggestion-chip .sugg-btn.sugg-accept { + color: #8be78b; + border-color: rgba(139, 231, 139, 0.35); +} +.suggestion-chip .sugg-btn.sugg-accept:hover { + background: rgba(139, 231, 139, 0.12); + border-color: rgba(139, 231, 139, 0.6); +} +.suggestion-chip .sugg-btn.sugg-reject { + color: #e78b8b; + border-color: rgba(231, 139, 139, 0.35); +} +.suggestion-chip .sugg-btn.sugg-reject:hover { + background: rgba(231, 139, 139, 0.12); + border-color: rgba(231, 139, 139, 0.6); +} +.suggestion-chip.leaving { + opacity: 0; + transform: translateX(6px); +} + /* Tag form with autocomplete */ .tag-form { display: flex; diff --git a/app/tasks/import_file.py b/app/tasks/import_file.py index bf542e5..0fce55b 100644 --- a/app/tasks/import_file.py +++ b/app/tasks/import_file.py @@ -364,6 +364,14 @@ def import_media_file(self, task_id: int): if sidecar_path: apply_sidecar_metadata.delay(record.id, sidecar_path) + # Queue ML inference (WD14 tagging + SigLIP embedding) + try: + from app.tasks.ml import tag_and_embed + tag_and_embed.apply_async(args=[record.id], queue='ml') + except Exception as ml_err: + # Never let an enqueue failure roll back an otherwise-successful import. + log.warning(f"Could not enqueue tag_and_embed for image {record.id}: {ml_err}") + # Update batch stats if task.batch_id: from app.tasks.scan import update_batch_stats diff --git a/app/tasks/maintenance.py b/app/tasks/maintenance.py new file mode 100644 index 0000000..a381d9f --- /dev/null +++ b/app/tasks/maintenance.py @@ -0,0 +1,66 @@ +"""Maintenance-queue Celery tasks. Lightweight, user-triggered, non-ML.""" +import logging +from celery import shared_task +from sqlalchemy import text + +from app import db +from app.models import Tag + +log = logging.getLogger(__name__) + + +@shared_task( + name='app.tasks.maintenance.sync_character_fandoms', + soft_time_limit=120, + time_limit=180, +) +def sync_character_fandoms(): + """For every character tag with a non-null fandom_id, attach the fandom + tag to every image tagged with the character but not the fandom. + + Additive only — never removes fandom tags. + + Returns a summary dict with counts. + """ + chars = ( + Tag.query + .filter_by(kind='character') + .filter(Tag.fandom_id.isnot(None)) + .all() + ) + total_added = 0 + failed = 0 + for char in chars: + try: + result = db.session.execute( + text(""" + INSERT INTO image_tags (image_id, tag_id) + SELECT it.image_id, :fandom_id + FROM image_tags it + WHERE it.tag_id = :char_id + AND NOT EXISTS ( + SELECT 1 FROM image_tags it2 + WHERE it2.image_id = it.image_id + AND it2.tag_id = :fandom_id + ) + """), + {'char_id': char.id, 'fandom_id': char.fandom_id}, + ) + total_added += result.rowcount or 0 + db.session.commit() + except Exception as e: + db.session.rollback() + failed += 1 + log.warning( + "sync_character_fandoms: char tag %s (%s) failed: %s", + char.id, char.name, e, + ) + log.info( + "sync_character_fandoms: scanned %d characters, added %d image↔fandom links, %d failed", + len(chars), total_added, failed, + ) + return { + 'characters_scanned': len(chars), + 'links_added': total_added, + 'characters_failed': failed, + } diff --git a/app/tasks/ml.py b/app/tasks/ml.py new file mode 100644 index 0000000..91bb4d4 --- /dev/null +++ b/app/tasks/ml.py @@ -0,0 +1,239 @@ +"""ML inference Celery tasks (WD14 tagging, SigLIP embedding, centroid recompute).""" +from __future__ import annotations +import logging +import os +import time + +import numpy as np +from sqlalchemy import and_, func +from sqlalchemy.dialects.postgresql import insert as pg_insert + +from app.celery_app import celery +from app import db +from app.models import ( + ImageRecord, + Tag, + ImageTagPrediction, + ImageEmbedding, + TagReferenceEmbedding, + TagSuggestionConfig, + image_tags, +) + +log = logging.getLogger('celery.tasks.ml') + +# Minimum raw WD14 confidence to bother storing. Below this, rows are noise. +WD14_STORE_FLOOR = float(os.environ.get('WD14_STORE_FLOOR', '0.05')) + + +def _config_value(key: str, default: str) -> str: + row = TagSuggestionConfig.query.filter_by(key=key).first() + return row.value if row else default + + +@celery.task(bind=True, name='app.tasks.ml.tag_and_embed', + max_retries=2, default_retry_delay=60, + soft_time_limit=120, time_limit=180) +def tag_and_embed(self, image_id: int): + """Run WD14 + SigLIP on one image and persist predictions and embedding.""" + from app.ml import wd14, siglip # lazy import so web process never loads torch + + image = ImageRecord.query.get(image_id) + if image is None: + log.warning(f"tag_and_embed: image {image_id} not found") + return {'status': 'missing'} + + if not os.path.exists(image.filepath): + log.warning(f"tag_and_embed: file missing at {image.filepath}") + return {'status': 'file_missing'} + + try: + t0 = time.time() + raw = wd14.infer_filtered(image.filepath, min_any=WD14_STORE_FLOOR) + embedding = siglip.infer(image.filepath) + t_inf = time.time() - t0 + + # Remove any pre-existing predictions/embedding for this image+model_version pair + db.session.query(ImageTagPrediction).filter( + ImageTagPrediction.image_id == image_id, + ImageTagPrediction.model_version == wd14.MODEL_VERSION, + ).delete(synchronize_session=False) + db.session.query(ImageEmbedding).filter( + ImageEmbedding.image_id == image_id, + ImageEmbedding.model_version == siglip.MODEL_VERSION, + ).delete(synchronize_session=False) + + for pred in raw: + db.session.add(ImageTagPrediction( + image_id=image_id, + tag_name=pred['name'], + tag_category=pred['category'], + confidence=pred['confidence'], + model_version=wd14.MODEL_VERSION, + )) + + db.session.add(ImageEmbedding( + image_id=image_id, + model_version=siglip.MODEL_VERSION, + embedding=embedding.tolist(), + )) + + db.session.commit() + log.info(f"tag_and_embed: image {image_id} done in {t_inf:.2f}s ({len(raw)} predictions)") + return {'status': 'ok', 'predictions': len(raw), 'duration_s': t_inf} + except Exception as e: + db.session.rollback() + log.error(f"tag_and_embed failed for image {image_id}: {e}", exc_info=True) + raise self.retry(exc=e) + + +@celery.task(bind=True, name='app.tasks.ml.backfill', + soft_time_limit=None, time_limit=None) +def backfill(self, batch_size: int = 50, pause_seconds: float = 0.5): + """Enqueue tag_and_embed for every image missing predictions or embeddings for the current model versions. + + Uses keyset pagination on image_record.id so the loop moves forward monotonically. + Re-running after some tag_and_embed tasks have completed is safe — those images + simply drop out of the filter on the next run. + + Runs on the ml queue with concurrency=1, so enqueued tag_and_embed tasks only + start executing after this task finishes. Keyset pagination prevents the loop + from seeing its own pending enqueues as "still missing" and re-enqueueing them. + """ + from app.ml.wd14 import MODEL_VERSION as WD14_VER + from app.ml.siglip import MODEL_VERSION as SIGLIP_VER + + enqueued_total = 0 + last_id = 0 + while True: + q = ( + db.session.query(ImageRecord.id) + .outerjoin( + ImageTagPrediction, + and_( + ImageTagPrediction.image_id == ImageRecord.id, + ImageTagPrediction.model_version == WD14_VER, + ), + ) + .outerjoin( + ImageEmbedding, + and_( + ImageEmbedding.image_id == ImageRecord.id, + ImageEmbedding.model_version == SIGLIP_VER, + ), + ) + .filter(ImageRecord.id > last_id) + .filter( + (ImageTagPrediction.image_id.is_(None)) + | (ImageEmbedding.image_id.is_(None)) + ) + .order_by(ImageRecord.id) + .limit(batch_size) + ) + ids = [row[0] for row in q.all()] + if not ids: + break + for image_id in ids: + tag_and_embed.apply_async(args=[image_id], queue='ml') + enqueued_total += len(ids) + last_id = ids[-1] + log.info(f"backfill: enqueued batch of {len(ids)} (total {enqueued_total}, last_id {last_id})") + time.sleep(pause_seconds) + log.info(f"backfill: complete, enqueued {enqueued_total} images") + return {'status': 'ok', 'enqueued': enqueued_total} + + +@celery.task(bind=True, name='app.tasks.ml.recompute_centroid', + soft_time_limit=60, time_limit=120) +def recompute_centroid(self, tag_name: str): + """Recompute the mean embedding for an eligible tag from its currently-associated images. + + Eligible tag kinds are defined by ELIGIBLE_CENTROID_KINDS in + app.services.tag_suggestions. Tags of any other kind return early with + status='ineligible_kind' and do not write a centroid row. + """ + from app.ml.siglip import MODEL_VERSION as SIGLIP_VER + from app.services.tag_suggestions import ELIGIBLE_CENTROID_KINDS + + tag = Tag.query.filter_by(name=tag_name).first() + if tag is None: + log.warning(f"recompute_centroid: tag {tag_name!r} not found") + return {'status': 'missing_tag'} + + if tag.kind not in ELIGIBLE_CENTROID_KINDS: + log.info(f"recompute_centroid: tag {tag_name!r} has ineligible kind {tag.kind!r}") + return {'status': 'ineligible_kind'} + + rows = ( + db.session.query(ImageEmbedding.embedding) + .join(image_tags, image_tags.c.image_id == ImageEmbedding.image_id) + .filter(image_tags.c.tag_id == tag.id) + .filter(ImageEmbedding.model_version == SIGLIP_VER) + .all() + ) + if not rows: + log.info(f"recompute_centroid: no embeddings yet for {tag_name}") + return {'status': 'no_embeddings'} + + vectors = np.array([np.array(r[0], dtype=np.float32) for r in rows]) + centroid = vectors.mean(axis=0) + + stmt = pg_insert(TagReferenceEmbedding).values( + tag_name=tag_name, + tag_kind=tag.kind, + model_version=SIGLIP_VER, + centroid=centroid.tolist(), + reference_count=len(rows), + computed_at=func.now(), + ) + stmt = stmt.on_conflict_do_update( + index_elements=['tag_name', 'model_version'], + set_={ + 'tag_kind': stmt.excluded.tag_kind, + 'centroid': stmt.excluded.centroid, + 'reference_count': stmt.excluded.reference_count, + 'computed_at': func.now(), + }, + ) + db.session.execute(stmt) + db.session.commit() + log.info(f"recompute_centroid: {tag_name} (kind={tag.kind}) -> n={len(rows)}") + return {'status': 'ok', 'reference_count': len(rows), 'tag_kind': tag.kind} + + +@celery.task(bind=True, name='app.tasks.ml.recompute_all_centroids', + soft_time_limit=None, time_limit=None) +def recompute_all_centroids(self): + """Enqueue recompute_centroid for every eligible tag with enough reference images. + + Uses a single aggregate query to find tags with >= min_reference_images applied + images, then enqueues one recompute_centroid task per tag on the ml queue. + """ + from app.services.tag_suggestions import ELIGIBLE_CENTROID_KINDS + + min_refs_row = TagSuggestionConfig.query.filter_by(key='min_reference_images').first() + min_refs = int(min_refs_row.value) if min_refs_row else 5 + + # Build an IS NULL / IN filter that covers ELIGIBLE_CENTROID_KINDS including None. + kinds_not_null = [k for k in ELIGIBLE_CENTROID_KINDS if k is not None] + allow_null = None in ELIGIBLE_CENTROID_KINDS + + kind_filter = Tag.kind.in_(kinds_not_null) + if allow_null: + kind_filter = kind_filter | Tag.kind.is_(None) + + rows = ( + db.session.query(Tag.name, func.count(image_tags.c.image_id).label('n')) + .join(image_tags, image_tags.c.tag_id == Tag.id) + .filter(kind_filter) + .group_by(Tag.name) + .having(func.count(image_tags.c.image_id) >= min_refs) + .all() + ) + + enqueued = 0 + for tag_name, n in rows: + recompute_centroid.apply_async(args=[tag_name], queue='ml') + enqueued += 1 + log.info(f"recompute_all_centroids: enqueued {enqueued} tags (min_refs={min_refs})") + return {'status': 'ok', 'enqueued': enqueued, 'min_refs': min_refs} diff --git a/app/templates/_gallery_modal.html b/app/templates/_gallery_modal.html index 69b373e..81bbd66 100644 --- a/app/templates/_gallery_modal.html +++ b/app/templates/_gallery_modal.html @@ -47,6 +47,11 @@ + + +
@@ -60,6 +65,15 @@
+ + + diff --git a/app/templates/settings.html b/app/templates/settings.html index a260745..9396146 100644 --- a/app/templates/settings.html +++ b/app/templates/settings.html @@ -436,6 +436,35 @@

+ +
+

ML tag suggestions

+

+ Run inference on all images missing predictions or embeddings for the current model versions. + Resumable and safe to re-run. Backfill enqueues work on the ml queue; monitor progress in the ml-worker logs. +

+
+ +
+ +

+ Recompute centroids for all eligible tags (character, fandom, and general/topic) that have at least + the minimum number of reference images. Run this after the initial backfill, or any time you've + applied many tags manually and want the suggestions to catch up. +

+
+ +
+ +

+ Attach each character's fandom to every image already tagged with that character. + Additive only — existing fandom tags are never removed. Run after you've set a + fandom on an existing character, or whenever you suspect drift. +

+
+ +
+
diff --git a/app/utils/tag_names.py b/app/utils/tag_names.py new file mode 100644 index 0000000..66e4d47 --- /dev/null +++ b/app/utils/tag_names.py @@ -0,0 +1,49 @@ +"""Normalization helpers for Tag display names. + +Keeps the first letter of every whitespace-separated word upper-case and +preserves internal punctuation. Called from every character/fandom write +path so typos like "misty" don't accumulate alongside "Misty". +""" + + +def underscores_to_spaces(s: str) -> str: + """Replace every underscore with a space. + + Used by general/null-kind write paths and as a pre-step inside + normalize_display_name so WD14's 'misty_ketchum' and a hand-typed + 'Misty Ketchum' collapse onto the same tag name. + + Examples: + "big_hair" -> "big hair" + "long_blue_hair" -> "long blue hair" + "" -> "" + """ + return s.replace('_', ' ') if s else s + + +def normalize_display_name(s: str) -> str: + """Title-case each whitespace-separated word. Preserves internal punctuation. + + Underscores are converted to spaces first so WD14-style names normalize + identically to hand-typed names. + + Examples: + "misty" -> "Misty" + "misty ketchum" -> "Misty Ketchum" + "misty_ketchum" -> "Misty Ketchum" + "o'brien" -> "O'brien" + "mary-kate" -> "Mary-kate" + "" -> "" + " " -> " " (only whitespace — returned unchanged) + """ + if not s: + return s + s = underscores_to_spaces(s) + parts = s.split(' ') + out: list[str] = [] + for p in parts: + if p == '': + out.append(p) + else: + out.append(p[:1].upper() + p[1:]) + return ' '.join(out) diff --git a/docker-compose.yml b/docker-compose.yml index 459649e..952b351 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: # PostgreSQL database postgres: - image: postgres:16-alpine + image: pgvector/pgvector:pg16 environment: POSTGRES_USER: ${DB_USER:-imagerepo} POSTGRES_PASSWORD: ${DB_PASS:-postgres} @@ -105,6 +105,37 @@ services: condition: service_healthy restart: unless-stopped + # ML inference worker (CPU-only): WD14 tagging + SigLIP embeddings + # Handles: ml queue + ml-worker: + build: + context: . + dockerfile: Dockerfile.ml + command: celery -A app.celery_app:celery worker --loglevel=info -Q ml --concurrency=1 + environment: + - TZ=${TZ:-America/New_York} + - DB_USER=${DB_USER:-imagerepo} + - DB_PASS=${DB_PASS:-postgres} + - DB_HOST=postgres + - DB_PORT=5432 + - DB_NAME=${DB_NAME:-imagerepo} + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + - ML_MODEL_DIR=/models + volumes: + - ./imagerepo/images:/images:ro + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + deploy: + resources: + limits: + cpus: '4.0' + memory: 8G + restart: unless-stopped + volumes: redis_data: postgres_data: diff --git a/docs/superpowers/plans/2026-04-18-clickable-post-tag.md b/docs/superpowers/plans/2026-04-18-clickable-post-tag.md new file mode 100644 index 0000000..0b9e078 --- /dev/null +++ b/docs/superpowers/plans/2026-04-18-clickable-post-tag.md @@ -0,0 +1,318 @@ +# Clickable Post Tag Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** In the gallery/showcase image modal, move the system-generated `post` tag out of the editable tag list into a dedicated, visually distinct provenance section whose chip links to the tag-filtered gallery view. + +**Architecture:** Frontend-only (Jinja template + vanilla JS + CSS). No backend or API changes — the existing `/image//tags` and `/api/post-metadata/by-tag-name/` endpoints cover everything. Tags returned by the image-tags endpoint are partitioned client-side into editable vs. provenance lists and rendered into separate containers. + +**Tech Stack:** Jinja2 templates, vanilla JavaScript, single CSS file (`app/static/style.css`) + +**Spec:** `docs/superpowers/specs/2026-04-18-clickable-post-tag-design.md` + +**Testing:** This repo has no automated test harness for the frontend. Verification is a manual browser check at the end of the plan. Each earlier task ends with a commit; the browser check is gated to after all code is in place. + +--- + +## File Map + +| File | Task(s) | What changes | +|------|---------|--------------| +| `app/templates/_gallery_modal.html` | 1 | New `modal-provenance-section` between Series and Tags sections | +| `app/static/style.css` | 2 | `.modal-provenance-section`, `.provenance-chip`, platform accent selectors | +| `app/static/js/view-modal.js` | 3 | Cache provenance DOM refs; partition tags in `loadTags`; add `renderProvenance`, `getProvenanceDisplayName`, `upgradeProvenanceChip`; clear provenance in `closeModal` | + +--- + +## Task 1: Add provenance section markup to the modal + +**Files:** +- Modify: `app/templates/_gallery_modal.html` (insert between line 48 and line 50) + +### Step 1.1: Open the modal template + +- [ ] Open `app/templates/_gallery_modal.html`. +- [ ] Locate the end of the Series section — the `` on line 48 that closes `modal-series-section`, immediately followed by `` on line 50. + +### Step 1.2: Insert the provenance section between Series and Tags + +- [ ] Insert this block on a new line between the closing `` of the Series section and the `` comment: + +```html + + + +``` + +The `style="display: none;"` matches the inline-style pattern already used by `modalSeriesInfo` and `modalAddToSeries` in this same file (per the project's "inline style over CSS class toggling" convention from commit `e37ce27`). + +### Step 1.3: Verify the insertion + +- [ ] Run: `grep -n "modalProvenanceSection\|modal-tags-section" app/templates/_gallery_modal.html` +- [ ] Expected: `modalProvenanceSection` appears on an earlier line than `modal-tags-section`. + +### Step 1.4: Commit + +```bash +git add app/templates/_gallery_modal.html +git commit -m "add empty provenance section to gallery modal sidebar" +``` + +--- + +## Task 2: Add CSS for provenance section and chip + +**Files:** +- Modify: `app/static/style.css` (append new block) + +### Step 2.1: Find the end of the modal sidebar / tag chip section in the stylesheet + +- [ ] Open `app/static/style.css`. +- [ ] Run: `grep -n "Tag chips in modal - with remove button" app/static/style.css` +- [ ] Navigate to that block (around line 816). Scroll to the end of the rule block that styles `.tag-editor .tag-chip` — the last closing `}` of that rule group. + +### Step 2.2: Append the provenance CSS + +- [ ] Immediately after that closing `}`, append: + +```css + +/* Modal provenance section — system-generated origin tags (post; future: source/archive) */ +.modal-provenance-section .provenance-list { + display: flex; + flex-wrap: wrap; + gap: 0.25rem; +} + +.provenance-chip { + display: inline-flex; + align-items: center; + gap: 0.4rem; + padding: 2px 8px 2px 10px; + border-radius: 999px; + background: transparent; + color: var(--text-dim); + text-decoration: none; + border: 1px solid rgba(255, 255, 255, 0.15); + border-left-width: 2px; + border-left-color: rgba(255, 255, 255, 0.4); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + font: 12px/1.6 system-ui, sans-serif; + max-width: clamp(8rem, 22vw, 14rem); + transition: border-color 0.15s ease, background 0.15s ease, color 0.15s ease; +} + +.provenance-chip:hover { + border-color: rgba(255, 255, 255, 0.35); + background: rgba(255, 255, 255, 0.04); + color: var(--text); +} + +.provenance-chip[data-platform="patreon"] { + border-left-color: #f96854; +} +.provenance-chip[data-platform="subscribestar"] { + border-left-color: #009cde; +} +.provenance-chip[data-platform="hentaifoundry"] { + border-left-color: #c8232c; +} + +.provenance-chip .provenance-arrow { + opacity: 0.6; + font-size: 0.85em; +} +``` + +### Step 2.3: Commit + +```bash +git add app/static/style.css +git commit -m "style provenance chip with platform-tinted ghost treatment" +``` + +--- + +## Task 3: Partition tags and render the provenance chip in view-modal.js + +**Files:** +- Modify: `app/static/js/view-modal.js` + +This task has several edits inside one file. Make them in order; commit once at the end. + +### Step 3.1: Cache the new DOM references + +- [ ] Open `app/static/js/view-modal.js`. +- [ ] Find the block that caches tag editor elements (around lines 11-16), beginning with `const tagEditor = document.getElementById('modalTagEditor');`. +- [ ] Immediately after the line `const tagActionFeedback = document.getElementById('tagActionFeedback');`, insert: + +```javascript + const provenanceSection = document.getElementById('modalProvenanceSection'); + const provenanceList = document.getElementById('modalProvenanceList'); +``` + +### Step 3.2: Add the provenance display-name helper + +- [ ] Find the `renderTags` function (starts around line 85). +- [ ] Immediately **above** `function renderTags(tags) {`, insert this helper: + +```javascript + function getProvenanceDisplayName(name) { + // Post tag names are "post:platform:artist:id". Strip the "post:" prefix only. + if (name.startsWith('post:')) return name.slice(5); + return name; + } + +``` + +### Step 3.3: Add renderProvenance and upgradeProvenanceChip + +- [ ] Directly **below** the existing `renderTags` function (after its closing `}` around line 96), insert: + +```javascript + + function renderProvenance(tags) { + if (!provenanceSection || !provenanceList) return; + provenanceList.innerHTML = ''; + if (!tags || tags.length === 0) { + provenanceSection.style.display = 'none'; + return; + } + tags.forEach(t => { + const chip = document.createElement('a'); + chip.className = 'provenance-chip'; + chip.href = `/gallery?tag=${encodeURIComponent(t.name)}`; + chip.title = t.name; + chip.dataset.tagName = t.name; + const placeholder = escapeHtml(getProvenanceDisplayName(t.name)); + chip.innerHTML = `📌 ${placeholder} ↗`; + provenanceList.appendChild(chip); + upgradeProvenanceChip(chip, t.name); + }); + provenanceSection.style.display = ''; + } + + async function upgradeProvenanceChip(chip, tagName) { + try { + const r = await fetch(`/api/post-metadata/by-tag-name/${encodeURIComponent(tagName)}`); + const j = await r.json(); + if (!j.ok || !j.metadata) return; + const pm = j.metadata; + if (pm.platform) chip.dataset.platform = pm.platform; + const label = pm.artist || pm.title; + if (label) { + const labelEl = chip.querySelector('.provenance-label'); + if (labelEl) labelEl.textContent = label; + } + } catch { + // Fetch failed — chip keeps its placeholder label and neutral accent. + } + } +``` + +### Step 3.4: Partition tags in loadTags + +- [ ] Find the current `loadTags` function (around lines 97-106): + +```javascript + async function loadTags(imageId) { + if (!imageId) { renderTags([]); return; } + try { + const r = await fetch(`/image/${imageId}/tags`); + const j = await r.json(); + if (j.ok) renderTags(j.tags); + } catch { + // ignore + } + } +``` + +- [ ] Replace it with: + +```javascript + async function loadTags(imageId) { + if (!imageId) { renderTags([]); renderProvenance([]); return; } + try { + const r = await fetch(`/image/${imageId}/tags`); + const j = await r.json(); + if (j.ok) { + const all = j.tags || []; + const postTags = all.filter(t => t.kind === 'post'); + const editableTags = all.filter(t => t.kind !== 'post'); + renderTags(editableTags); + renderProvenance(postTags); + } + } catch { + // ignore + } + } +``` + +### Step 3.5: Clear provenance on modal close + +- [ ] Find the `closeModal` function (around lines 615-634). +- [ ] Locate the line `renderTags([]);` inside it (around line 625). +- [ ] Immediately **after** that line, add: + +```javascript + renderProvenance([]); +``` + +### Step 3.6: Commit + +```bash +git add app/static/js/view-modal.js +git commit -m "split post tags into provenance section in image modal" +``` + +--- + +## Task 4: Manual browser verification + +**No code changes. This is a gate before declaring the feature done.** + +### Step 4.1: Start the app + +- [ ] Follow whatever start procedure this project uses locally (e.g., `docker compose up` or `python run.py` — consult `README.md` / `entrypoint.sh` if unsure). +- [ ] Open the showcase in a browser. + +### Step 4.2: Verify with an image that has a post tag + +- [ ] In the Tags nav, pick any post-backed image — or open the showcase and click any image known to come from an imported post (Patreon / SubscribeStar / HentaiFoundry). +- [ ] Confirm: + - A new chip-only section appears in the modal sidebar, **above** the editable tag list and **below** the Series section. + - The chip displays 📌, a text label, and a trailing ↗. + - The chip's **left border** is tinted with the platform color (orange / teal / red). + - The label starts as the raw `platform:artist:id` placeholder, then updates to the artist (or title, if no artist) once the platform metadata request resolves. + - The post tag is **not** present in the editable tag list below. + - Hovering the chip brightens the border and background. + - Clicking the chip navigates to `/gallery?tag=` and the gallery page shows the post's title/description/source URL block. + +### Step 4.3: Verify with an image that has NO post tag + +- [ ] Open an image with no post tag (e.g., a manually uploaded image without import metadata). +- [ ] Confirm: + - The provenance section is **not visible** — no empty heading, no empty chip row, no stray divider. + - The editable tag list still renders unchanged. + +### Step 4.4: Verify the metadata-fetch failure path (optional smoke test) + +- [ ] In DevTools, block requests to `/api/post-metadata/by-tag-name/*` and reopen the modal on a post-backed image. +- [ ] Confirm the chip renders with the neutral-grey left border (no platform tint) and the raw `platform:artist:id` label, and clicking it still navigates to the filtered gallery. + +### Step 4.5: Navigate between images with arrow keys + +- [ ] From a post-backed image modal, press the right-arrow key to advance through several images of mixed types (with and without post tags). +- [ ] Confirm the provenance section shows and hides correctly per image, with no stale chip left over from the previous image. + +--- + +## Self-Review Notes + +- Spec coverage: placement between Series and Tags ✓ (Task 1), hidden when empty ✓ (Task 3.3, 3.4, 3.5), platform-tinted ghost chip ✓ (Task 2), click → filtered gallery ✓ (Task 3.3), partition so post tag is removed from editable list ✓ (Task 3.4), placeholder + metadata upgrade ✓ (Task 3.3), fallback accent on fetch failure ✓ (Task 4.4 verifies). +- No placeholders in the plan: every code step shows the full snippet to insert; no "similar to" references. +- Type consistency: `renderProvenance`, `upgradeProvenanceChip`, `getProvenanceDisplayName` are named consistently across steps; DOM ids (`modalProvenanceSection`, `modalProvenanceList`) match between template and JS. diff --git a/docs/superpowers/plans/2026-04-18-tag-suggestions.md b/docs/superpowers/plans/2026-04-18-tag-suggestions.md new file mode 100644 index 0000000..c0828b6 --- /dev/null +++ b/docs/superpowers/plans/2026-04-18-tag-suggestions.md @@ -0,0 +1,1825 @@ +# Tag Suggestions Implementation Plan (Scan + Modal Phase) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship the ML-backed tag suggestion pipeline (WD14 tagger + SigLIP embeddings) and surface suggestions in the image modal with accept/reject controls. Multi-edit gallery suggestions are deferred. + +**Architecture:** A new `ml-worker` Celery service runs CPU-only inference on both models, writing predictions and embeddings to pgvector-backed tables. Suggestion generation happens on-read in the Flask web process. The modal loads suggestions alongside existing tag lists and lets the user apply a hybrid vocabulary policy: auto-create character/fandom/rating tags on accept, keep general tags curated. + +**Tech Stack:** Python 3.12, Flask, SQLAlchemy + pgvector, PostgreSQL 16, Celery 5 + Redis, ONNX Runtime (CPU), PyTorch (CPU) + Transformers for SigLIP, Pillow, NumPy, Alembic, vanilla JS, Jinja2. + +**Parent spec:** `docs/superpowers/specs/2026-04-18-tag-suggestions-integration.md` (which layers on `docs/superpowers/tag-suggestion-system-spec.md`). + +**Testing:** This repo has no automated test harness. Per-task verification is structural (grep / read) and a manual inference smoke test for ML tasks; end-to-end verification is a gated manual browser check at the very end. + +--- + +## File Map + +| File | Task | Responsibility | +|------|------|----------------| +| `requirements.txt` | 1 | Add `pgvector` so the web process can read `Vector` columns | +| `migrations/versions/_add_tag_suggestions.py` | 1 | Enable `vector` extension, create 5 tables, HNSW index, seed config | +| `app/models.py` | 2 | Append 5 SQLAlchemy models backing the new tables | +| `app/ml/__init__.py` | 3 | Mark package | +| `app/ml/wd14.py` | 3 | Load WD14 ONNX, preprocess, run inference | +| `app/ml/siglip.py` | 3 | Load SigLIP via transformers, preprocess, run inference | +| `requirements-ml.txt` | 4 | ML-worker-only Python deps | +| `scripts/download_models.py` | 4 | Pre-fetches model weights at image build time | +| `Dockerfile.ml` | 4 | Build ml-worker image with CPU ONNX Runtime, PyTorch CPU, models baked in | +| `app/tasks/ml.py` | 5 | Celery tasks: `tag_and_embed`, `backfill`, `recompute_centroid` | +| `docker-compose.yml` | 6 | Add `ml-worker` service on the `ml` queue | +| `app/celery_app.py` | 6 | Route `app.tasks.ml.*` to the `ml` queue | +| `app/tasks/import_file.py` | 7 | Chain `tag_and_embed` after ImageRecord commit | +| `app/services/__init__.py` | 8 | Mark package | +| `app/services/tag_suggestions.py` | 8 | Generate merged suggestion list on read | +| `app/main.py` | 9, 11 | Three suggestion endpoints + backfill trigger endpoint | +| `app/templates/_gallery_modal.html` | 10 | Add suggestions section markup | +| `app/static/style.css` | 10 | Suggestion section + chip styles | +| `app/static/js/view-modal.js` | 10 | `loadSuggestions`, `acceptSuggestion`, `rejectSuggestion`, lifecycle hooks | +| `app/templates/_settings_maintenance.html` (or equivalent) | 11 | "Run ML backfill" button | + +--- + +## Task 1: Database — pgvector extension, 5 tables, seed config + +**Files:** +- Modify: `requirements.txt` +- Create: `migrations/versions/_add_tag_suggestions.py` + +### Step 1.1: Add pgvector to base requirements + +- [ ] Open `requirements.txt`. +- [ ] At the end of the file, add: + +``` +pgvector>=0.2.5 +``` + +### Step 1.2: Rebuild the web image so pgvector is installed + +- [ ] Run: `docker compose build web worker scheduler` +- [ ] Expected: build completes; `pgvector` pulls in successfully. + +### Step 1.3: Generate the migration skeleton + +- [ ] Run: `docker compose run --rm web flask db revision -m "add tag suggestions"` +- [ ] Expected: a new file appears in `migrations/versions/` with a generated revision id. Note the path. + +### Step 1.4: Replace the migration body + +- [ ] Open the newly generated migration file. +- [ ] Replace its `upgrade()` and `downgrade()` with: + +```python +"""add tag suggestions + +Revision ID: +Revises: +Create Date: +""" +from alembic import op +import sqlalchemy as sa +from pgvector.sqlalchemy import Vector + + +# revision identifiers, used by Alembic. +revision = '' +down_revision = '' +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute("CREATE EXTENSION IF NOT EXISTS vector") + + op.create_table( + 'image_tag_prediction', + sa.Column('id', sa.Integer(), primary_key=True), + sa.Column('image_id', sa.Integer(), nullable=False), + sa.Column('tag_name', sa.Text(), nullable=False), + sa.Column('tag_category', sa.Text(), nullable=False), + sa.Column('confidence', sa.Float(), nullable=False), + sa.Column('model_version', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.ForeignKeyConstraint(['image_id'], ['image_record.id'], ondelete='CASCADE'), + ) + op.create_index('idx_tag_predictions_image', 'image_tag_prediction', ['image_id']) + op.create_index('idx_tag_predictions_tag', 'image_tag_prediction', ['tag_name']) + op.create_index('idx_tag_predictions_model', 'image_tag_prediction', ['model_version']) + + op.create_table( + 'image_embedding', + sa.Column('image_id', sa.Integer(), nullable=False), + sa.Column('model_version', sa.Text(), nullable=False), + sa.Column('embedding', Vector(1152), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.PrimaryKeyConstraint('image_id', 'model_version'), + sa.ForeignKeyConstraint(['image_id'], ['image_record.id'], ondelete='CASCADE'), + ) + op.execute( + "CREATE INDEX idx_embeddings_hnsw ON image_embedding " + "USING hnsw (embedding vector_cosine_ops)" + ) + + op.create_table( + 'character_reference_embedding', + sa.Column('character_tag', sa.Text(), nullable=False), + sa.Column('model_version', sa.Text(), nullable=False), + sa.Column('centroid', Vector(1152), nullable=False), + sa.Column('reference_count', sa.Integer(), nullable=False), + sa.Column('computed_at', sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.PrimaryKeyConstraint('character_tag', 'model_version'), + ) + + op.create_table( + 'suggestion_feedback', + sa.Column('id', sa.Integer(), primary_key=True), + sa.Column('image_id', sa.Integer(), nullable=False), + sa.Column('tag_name', sa.Text(), nullable=False), + sa.Column('suggestion_source', sa.Text(), nullable=False), + sa.Column('confidence', sa.Float(), nullable=False), + sa.Column('decision', sa.Text(), nullable=False), + sa.Column('decided_at', sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.ForeignKeyConstraint(['image_id'], ['image_record.id'], ondelete='CASCADE'), + ) + op.create_index('idx_feedback_image', 'suggestion_feedback', ['image_id']) + op.create_index('idx_feedback_tag', 'suggestion_feedback', ['tag_name']) + + op.create_table( + 'tag_suggestion_config', + sa.Column('key', sa.Text(), primary_key=True), + sa.Column('value', sa.Text(), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + ) + + # Seed default thresholds + op.bulk_insert( + sa.table( + 'tag_suggestion_config', + sa.column('key', sa.Text()), + sa.column('value', sa.Text()), + sa.column('description', sa.Text()), + ), + [ + {'key': 'threshold_general', 'value': '0.35', 'description': 'Min confidence for general tag suggestions'}, + {'key': 'threshold_character_wd14', 'value': '0.75', 'description': 'Min confidence for WD14 character tags'}, + {'key': 'threshold_copyright', 'value': '0.5', 'description': 'Min confidence for copyright/fandom tags'}, + {'key': 'threshold_rating', 'value': '0.5', 'description': 'Min confidence for rating tags'}, + {'key': 'threshold_meta', 'value': '0.5', 'description': 'Min confidence for meta tags (omitted from modal)'}, + {'key': 'threshold_embedding_character', 'value': '0.85', 'description': 'Min cosine similarity for character via embeddings'}, + {'key': 'min_reference_images_per_character', 'value': '5', 'description': 'Characters with fewer confirmed refs are excluded from embedding suggestions'}, + {'key': 'centroid_recompute_delta', 'value': '3', 'description': 'Recompute character centroid after N new confirmed references'}, + {'key': 'wd14_model_version', 'value': 'wd-eva02-large-tagger-v3', 'description': 'Current WD14 model version'}, + {'key': 'siglip_model_version', 'value': 'siglip-so400m-patch14-384', 'description': 'Current SigLIP embedding model version'}, + ], + ) + + +def downgrade(): + op.drop_table('tag_suggestion_config') + op.drop_index('idx_feedback_tag', table_name='suggestion_feedback') + op.drop_index('idx_feedback_image', table_name='suggestion_feedback') + op.drop_table('suggestion_feedback') + op.drop_table('character_reference_embedding') + op.execute("DROP INDEX IF EXISTS idx_embeddings_hnsw") + op.drop_table('image_embedding') + op.drop_index('idx_tag_predictions_model', table_name='image_tag_prediction') + op.drop_index('idx_tag_predictions_tag', table_name='image_tag_prediction') + op.drop_index('idx_tag_predictions_image', table_name='image_tag_prediction') + op.drop_table('image_tag_prediction') + op.execute("DROP EXTENSION IF EXISTS vector") +``` + +### Step 1.5: Apply the migration + +- [ ] Run: `docker compose run --rm web flask db upgrade` +- [ ] Expected: output ends with `Running upgrade -> , add tag suggestions` and no error. + +### Step 1.6: Structural verification + +- [ ] Run: `docker compose exec postgres psql -U imagerepo -d imagerepo -c "\dt"` +- [ ] Expected: the five new tables (`image_tag_prediction`, `image_embedding`, `character_reference_embedding`, `suggestion_feedback`, `tag_suggestion_config`) appear in the list. +- [ ] Run: `docker compose exec postgres psql -U imagerepo -d imagerepo -c "SELECT count(*) FROM tag_suggestion_config"` +- [ ] Expected: count is `10`. + +### Step 1.7: Commit + +```bash +git add requirements.txt migrations/versions/ +git commit -m "feat(db): add pgvector + tag-suggestion tables + seed thresholds" +``` + +--- + +## Task 2: SQLAlchemy models for the new tables + +**Files:** +- Modify: `app/models.py` (append at end) + +### Step 2.1: Append the new models + +- [ ] Open `app/models.py`. +- [ ] At the end of the file, append: + +```python +from pgvector.sqlalchemy import Vector + + +class ImageTagPrediction(db.Model): + __tablename__ = "image_tag_prediction" + id = db.Column(db.Integer, primary_key=True) + image_id = db.Column(db.Integer, db.ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False, index=True) + tag_name = db.Column(db.Text, nullable=False, index=True) + tag_category = db.Column(db.Text, nullable=False) # general/character/copyright/rating/meta + confidence = db.Column(db.Float, nullable=False) + model_version = db.Column(db.Text, nullable=False, index=True) + created_at = db.Column(db.DateTime(timezone=True), server_default=db.func.now(), nullable=False) + + +class ImageEmbedding(db.Model): + __tablename__ = "image_embedding" + image_id = db.Column(db.Integer, db.ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True) + model_version = db.Column(db.Text, primary_key=True) + embedding = db.Column(Vector(1152), nullable=False) + created_at = db.Column(db.DateTime(timezone=True), server_default=db.func.now(), nullable=False) + + +class CharacterReferenceEmbedding(db.Model): + __tablename__ = "character_reference_embedding" + character_tag = db.Column(db.Text, primary_key=True) + model_version = db.Column(db.Text, primary_key=True) + centroid = db.Column(Vector(1152), nullable=False) + reference_count = db.Column(db.Integer, nullable=False) + computed_at = db.Column(db.DateTime(timezone=True), server_default=db.func.now(), nullable=False) + + +class SuggestionFeedback(db.Model): + __tablename__ = "suggestion_feedback" + id = db.Column(db.Integer, primary_key=True) + image_id = db.Column(db.Integer, db.ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False, index=True) + tag_name = db.Column(db.Text, nullable=False, index=True) + suggestion_source = db.Column(db.Text, nullable=False) # wd14 | embedding_similarity + confidence = db.Column(db.Float, nullable=False) + decision = db.Column(db.Text, nullable=False) # accepted | rejected + decided_at = db.Column(db.DateTime(timezone=True), server_default=db.func.now(), nullable=False) + + +class TagSuggestionConfig(db.Model): + __tablename__ = "tag_suggestion_config" + key = db.Column(db.Text, primary_key=True) + value = db.Column(db.Text, nullable=False) + description = db.Column(db.Text, nullable=True) +``` + +### Step 2.2: Verify the web process still imports cleanly + +- [ ] Run: `docker compose run --rm web python -c "from app.models import ImageTagPrediction, ImageEmbedding, CharacterReferenceEmbedding, SuggestionFeedback, TagSuggestionConfig; print('ok')"` +- [ ] Expected: prints `ok` with no exceptions. + +### Step 2.3: Commit + +```bash +git add app/models.py +git commit -m "feat(models): add SQLAlchemy models for tag suggestions" +``` + +--- + +## Task 3: ML inference wrappers (WD14 + SigLIP) + +**Files:** +- Create: `app/ml/__init__.py` +- Create: `app/ml/wd14.py` +- Create: `app/ml/siglip.py` + +Both wrappers load lazily on first call and cache the session/model at module scope. They expect model files to live under `$ML_MODEL_DIR` (default `/models/`), which the ml-worker Dockerfile populates. + +### Step 3.1: Create the package + +- [ ] Create `app/ml/__init__.py` with contents: + +```python +"""Machine-learning inference wrappers used by the ml-worker.""" +``` + +### Step 3.2: Write the WD14 wrapper + +- [ ] Create `app/ml/wd14.py` with contents: + +```python +"""WD14 EVA02-Large tagger (ONNX CPU inference).""" +from __future__ import annotations +import csv +import os +from typing import Iterable + +import numpy as np +import onnxruntime as ort +from PIL import Image + +MODEL_VERSION = os.environ.get('WD14_MODEL_VERSION', 'wd-eva02-large-tagger-v3') +_MODEL_DIR = os.environ.get('ML_MODEL_DIR', '/models') +_WD14_DIR = os.path.join(_MODEL_DIR, 'wd14') +_MODEL_PATH = os.path.join(_WD14_DIR, 'model.onnx') +_TAGS_PATH = os.path.join(_WD14_DIR, 'selected_tags.csv') + +# WD14 selected_tags.csv uses Danbooru category ids: +# 0=general, 1=artist, 3=copyright, 4=character, 5=meta, 9=rating +_CATEGORY_MAP = {0: 'general', 1: 'artist', 3: 'copyright', 4: 'character', 5: 'meta', 9: 'rating'} + +_session: ort.InferenceSession | None = None +_tag_meta: list[dict] | None = None +_input_name: str | None = None +_output_name: str | None = None +_input_size: int = 448 + + +def _load() -> None: + global _session, _tag_meta, _input_name, _output_name, _input_size + if _session is not None: + return + + tag_meta: list[dict] = [] + with open(_TAGS_PATH, newline='') as f: + reader = csv.DictReader(f) + for row in reader: + tag_meta.append({ + 'name': row['name'], + 'category': _CATEGORY_MAP.get(int(row['category']), 'unknown'), + }) + _tag_meta = tag_meta + + session = ort.InferenceSession( + _MODEL_PATH, + providers=['CPUExecutionProvider'], + ) + _session = session + _input_name = session.get_inputs()[0].name + _output_name = session.get_outputs()[0].name + # Input shape is usually [batch, H, W, 3] NHWC; pick the spatial dim + input_shape = session.get_inputs()[0].shape + for dim in input_shape: + if isinstance(dim, int) and dim > 1: + _input_size = dim + break + + +def _preprocess(image_path: str) -> np.ndarray: + img = Image.open(image_path) + if img.mode != 'RGBA': + img = img.convert('RGBA') + # Composite onto white background so transparency doesn't bias the model + bg = Image.new('RGBA', img.size, (255, 255, 255, 255)) + bg.paste(img, mask=img.split()[3] if img.mode == 'RGBA' else None) + img = bg.convert('RGB') + + w, h = img.size + side = max(w, h) + square = Image.new('RGB', (side, side), (255, 255, 255)) + square.paste(img, ((side - w) // 2, (side - h) // 2)) + square = square.resize((_input_size, _input_size), Image.BICUBIC) + + arr = np.array(square, dtype=np.float32) + # WD14 was trained on BGR + arr = arr[:, :, ::-1] + return arr[np.newaxis, :, :, :] # NHWC + + +def infer(image_path: str) -> list[dict]: + """Run WD14 on one image. Returns a list of {name, category, confidence}.""" + _load() + x = _preprocess(image_path) + out = _session.run([_output_name], {_input_name: x})[0][0] + results: list[dict] = [] + for idx, score in enumerate(out): + meta = _tag_meta[idx] + results.append({ + 'name': meta['name'], + 'category': meta['category'], + 'confidence': float(score), + }) + return results + + +def infer_filtered(image_path: str, min_any: float = 0.05) -> list[dict]: + """Same as infer() but drops tags below a floor to keep DB rows reasonable.""" + return [r for r in infer(image_path) if r['confidence'] >= min_any] +``` + +### Step 3.3: Write the SigLIP wrapper + +- [ ] Create `app/ml/siglip.py` with contents: + +```python +"""SigLIP SO400M image-embedding wrapper (PyTorch CPU).""" +from __future__ import annotations +import os + +import numpy as np +import torch +from PIL import Image +from transformers import AutoModel, AutoProcessor + +MODEL_NAME = os.environ.get('SIGLIP_MODEL_NAME', 'google/siglip-so400m-patch14-384') +MODEL_VERSION = os.environ.get('SIGLIP_MODEL_VERSION', 'siglip-so400m-patch14-384') +_CACHE_DIR = os.path.join(os.environ.get('ML_MODEL_DIR', '/models'), 'siglip') + +_model = None +_processor = None + + +def _load() -> None: + global _model, _processor + if _model is not None: + return + _processor = AutoProcessor.from_pretrained(MODEL_NAME, cache_dir=_CACHE_DIR) + model = AutoModel.from_pretrained(MODEL_NAME, cache_dir=_CACHE_DIR) + model.eval() + _model = model + + +def infer(image_path: str) -> np.ndarray: + """Return a 1152-dim float32 numpy embedding for the image.""" + _load() + img = Image.open(image_path).convert('RGB') + with torch.no_grad(): + inputs = _processor(images=img, return_tensors='pt') + features = _model.get_image_features(**inputs) # shape [1, 1152] + return features[0].numpy().astype(np.float32) +``` + +### Step 3.4: Structural check + +- [ ] Run: `grep -n "def infer" app/ml/wd14.py app/ml/siglip.py` +- [ ] Expected: `infer` (and for wd14, `infer_filtered`) appear in both files. + +### Step 3.5: Commit + +```bash +git add app/ml/__init__.py app/ml/wd14.py app/ml/siglip.py +git commit -m "feat(ml): add WD14 and SigLIP inference wrappers" +``` + +--- + +## Task 4: ml-worker Dockerfile, requirements, and model-download script + +**Files:** +- Create: `requirements-ml.txt` +- Create: `scripts/download_models.py` +- Create: `Dockerfile.ml` + +### Step 4.1: Create the ML requirements list + +- [ ] Create `requirements-ml.txt` with contents: + +``` +# Base app deps (mirrors requirements.txt so the ml-worker can import app code) +Flask +Flask-SQLAlchemy +Flask-Migrate +psycopg2-binary +pillow +exifread +imagehash +gunicorn +celery[redis]>=5.3.0 +redis>=5.0.0 +pgvector>=0.2.5 + +# ML-specific deps (CPU-only) +numpy>=1.26 +onnxruntime>=1.17 # CPU build; do not install onnxruntime-gpu here +transformers>=4.40 +huggingface_hub>=0.22 +``` + +Note: `torch` CPU is installed separately in the Dockerfile via PyTorch's CPU wheel index, to avoid pulling the CUDA build. + +### Step 4.2: Create the model-download script + +- [ ] Create `scripts/download_models.py` with contents: + +```python +"""Pre-fetch WD14 and SigLIP model files into $ML_MODEL_DIR at image build time.""" +from __future__ import annotations +import os +import sys + +from huggingface_hub import hf_hub_download, snapshot_download + +MODEL_DIR = os.environ.get('ML_MODEL_DIR', '/models') +WD14_REPO = os.environ.get('WD14_REPO', 'SmilingWolf/wd-eva02-large-tagger-v3') +SIGLIP_REPO = os.environ.get('SIGLIP_REPO', 'google/siglip-so400m-patch14-384') + + +def download_wd14() -> None: + target = os.path.join(MODEL_DIR, 'wd14') + os.makedirs(target, exist_ok=True) + hf_hub_download(repo_id=WD14_REPO, filename='model.onnx', local_dir=target, local_dir_use_symlinks=False) + hf_hub_download(repo_id=WD14_REPO, filename='selected_tags.csv', local_dir=target, local_dir_use_symlinks=False) + print(f"WD14 downloaded to {target}") + + +def download_siglip() -> None: + target = os.path.join(MODEL_DIR, 'siglip') + os.makedirs(target, exist_ok=True) + snapshot_download( + repo_id=SIGLIP_REPO, + local_dir=target, + local_dir_use_symlinks=False, + allow_patterns=[ + 'config.json', + 'preprocessor_config.json', + 'tokenizer_config.json', + 'tokenizer.json', + 'special_tokens_map.json', + 'spiece.model', + 'model.safetensors', + 'pytorch_model.bin', + ], + ) + print(f"SigLIP downloaded to {target}") + + +if __name__ == '__main__': + download_wd14() + download_siglip() + sys.exit(0) +``` + +### Step 4.3: Create the ml-worker Dockerfile + +- [ ] Create `Dockerfile.ml` with contents: + +```dockerfile +# syntax=docker/dockerfile:1 +FROM python:3.12-slim AS base + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + ML_MODEL_DIR=/models + +# System deps: Pillow / image handling + psycopg2 build deps (binary wheel usually fine but libpq is needed at runtime on slim) +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgl1 \ + libglib2.0-0 \ + libpq5 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Install PyTorch CPU wheel first from its dedicated index to avoid pulling CUDA libs +RUN pip install --no-cache-dir \ + --index-url https://download.pytorch.org/whl/cpu \ + torch==2.3.1 + +# Install ML requirements +COPY requirements-ml.txt /app/requirements-ml.txt +RUN pip install --no-cache-dir -r requirements-ml.txt + +# Download models at build time so container starts fast and has no network dependency +COPY scripts/download_models.py /app/scripts/download_models.py +RUN mkdir -p ${ML_MODEL_DIR} && python /app/scripts/download_models.py + +# Copy application code (needed so Celery can import app.tasks.ml and app.ml) +COPY app /app/app + +# Celery needs these env vars set at runtime via docker-compose. +CMD ["celery", "-A", "app.celery_app:celery", "worker", "--loglevel=info", "-Q", "ml", "--concurrency=1"] +``` + +### Step 4.4: Build the ml-worker image + +- [ ] Run: `docker build -f Dockerfile.ml -t imagerepo-ml-worker:dev .` +- [ ] Expected: build succeeds. Expect it to take 10-20 minutes the first time because of model downloads (~2 GB). Image size will be large. + +### Step 4.5: Smoke-test the wrappers inside the image + +- [ ] Run (adjust path if your dev image fixtures live elsewhere): + +```bash +docker run --rm -v /path/to/a/real/image.jpg:/tmp/test.jpg \ + imagerepo-ml-worker:dev \ + python -c "from app.ml import wd14, siglip; r=wd14.infer('/tmp/test.jpg'); print(len(r), sorted(r, key=lambda x:-x['confidence'])[:5]); e=siglip.infer('/tmp/test.jpg'); print(e.shape, e.dtype)" +``` + +- [ ] Expected: WD14 prints a count (~10k) and the top-5 tags with descending confidence; SigLIP prints `(1152,) float32`. +- [ ] If WD14's input tensor name doesn't match, re-check `session.get_inputs()[0].name` — the wrapper already uses it dynamically, so this should Just Work. + +### Step 4.6: Commit + +```bash +git add requirements-ml.txt scripts/download_models.py Dockerfile.ml +git commit -m "feat(ml): add ml-worker Dockerfile, requirements, and model downloader" +``` + +--- + +## Task 5: Celery tasks — tag_and_embed, backfill, recompute_centroid + +**Files:** +- Create: `app/tasks/ml.py` + +### Step 5.1: Write the Celery task module + +- [ ] Create `app/tasks/ml.py` with contents: + +```python +"""ML inference Celery tasks (WD14 tagging, SigLIP embedding, centroid recompute).""" +from __future__ import annotations +import logging +import os +import time + +import numpy as np +from sqlalchemy import and_, func +from sqlalchemy.dialects.postgresql import insert as pg_insert + +from app.celery_app import celery +from app import db +from app.models import ( + ImageRecord, + Tag, + ImageTagPrediction, + ImageEmbedding, + CharacterReferenceEmbedding, + TagSuggestionConfig, + image_tags, +) + +log = logging.getLogger('celery.tasks.ml') + +# Minimum raw WD14 confidence to bother storing. Below this, rows are noise. +WD14_STORE_FLOOR = float(os.environ.get('WD14_STORE_FLOOR', '0.05')) + + +def _config_value(key: str, default: str) -> str: + row = TagSuggestionConfig.query.filter_by(key=key).first() + return row.value if row else default + + +@celery.task(bind=True, name='app.tasks.ml.tag_and_embed', + max_retries=2, default_retry_delay=60, + soft_time_limit=120, time_limit=180) +def tag_and_embed(self, image_id: int): + """Run WD14 + SigLIP on one image and persist predictions and embedding.""" + from app.ml import wd14, siglip # lazy import so web process never loads torch + + image = ImageRecord.query.get(image_id) + if image is None: + log.warning(f"tag_and_embed: image {image_id} not found") + return {'status': 'missing'} + + if not os.path.exists(image.filepath): + log.warning(f"tag_and_embed: file missing at {image.filepath}") + return {'status': 'file_missing'} + + try: + t0 = time.time() + raw = wd14.infer_filtered(image.filepath, min_any=WD14_STORE_FLOOR) + embedding = siglip.infer(image.filepath) + t_inf = time.time() - t0 + + # Remove any pre-existing predictions/embedding for this image+model_version pair + db.session.query(ImageTagPrediction).filter( + ImageTagPrediction.image_id == image_id, + ImageTagPrediction.model_version == wd14.MODEL_VERSION, + ).delete(synchronize_session=False) + db.session.query(ImageEmbedding).filter( + ImageEmbedding.image_id == image_id, + ImageEmbedding.model_version == siglip.MODEL_VERSION, + ).delete(synchronize_session=False) + + for pred in raw: + db.session.add(ImageTagPrediction( + image_id=image_id, + tag_name=pred['name'], + tag_category=pred['category'], + confidence=pred['confidence'], + model_version=wd14.MODEL_VERSION, + )) + + db.session.add(ImageEmbedding( + image_id=image_id, + model_version=siglip.MODEL_VERSION, + embedding=embedding.tolist(), + )) + + db.session.commit() + log.info(f"tag_and_embed: image {image_id} done in {t_inf:.2f}s ({len(raw)} predictions)") + return {'status': 'ok', 'predictions': len(raw), 'duration_s': t_inf} + except Exception as e: + db.session.rollback() + log.error(f"tag_and_embed failed for image {image_id}: {e}", exc_info=True) + raise self.retry(exc=e) + + +@celery.task(bind=True, name='app.tasks.ml.backfill', + soft_time_limit=None, time_limit=None) +def backfill(self, batch_size: int = 50, pause_seconds: float = 0.5): + """Enqueue tag_and_embed for every image missing predictions or embeddings for the current model versions. + + Resumable by construction: the query re-runs on each batch and only picks up images still without rows. + """ + from app.ml.wd14 import MODEL_VERSION as WD14_VER + from app.ml.siglip import MODEL_VERSION as SIGLIP_VER + + enqueued_total = 0 + while True: + # Images missing a prediction row OR an embedding row for current model versions + q = ( + db.session.query(ImageRecord.id) + .outerjoin( + ImageTagPrediction, + and_( + ImageTagPrediction.image_id == ImageRecord.id, + ImageTagPrediction.model_version == WD14_VER, + ), + ) + .outerjoin( + ImageEmbedding, + and_( + ImageEmbedding.image_id == ImageRecord.id, + ImageEmbedding.model_version == SIGLIP_VER, + ), + ) + .filter( + (ImageTagPrediction.image_id.is_(None)) + | (ImageEmbedding.image_id.is_(None)) + ) + .order_by(ImageRecord.id) + .limit(batch_size) + ) + ids = [row[0] for row in q.all()] + if not ids: + break + for image_id in ids: + tag_and_embed.apply_async(args=[image_id], queue='ml') + enqueued_total += len(ids) + log.info(f"backfill: enqueued batch of {len(ids)} (total {enqueued_total})") + time.sleep(pause_seconds) + log.info(f"backfill: complete, enqueued {enqueued_total} images") + return {'status': 'ok', 'enqueued': enqueued_total} + + +@celery.task(bind=True, name='app.tasks.ml.recompute_centroid', + soft_time_limit=60, time_limit=120) +def recompute_centroid(self, character_tag_name: str): + """Recompute the mean embedding for a character tag from all its currently-associated images.""" + from app.ml.siglip import MODEL_VERSION as SIGLIP_VER + + tag = Tag.query.filter_by(name=character_tag_name, kind='character').first() + if tag is None: + log.warning(f"recompute_centroid: character tag {character_tag_name!r} not found") + return {'status': 'missing_tag'} + + rows = ( + db.session.query(ImageEmbedding.embedding) + .join(image_tags, image_tags.c.image_id == ImageEmbedding.image_id) + .filter(image_tags.c.tag_id == tag.id) + .filter(ImageEmbedding.model_version == SIGLIP_VER) + .all() + ) + if not rows: + log.info(f"recompute_centroid: no embeddings yet for {character_tag_name}") + return {'status': 'no_embeddings'} + + vectors = np.array([np.array(r[0], dtype=np.float32) for r in rows]) + centroid = vectors.mean(axis=0) + + stmt = pg_insert(CharacterReferenceEmbedding).values( + character_tag=character_tag_name, + model_version=SIGLIP_VER, + centroid=centroid.tolist(), + reference_count=len(rows), + computed_at=func.now(), + ) + stmt = stmt.on_conflict_do_update( + index_elements=['character_tag', 'model_version'], + set_={ + 'centroid': stmt.excluded.centroid, + 'reference_count': stmt.excluded.reference_count, + 'computed_at': func.now(), + }, + ) + db.session.execute(stmt) + db.session.commit() + log.info(f"recompute_centroid: {character_tag_name} -> n={len(rows)}") + return {'status': 'ok', 'reference_count': len(rows)} +``` + +### Step 5.2: Structural check + +- [ ] Run: `grep -nE "^@celery\.task|def tag_and_embed|def backfill|def recompute_centroid" app/tasks/ml.py` +- [ ] Expected: four decorator/function matches for the three tasks. + +### Step 5.3: Commit + +```bash +git add app/tasks/ml.py +git commit -m "feat(ml): add Celery tasks for tagging, embedding, backfill, centroid recompute" +``` + +--- + +## Task 6: Wire ml-worker service + queue routing + +**Files:** +- Modify: `docker-compose.yml` +- Modify: `app/celery_app.py` + +### Step 6.1: Route the ml tasks to the ml queue + +- [ ] Open `app/celery_app.py`. +- [ ] Locate the `task_routes` dict (around line 62). +- [ ] Add an entry for the ml tasks. After the line: + +```python + 'app.tasks.sidecar.*': {'queue': 'sidecar'}, +``` + +insert: + +```python + + # ML inference tasks - handled by ml-worker + 'app.tasks.ml.*': {'queue': 'ml'}, +``` + +### Step 6.2: Add the ml-worker service to docker-compose.yml + +- [ ] Open `docker-compose.yml`. +- [ ] After the `scheduler` service block and before the `volumes:` block, insert: + +```yaml + # ML inference worker (CPU-only): WD14 tagging + SigLIP embeddings + # Handles: ml queue + ml-worker: + build: + context: . + dockerfile: Dockerfile.ml + command: celery -A app.celery_app:celery worker --loglevel=info -Q ml --concurrency=1 + environment: + - TZ=${TZ:-America/New_York} + - DB_USER=${DB_USER:-imagerepo} + - DB_PASS=${DB_PASS:-postgres} + - DB_HOST=postgres + - DB_PORT=5432 + - DB_NAME=${DB_NAME:-imagerepo} + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + - ML_MODEL_DIR=/models + volumes: + - ./imagerepo/images:/images:ro + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + deploy: + resources: + limits: + cpus: '4.0' + memory: 8G + restart: unless-stopped +``` + +### Step 6.3: Verify config parses + +- [ ] Run: `docker compose config > /dev/null` +- [ ] Expected: no output and exit code 0. +- [ ] Run: `grep -nE "app\.tasks\.ml|ml-worker:" app/celery_app.py docker-compose.yml` +- [ ] Expected: the ml route appears in `app/celery_app.py`, and the `ml-worker:` service key appears in `docker-compose.yml`. + +### Step 6.4: Bring up the ml-worker + +- [ ] Run: `docker compose up -d ml-worker` +- [ ] Run: `docker compose logs --tail=20 ml-worker` +- [ ] Expected: Celery banner lines show it connected to Redis and is listening on the `ml` queue. No tracebacks. + +### Step 6.5: Commit + +```bash +git add docker-compose.yml app/celery_app.py +git commit -m "feat(infra): add ml-worker service and route ml tasks to ml queue" +``` + +--- + +## Task 7: Chain tag_and_embed from the import pipeline + +**Files:** +- Modify: `app/tasks/import_file.py` + +### Step 7.1: Insert the chain call after ImageRecord commit + +- [ ] Open `app/tasks/import_file.py`. +- [ ] Find the block in `import_media_file` that currently queues the sidecar metadata task (around lines 361-365). It looks like: + +```python + # Queue sidecar metadata enrichment + # Uses archive sidecar as fallback for files from archives + sidecar_path = _find_best_sidecar(src_path, archive_path, archive_sidecar_path) + if sidecar_path: + apply_sidecar_metadata.delay(record.id, sidecar_path) +``` + +- [ ] Immediately **after** that block, add: + +```python + + # Queue ML inference (WD14 tagging + SigLIP embedding) + try: + from app.tasks.ml import tag_and_embed + tag_and_embed.apply_async(args=[record.id], queue='ml') + except Exception as ml_err: + # Never let an enqueue failure roll back an otherwise-successful import. + log.warning(f"Could not enqueue tag_and_embed for image {record.id}: {ml_err}") +``` + +### Step 7.2: Structural check + +- [ ] Run: `grep -n "tag_and_embed" app/tasks/import_file.py` +- [ ] Expected: two hits (the import and the `apply_async` call). + +### Step 7.3: Commit + +```bash +git add app/tasks/import_file.py +git commit -m "feat(import): chain tag_and_embed after ImageRecord commit" +``` + +--- + +## Task 8: Suggestion-generation service (read path) + +**Files:** +- Create: `app/services/__init__.py` +- Create: `app/services/tag_suggestions.py` + +### Step 8.1: Create the services package + +- [ ] Create `app/services/__init__.py` with contents: + +```python +"""Read-path service modules called from Flask routes.""" +``` + +### Step 8.2: Write the suggestion-generation service + +- [ ] Create `app/services/tag_suggestions.py` with contents: + +```python +"""Compute merged tag suggestions for an image on demand. + +Sources: + - WD14 predictions filtered by per-category thresholds + - Embedding similarity vs. character centroids (for character tags only) + +The result is grouped by category and annotated with whether each tag already +exists in the Tag table (the modal dims disallowed auto-creations). +""" +from __future__ import annotations +from typing import Iterable + +from sqlalchemy import select + +from app import db +from app.models import ( + ImageTagPrediction, + ImageEmbedding, + CharacterReferenceEmbedding, + TagSuggestionConfig, + ImageRecord, + Tag, + image_tags, +) + +# Config keys with safe defaults (used if the row is missing from tag_suggestion_config). +_DEFAULTS = { + 'threshold_general': 0.35, + 'threshold_character_wd14': 0.75, + 'threshold_copyright': 0.5, + 'threshold_rating': 0.5, + 'threshold_meta': 0.5, + 'threshold_embedding_character': 0.85, + 'min_reference_images_per_character': 5.0, + 'wd14_model_version': 'wd-eva02-large-tagger-v3', + 'siglip_model_version': 'siglip-so400m-patch14-384', +} + + +def _config() -> dict: + rows = TagSuggestionConfig.query.all() + cfg = dict(_DEFAULTS) + for r in rows: + try: + cfg[r.key] = float(r.value) + except ValueError: + cfg[r.key] = r.value + return cfg + + +def _category_threshold(category: str, cfg: dict) -> float: + return { + 'general': cfg['threshold_general'], + 'character': cfg['threshold_character_wd14'], + 'copyright': cfg['threshold_copyright'], + 'rating': cfg['threshold_rating'], + 'meta': cfg['threshold_meta'], + }.get(category, 1.01) # unknown → effectively disabled + + +def _existing_tag_names_for_image(image_id: int) -> set[str]: + rows = ( + db.session.query(Tag.name) + .join(image_tags, image_tags.c.tag_id == Tag.id) + .filter(image_tags.c.image_id == image_id) + .all() + ) + return {row[0] for row in rows} + + +def _existing_tag_names() -> set[str]: + return {row[0] for row in db.session.query(Tag.name).all()} + + +def _wd14_suggestions(image_id: int, cfg: dict, already: set[str]) -> list[dict]: + wd14_ver = cfg.get('wd14_model_version', _DEFAULTS['wd14_model_version']) + preds = ( + ImageTagPrediction.query + .filter_by(image_id=image_id, model_version=wd14_ver) + .all() + ) + out: list[dict] = [] + for p in preds: + threshold = _category_threshold(p.tag_category, cfg) + if p.confidence < threshold: + continue + if p.tag_name in already: + continue + out.append({ + 'name': p.tag_name, + 'category': p.tag_category, + 'confidence': p.confidence, + 'source': 'wd14', + }) + return out + + +def _embedding_character_suggestions(image_id: int, cfg: dict, already: set[str]) -> list[dict]: + siglip_ver = cfg.get('siglip_model_version', _DEFAULTS['siglip_model_version']) + min_refs = int(cfg.get('min_reference_images_per_character', 5)) + threshold = cfg.get('threshold_embedding_character', 0.85) + + image_emb = ( + ImageEmbedding.query + .filter_by(image_id=image_id, model_version=siglip_ver) + .first() + ) + if image_emb is None: + return [] + + # pgvector cosine distance; similarity = 1 - distance + distance = CharacterReferenceEmbedding.centroid.cosine_distance(image_emb.embedding) + rows = ( + db.session.query( + CharacterReferenceEmbedding.character_tag, + CharacterReferenceEmbedding.reference_count, + distance.label('distance'), + ) + .filter(CharacterReferenceEmbedding.model_version == siglip_ver) + .filter(CharacterReferenceEmbedding.reference_count >= min_refs) + .order_by(distance) + .limit(20) + .all() + ) + out: list[dict] = [] + for tag_name, ref_count, dist in rows: + similarity = 1.0 - float(dist) + if similarity < threshold: + continue + if tag_name in already: + continue + out.append({ + 'name': tag_name, + 'category': 'character', + 'confidence': similarity, + 'source': 'embedding_similarity', + }) + return out + + +def _merge(wd14: list[dict], embedding: list[dict]) -> list[dict]: + by_name: dict[str, dict] = {} + for s in wd14 + embedding: + existing = by_name.get(s['name']) + if existing is None or s['confidence'] > existing['confidence']: + by_name[s['name']] = s + return list(by_name.values()) + + +def get_suggestions(image_id: int, top_k_per_category: int = 10) -> dict: + """Return suggestions grouped by category for one image. + + Shape: + { + 'character': [{name, confidence, source, exists_in_db}, ...], + 'copyright': [...], + 'rating': [...], + 'general': [...], + } + Ordered by confidence desc within each category. 'meta' is omitted. + """ + if ImageRecord.query.get(image_id) is None: + return {'character': [], 'copyright': [], 'rating': [], 'general': []} + + cfg = _config() + already = _existing_tag_names_for_image(image_id) + + merged = _merge( + _wd14_suggestions(image_id, cfg, already), + _embedding_character_suggestions(image_id, cfg, already), + ) + + existing_all = _existing_tag_names() + + grouped: dict[str, list[dict]] = {'character': [], 'copyright': [], 'rating': [], 'general': []} + for s in merged: + if s['category'] not in grouped: + continue # meta / artist / unknown are dropped + s['exists_in_db'] = s['name'] in existing_all + grouped[s['category']].append(s) + + for cat in grouped: + grouped[cat].sort(key=lambda x: x['confidence'], reverse=True) + grouped[cat] = grouped[cat][:top_k_per_category] + + return grouped +``` + +### Step 8.3: Structural check + +- [ ] Run: `grep -n "def get_suggestions" app/services/tag_suggestions.py` +- [ ] Expected: exactly one match on a line near the bottom. + +### Step 8.4: Commit + +```bash +git add app/services/__init__.py app/services/tag_suggestions.py +git commit -m "feat(services): add tag-suggestion generation service" +``` + +--- + +## Task 9: Flask endpoints for suggestions (GET / accept / reject) + +**Files:** +- Modify: `app/main.py` + +### Step 9.1: Identify an appropriate insertion point + +- [ ] Open `app/main.py`. +- [ ] Run: `grep -n "def .*tags\|@app.route.*/image" app/main.py` +- [ ] Locate the existing `/image//tags` handler. Insert the new routes next to it (directly below is fine). + +### Step 9.2: Insert the three endpoints + +- [ ] Directly below the existing `/image//tags` handler, paste: + +```python +# --------------------------------------------------------------------------- +# Tag suggestions (ML-backed) +# --------------------------------------------------------------------------- + +_WD14_CATEGORY_TO_KIND = { + 'character': 'character', + 'copyright': 'fandom', # ImageRepo uses 'fandom' for IP/franchise + 'rating': 'rating', +} + + +@app.route('/image//suggestions', methods=['GET']) +def get_image_suggestions(image_id): + from app.services.tag_suggestions import get_suggestions + suggestions = get_suggestions(image_id) + return jsonify({'ok': True, 'suggestions': suggestions}) + + +@app.route('/image//suggestions/accept', methods=['POST']) +def accept_image_suggestion(image_id): + from app.models import ( + ImageRecord, Tag, SuggestionFeedback, + ) + payload = request.get_json(force=True, silent=True) or {} + tag_name = (payload.get('tag_name') or '').strip() + source = payload.get('source') or '' + category = payload.get('category') or '' + confidence = float(payload.get('confidence') or 0.0) + if not tag_name or not source or not category: + return jsonify({'ok': False, 'error': 'missing_fields'}), 400 + + image = ImageRecord.query.get(image_id) + if image is None: + return jsonify({'ok': False, 'error': 'image_not_found'}), 404 + + tag = Tag.query.filter_by(name=tag_name).first() + if tag is None: + # Hybrid policy: auto-create for character/copyright/rating; refuse general. + kind = _WD14_CATEGORY_TO_KIND.get(category) + if kind is None: + return jsonify({'ok': False, 'error': 'unknown_general_tag'}), 400 + tag = Tag(name=tag_name, kind=kind) + db.session.add(tag) + db.session.flush() # assign tag.id + + if tag not in image.tags: + image.tags.append(tag) + + db.session.add(SuggestionFeedback( + image_id=image_id, + tag_name=tag_name, + suggestion_source=source, + confidence=confidence, + decision='accepted', + )) + db.session.commit() + + # Schedule centroid recompute if this was a character and delta is exceeded. + if tag.kind == 'character': + from app.models import CharacterReferenceEmbedding, TagSuggestionConfig, image_tags + from app.tasks.ml import recompute_centroid + from app.ml.siglip import MODEL_VERSION as SIGLIP_VER + + delta_cfg = TagSuggestionConfig.query.filter_by(key='centroid_recompute_delta').first() + delta_threshold = int(delta_cfg.value) if delta_cfg else 3 + + current_count = ( + db.session.query(db.func.count(image_tags.c.image_id)) + .filter(image_tags.c.tag_id == tag.id) + .scalar() + ) or 0 + ref = ( + CharacterReferenceEmbedding.query + .filter_by(character_tag=tag_name, model_version=SIGLIP_VER) + .first() + ) + last_count = ref.reference_count if ref else 0 + if current_count - last_count >= delta_threshold: + try: + recompute_centroid.apply_async(args=[tag_name], queue='ml') + except Exception: + # Don't fail the accept if enqueue fails + pass + + return jsonify({ + 'ok': True, + 'tag': {'id': tag.id, 'name': tag.name, 'kind': tag.kind}, + }) + + +@app.route('/image//suggestions/reject', methods=['POST']) +def reject_image_suggestion(image_id): + from app.models import SuggestionFeedback, ImageRecord + payload = request.get_json(force=True, silent=True) or {} + tag_name = (payload.get('tag_name') or '').strip() + source = payload.get('source') or '' + confidence = float(payload.get('confidence') or 0.0) + if not tag_name or not source: + return jsonify({'ok': False, 'error': 'missing_fields'}), 400 + + if ImageRecord.query.get(image_id) is None: + return jsonify({'ok': False, 'error': 'image_not_found'}), 404 + + db.session.add(SuggestionFeedback( + image_id=image_id, + tag_name=tag_name, + suggestion_source=source, + confidence=confidence, + decision='rejected', + )) + db.session.commit() + return jsonify({'ok': True}) +``` + +Note: if `app/main.py` uses a blueprint rather than `@app.route`, adapt the decorator accordingly (e.g. `@bp.route(...)`). The `grep` in Step 9.1 reveals which pattern the existing `/image//tags` handler uses; follow it exactly. + +### Step 9.3: Verify the routes register + +- [ ] Run: `docker compose up -d web && docker compose exec web python -c "from app import app; print('\n'.join(sorted(str(r) for r in app.url_map.iter_rules() if 'suggestion' in str(r))))"` +- [ ] Expected: three rules printed — the GET, accept POST, and reject POST. + +### Step 9.4: Commit + +```bash +git add app/main.py +git commit -m "feat(api): add GET/accept/reject endpoints for tag suggestions" +``` + +--- + +## Task 10: Modal UI — template + CSS + JS + +**Files:** +- Modify: `app/templates/_gallery_modal.html` +- Modify: `app/static/style.css` +- Modify: `app/static/js/view-modal.js` + +### Step 10.1: Add the suggestions section to the modal template + +- [ ] Open `app/templates/_gallery_modal.html`. +- [ ] Run: `grep -n "SECTION: Tags\|modal-tags-section" app/templates/_gallery_modal.html` +- [ ] Locate the closing `` of the Tags section (the sibling block below the existing `` comment). +- [ ] Immediately after that closing `` (and before the next section or sidebar-wrapper close), insert: + +```html + + +``` + +### Step 10.2: Add CSS for the suggestions section + +- [ ] Open `app/static/style.css`. +- [ ] Run: `grep -n "modal-provenance-section" app/static/style.css` +- [ ] Scroll to the end of that provenance block (the last `}` before the next unrelated rule group). +- [ ] Append: + +```css + +/* Modal suggestions section — ML-backed chips with accept/reject */ +.modal-suggestions-section .section-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 0.4rem; +} +.modal-suggestions-section .section-title { + font-size: 0.85rem; + font-weight: 600; + color: var(--text-dim); + letter-spacing: 0.04em; + text-transform: uppercase; +} +.modal-suggestions-section .suggestions-refresh { + background: transparent; + border: none; + color: var(--text-dim); + cursor: pointer; + font-size: 0.9rem; + padding: 0 0.25rem; +} +.modal-suggestions-section .suggestions-refresh:hover { + color: var(--text); +} +.suggestions-list { + display: flex; + flex-direction: column; + gap: 0.35rem; +} +.suggestions-category { + display: flex; + flex-direction: column; + gap: 0.2rem; +} +.suggestions-category-label { + font-size: 0.7rem; + color: var(--text-dim); + opacity: 0.65; + letter-spacing: 0.05em; + text-transform: uppercase; +} +.suggestions-chips { + display: flex; + flex-wrap: wrap; + gap: 0.25rem; +} +.suggestion-chip { + display: inline-flex; + align-items: center; + gap: 0.35rem; + padding: 2px 8px; + border-radius: 999px; + background: transparent; + color: var(--text-dim); + border: 1px dashed rgba(255, 255, 255, 0.25); + font: 12px/1.6 system-ui, sans-serif; + max-width: clamp(8rem, 26vw, 16rem); + transition: border-color 0.15s ease, background 0.15s ease, color 0.15s ease, opacity 0.15s ease; +} +.suggestion-chip:hover { + border-color: rgba(255, 255, 255, 0.45); + background: rgba(255, 255, 255, 0.04); + color: var(--text); +} +.suggestion-chip[data-disabled="true"] { + opacity: 0.45; + cursor: not-allowed; +} +.suggestion-chip .sugg-label { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.suggestion-chip .sugg-confidence { + font-size: 0.7rem; + opacity: 0.6; +} +.suggestion-chip .sugg-btn { + background: transparent; + border: none; + color: inherit; + cursor: pointer; + padding: 0 2px; + font-size: 0.85rem; + line-height: 1; +} +.suggestion-chip .sugg-btn.sugg-accept:hover { color: #8be78b; } +.suggestion-chip .sugg-btn.sugg-reject:hover { color: #e78b8b; } +.suggestion-chip.leaving { + opacity: 0; + transform: translateX(6px); + transition: opacity 0.18s ease, transform 0.18s ease; +} +``` + +### Step 10.3: Wire up the JS lifecycle in view-modal.js + +- [ ] Open `app/static/js/view-modal.js`. +- [ ] Run: `grep -n "modalProvenanceSection\|renderProvenance\|const tagActionFeedback\|function loadTags\|function closeModal" app/static/js/view-modal.js` +- [ ] Note the line numbers — you'll be inserting near these anchors. + +- [ ] **Add DOM refs** near the existing provenance refs. Find: + +```javascript + const provenanceSection = document.getElementById('modalProvenanceSection'); + const provenanceList = document.getElementById('modalProvenanceList'); +``` + +Immediately below those two lines, insert: + +```javascript + const suggestionsSection = document.getElementById('modalSuggestionsSection'); + const suggestionsList = document.getElementById('modalSuggestionsList'); + const suggestionsRefresh = document.getElementById('suggestionsRefreshBtn'); +``` + +- [ ] **Add the category labels + helpers.** Find the existing `function renderProvenance(tags) {` line. Directly **above** it, insert: + +```javascript + const SUGGESTION_CATEGORY_ORDER = ['character', 'copyright', 'rating', 'general']; + const SUGGESTION_CATEGORY_LABEL = { + character: 'Characters', + copyright: 'Fandoms', + rating: 'Ratings', + general: 'General', + }; + + function clearSuggestions() { + if (suggestionsList) suggestionsList.innerHTML = ''; + if (suggestionsSection) suggestionsSection.style.display = 'none'; + } + + function buildSuggestionChip(suggestion, imageId) { + const chip = document.createElement('span'); + chip.className = 'suggestion-chip'; + chip.dataset.tagName = suggestion.name; + chip.dataset.source = suggestion.source; + chip.dataset.category = suggestion.category; + chip.dataset.confidence = String(suggestion.confidence); + + const blocksAutoCreate = suggestion.category === 'general' && !suggestion.exists_in_db; + if (blocksAutoCreate) { + chip.dataset.disabled = 'true'; + chip.title = 'Create this tag first in Tags admin'; + } else { + chip.title = `${suggestion.name} (${suggestion.source})`; + } + + const pct = Math.round(suggestion.confidence * 100); + chip.innerHTML = ` + + ${escapeHtml(suggestion.name)} + ${pct}% + + `; + + const acceptBtn = chip.querySelector('.sugg-accept'); + const rejectBtn = chip.querySelector('.sugg-reject'); + if (acceptBtn && !blocksAutoCreate) { + acceptBtn.addEventListener('click', () => acceptSuggestion(chip, imageId)); + } + if (rejectBtn) { + rejectBtn.addEventListener('click', () => rejectSuggestion(chip, imageId)); + } + return chip; + } + + function renderSuggestions(grouped, imageId) { + if (!suggestionsSection || !suggestionsList) return; + suggestionsList.innerHTML = ''; + let total = 0; + for (const cat of SUGGESTION_CATEGORY_ORDER) { + const items = (grouped && grouped[cat]) || []; + if (items.length === 0) continue; + total += items.length; + + const group = document.createElement('div'); + group.className = 'suggestions-category'; + group.dataset.category = cat; + + const label = document.createElement('div'); + label.className = 'suggestions-category-label'; + label.textContent = SUGGESTION_CATEGORY_LABEL[cat] || cat; + group.appendChild(label); + + const chips = document.createElement('div'); + chips.className = 'suggestions-chips'; + for (const item of items) { + chips.appendChild(buildSuggestionChip(item, imageId)); + } + group.appendChild(chips); + suggestionsList.appendChild(group); + } + suggestionsSection.style.display = total > 0 ? '' : 'none'; + } + + async function loadSuggestions(imageId) { + if (!imageId || !suggestionsSection) { clearSuggestions(); return; } + try { + const r = await fetch(`/image/${imageId}/suggestions`); + const j = await r.json(); + if (j.ok) renderSuggestions(j.suggestions || {}, imageId); + else clearSuggestions(); + } catch { + clearSuggestions(); + } + } + + function animateChipOut(chip, onDone) { + chip.classList.add('leaving'); + setTimeout(() => { + chip.remove(); + if (typeof onDone === 'function') onDone(); + }, 200); + } + + async function acceptSuggestion(chip, imageId) { + if (chip.dataset.disabled === 'true') return; + chip.style.pointerEvents = 'none'; + try { + const r = await fetch(`/image/${imageId}/suggestions/accept`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + tag_name: chip.dataset.tagName, + source: chip.dataset.source, + category: chip.dataset.category, + confidence: Number(chip.dataset.confidence) || 0, + }), + }); + const j = await r.json(); + if (!j.ok) { + chip.style.pointerEvents = ''; + if (tagActionFeedback) { + tagActionFeedback.textContent = `Couldn't accept: ${j.error || 'error'}`; + } + return; + } + animateChipOut(chip, () => { + if (tagActionFeedback) tagActionFeedback.textContent = `Added ${j.tag.name}`; + if (typeof loadTags === 'function') loadTags(imageId); + if (typeof window.refreshItemTags === 'function') window.refreshItemTags(imageId); + }); + } catch { + chip.style.pointerEvents = ''; + } + } + + async function rejectSuggestion(chip, imageId) { + chip.style.pointerEvents = 'none'; + try { + await fetch(`/image/${imageId}/suggestions/reject`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + tag_name: chip.dataset.tagName, + source: chip.dataset.source, + confidence: Number(chip.dataset.confidence) || 0, + }), + }); + animateChipOut(chip, () => { + if (tagActionFeedback) tagActionFeedback.textContent = `Rejected ${chip.dataset.tagName}`; + }); + } catch { + chip.style.pointerEvents = ''; + } + } + +``` + +- [ ] **Call `loadSuggestions` in `loadTags`.** Find the current `loadTags` function. After the `renderProvenance(postTags);` line (or wherever provenance is rendered), add: + +```javascript + loadSuggestions(imageId); +``` + +so the block looks like (approximately): + +```javascript + renderTags(editableTags); + renderProvenance(provenanceTags); + loadSuggestions(imageId); +``` + +- [ ] **Clear on modal close.** Find `closeModal`. After the existing `renderProvenance([]);` line, add: + +```javascript + clearSuggestions(); +``` + +- [ ] **Wire the refresh button.** Near the bottom of the IIFE/module body (after DOM refs are resolved but before the final `})();` if present), add: + +```javascript + if (suggestionsRefresh) { + suggestionsRefresh.addEventListener('click', () => { + const currentId = (typeof currentImageId !== 'undefined') ? currentImageId : (document.getElementById('modalImage')?.dataset?.imageId); + if (currentId) loadSuggestions(currentId); + }); + } +``` + +(Adapt `currentImageId` to whatever variable the existing modal tracks — run `grep -n "currentImageId\|modalImage" app/static/js/view-modal.js` if unsure.) + +### Step 10.4: Structural check + +- [ ] Run: `grep -n "modalSuggestionsSection\|loadSuggestions\|renderSuggestions\|acceptSuggestion\|rejectSuggestion" app/templates/_gallery_modal.html app/static/js/view-modal.js` +- [ ] Expected: the section id appears in the template; the function names appear in the JS. + +### Step 10.5: Commit + +```bash +git add app/templates/_gallery_modal.html app/static/style.css app/static/js/view-modal.js +git commit -m "feat(modal): add ML suggestions section with accept/reject chips" +``` + +--- + +## Task 11: Backfill trigger button on Settings → Maintenance + +**Files:** +- Modify: the Maintenance tab template (locate via grep below) +- Modify: `app/main.py` (or wherever settings routes live) + +### Step 11.1: Locate the Maintenance tab template + +- [ ] Run: `grep -rn "Maintenance\|maintenance-tab" app/templates/ | head -10` +- [ ] Open the file that renders the Maintenance tab content (likely `app/templates/settings_maintenance.html` or a block inside `app/templates/settings.html`). + +### Step 11.2: Add the backfill button + +- [ ] At a sensible spot in that template (near other admin action buttons if any exist), insert: + +```html +
+

ML tag suggestions

+

+ Run inference on all images missing predictions or embeddings for the current model versions. + Resumable and safe to re-run. Backfill enqueues work on the ml queue; monitor progress in the ml-worker logs. +

+
+ +
+
+``` + +### Step 11.3: Add the backfill-trigger route + +- [ ] Open `app/main.py` (or the blueprint file that hosts settings routes — follow existing settings-route locations). +- [ ] Near the existing maintenance routes, add: + +```python +@app.route('/settings/maintenance/ml-backfill', methods=['POST']) +def trigger_ml_backfill(): + from app.tasks.ml import backfill + backfill.apply_async(queue='ml') + # Redirect back to the Maintenance tab. Adjust endpoint name if needed. + return redirect(url_for('settings', tab='maintenance')) +``` + +Note: the redirect target must match whatever endpoint and query/path pattern the existing Settings → Maintenance tab uses. Run `grep -n "def settings\|/settings\b" app/main.py` if unsure; use the same endpoint. + +### Step 11.4: Structural check + +- [ ] Run: `grep -n "trigger_ml_backfill" app/main.py app/templates/` +- [ ] Expected: the function appears in `main.py`; `url_for('trigger_ml_backfill')` appears in the template. + +### Step 11.5: Commit + +```bash +git add app/main.py app/templates/ +git commit -m "feat(settings): add ML backfill trigger to Maintenance tab" +``` + +--- + +## Task 12: Manual browser verification + +**No code changes.** This is the gate before declaring the feature done. + +### Step 12.1: Bring up the full stack + +- [ ] Run: `docker compose up -d` +- [ ] Run: `docker compose ps` +- [ ] Expected: all services (`web`, `worker`, `scheduler`, `ml-worker`, `postgres`, `redis`) are Up / healthy. +- [ ] Run: `docker compose logs --tail=30 ml-worker` +- [ ] Expected: Celery banner, no tracebacks. + +### Step 12.2: Kick off backfill against existing images + +- [ ] Open Settings → Maintenance in a browser. Click **Run ML backfill**. +- [ ] Run: `docker compose logs -f ml-worker | head -30` +- [ ] Expected: you see tasks being received and completed (~1-2s each). Leave backfill running. + +### Step 12.3: Verify suggestions in the modal (post-backfill, ≥ few images processed) + +- [ ] Check a processed image exists: + `docker compose exec postgres psql -U imagerepo -d imagerepo -c "SELECT image_id, COUNT(*) FROM image_tag_prediction GROUP BY image_id LIMIT 5"`. +- [ ] Open one of those images in the gallery modal. +- [ ] Confirm the Suggestions section appears below Tags with chips grouped by category and descending confidence. +- [ ] Hover a chip — the ✕ becomes visible; cursor shows pointer. +- [ ] Click `+` on a plausible character/general suggestion; confirm: + - Chip animates out. + - The accepted tag appears in the editable Tags list. + - The gallery thumbnail (if visible) gets its tag-overlay refreshed. + - `tagActionFeedback` shows `Added `. +- [ ] Click `✕` on any remaining suggestion; confirm chip animates out and `Rejected ` is shown. +- [ ] Check feedback rows: `docker compose exec postgres psql -U imagerepo -d imagerepo -c "SELECT decision, suggestion_source, tag_name FROM suggestion_feedback ORDER BY decided_at DESC LIMIT 5"`. +- [ ] Expected: recent `accepted` and `rejected` rows matching your clicks. + +### Step 12.4: Verify the auto-create hybrid policy + +- [ ] Find (or craft) a suggestion whose tag does NOT already exist in your DB — e.g. a character WD14 knows that you haven't tagged yet. Accept it. +- [ ] Run: `docker compose exec postgres psql -U imagerepo -d imagerepo -c "SELECT id, name, kind FROM tag WHERE name="`. +- [ ] Expected: a new row exists with the correct kind (`character`, `fandom`, or `rating`). +- [ ] Find a general-category suggestion that does NOT exist in the DB. It should render dimmed with the `+` disabled. Confirm you cannot accept it from the UI. + +### Step 12.5: Verify per-upload chaining + +- [ ] Drop a new image into the import watch directory (or trigger your normal import path). +- [ ] After import completes, run: `docker compose exec postgres psql -U imagerepo -d imagerepo -c "SELECT COUNT(*) FROM image_tag_prediction WHERE image_id="`. +- [ ] Expected: within ~5-10s of import, the count is > 0. +- [ ] Same for `image_embedding`: count is exactly 1. + +### Step 12.6: Verify centroid recompute + +- [ ] Accept a character suggestion on several images (enough to exceed `centroid_recompute_delta=3`). +- [ ] Watch ml-worker logs: `docker compose logs -f ml-worker`. +- [ ] Expected: a `recompute_centroid` task fires; on completion, check: + `docker compose exec postgres psql -U imagerepo -d imagerepo -c "SELECT character_tag, reference_count, computed_at FROM character_reference_embedding ORDER BY computed_at DESC LIMIT 5"`. +- [ ] Expected: a row for the character with the current reference count. + +### Step 12.7: Verify arrow-key navigation in the modal + +- [ ] Use right-arrow to navigate between images. +- [ ] Expected: suggestions re-load per image; no stale chips carry over. + +### Step 12.8: Verify backfill resumability + +- [ ] Stop the ml-worker mid-backfill: `docker compose stop ml-worker`. +- [ ] Restart: `docker compose up -d ml-worker`. +- [ ] Click **Run ML backfill** again in Settings → Maintenance. +- [ ] Expected: it picks up exactly the images still missing predictions/embeddings. Rows counts in both tables climb monotonically. + +--- + +## Self-Review Notes + +**Spec coverage check (against `2026-04-18-tag-suggestions-integration.md`):** + +- §1 System layout → Tasks 4, 6. +- §2 Database → Task 1 (migration, pgvector), Task 2 (models). +- §3 Task flow → Task 5 (three tasks), Task 7 (chaining), accept-side centroid trigger in Task 9. +- §4 Web integration → Task 8 (service), Task 9 (routes), Task 10 (modal UI). +- §5 Hybrid vocabulary → Task 9 `_WD14_CATEGORY_TO_KIND` + general-tag guard, Task 10 chip-dim behavior. +- §6 Scope — in-scope items all mapped; deferred items intentionally not in the plan. +- §7 Verification → Task 12. + +**Placeholder scan:** No "TBD" / "implement later" / hand-wavy "add error handling" steps. Every file gets a concrete code block. Tensor-name lookups in the WD14 wrapper are dynamic, so no guessed string constants. + +**Type consistency:** Function/method names match across tasks: `tag_and_embed`, `backfill`, `recompute_centroid`, `get_suggestions`, `loadSuggestions`, `renderSuggestions`, `acceptSuggestion`, `rejectSuggestion`, `clearSuggestions`, `_WD14_CATEGORY_TO_KIND`. DOM ids match between template (`modalSuggestionsSection`, `modalSuggestionsList`, `suggestionsRefreshBtn`) and JS. + +**Known risks to watch at implementation time:** + +- ONNX Runtime input tensor shape/name varies by model revision. The wrapper uses `session.get_inputs()[0].name` + shape introspection to stay robust — but if SmilingWolf republishes with a different preprocessing convention (e.g. 512-px instead of 448), preprocessing may need a tweak. +- SigLIP's `get_image_features` signature is stable across recent `transformers` versions but should be re-checked if `transformers>=4.40` is not available. +- If `app/main.py` uses blueprints instead of `@app.route`, Task 9 and 11 decorators adapt trivially; the spec leaves that flexibility intentional. +- On first run the ml-worker image build downloads ~2 GB of models; this is a one-time cost. diff --git a/docs/superpowers/plans/2026-04-19-character-tag-integrity.md b/docs/superpowers/plans/2026-04-19-character-tag-integrity.md new file mode 100644 index 0000000..ac90956 --- /dev/null +++ b/docs/superpowers/plans/2026-04-19-character-tag-integrity.md @@ -0,0 +1,1045 @@ +# Character-Tag Integrity Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Guarantee that every image tagged with a character whose tag has a `fandom_id` also has the fandom tag attached, and enforce first-letter-of-each-word capitalization on all character and fandom display names. + +**Architecture:** A single `normalize_display_name` helper is called from every character/fandom write path. `set_tag_fandom` runs an additive backfill in a second transaction. A new `sync_character_fandoms` Celery maintenance task (`maintenance` queue) sweeps drift on demand. A one-time Alembic migration renames existing lowercase tags and merges duplicates. + +**Tech Stack:** Flask + SQLAlchemy + Postgres 16 + pgvector + Alembic (Flask-Migrate) + Celery 5. + +**Current alembic head:** `g26041901` — the new migration's `down_revision` is this value. + +--- + +## File Structure + +New: +- `app/utils/tag_names.py` — normalization helper; one function. +- `app/tasks/maintenance.py` — new module for non-ML maintenance Celery tasks. +- `migrations/versions/h26041901_normalize_character_fandom_tag_names.py` — one-time rename + merge. + +Modified: +- `app/main.py` — call normalizer from write paths, add backfill to `set_tag_fandom`, add `trigger_sync_character_fandoms` route. +- `app/celery_app.py` — add `app.tasks.maintenance` to `include=` list and route task to `maintenance` queue. +- `app/templates/settings.html` — add third Maintenance button. + +--- + +## Task 1: Normalization helper module + +**Files:** +- Create: `app/utils/tag_names.py` + +- [ ] **Step 1: Create the helper module** + +Write this exact file to `app/utils/tag_names.py`: + +```python +"""Normalization helpers for Tag display names. + +Keeps the first letter of every whitespace-separated word upper-case and +preserves internal punctuation. Called from every character/fandom write +path so typos like "misty" don't accumulate alongside "Misty". +""" + + +def normalize_display_name(s: str) -> str: + """Title-case each whitespace-separated word. Preserves internal punctuation. + + Examples: + "misty" -> "Misty" + "misty ketchum" -> "Misty Ketchum" + "o'brien" -> "O'brien" + "mary-kate" -> "Mary-kate" + "" -> "" + " " -> " " (only whitespace — returned unchanged) + """ + if not s: + return s + parts = s.split(' ') + out: list[str] = [] + for p in parts: + if p == '': + out.append(p) + else: + out.append(p[:1].upper() + p[1:]) + return ' '.join(out) +``` + +- [ ] **Step 2: Verify the module imports cleanly** + +Run: + +```bash +docker compose exec -T web python -c "from app.utils.tag_names import normalize_display_name; print(normalize_display_name('misty ketchum'))" +``` + +Expected output: `Misty Ketchum` + +- [ ] **Step 3: Commit** + +```bash +git add app/utils/tag_names.py +git commit -m "feat(tags): add normalize_display_name helper" +``` + +--- + +## Task 2: Apply normalization at every character/fandom write path + +**Files:** +- Modify: `app/main.py` + +- [ ] **Step 1: Import the helper** + +Find the imports near the top of `app/main.py` (around lines 1–16). Add to the existing imports (one new line near the other `from app...` imports): + +```python +from app.utils.tag_names import normalize_display_name +``` + +- [ ] **Step 2: Normalize inside `_ensure_fandom_tag`** + +Find (around `app/main.py:27`): + +```python +def _ensure_fandom_tag(fandom_name): + """Find or create a fandom tag by name. Returns the Tag object.""" + full_name = f"fandom:{fandom_name}" + fandom_tag = Tag.query.filter_by(name=full_name).first() + if not fandom_tag: + fandom_tag = Tag(name=full_name, kind="fandom") + db.session.add(fandom_tag) + db.session.flush() + return fandom_tag +``` + +Replace with: + +```python +def _ensure_fandom_tag(fandom_name): + """Find or create a fandom tag by name. Returns the Tag object. + + Normalizes the display name so callers don't have to remember. + """ + normalized = normalize_display_name(fandom_name.strip()) + full_name = f"fandom:{normalized}" + fandom_tag = Tag.query.filter_by(name=full_name).first() + if not fandom_tag: + fandom_tag = Tag(name=full_name, kind="fandom") + db.session.add(fandom_tag) + db.session.flush() + return fandom_tag +``` + +- [ ] **Step 3: Normalize inside `add_tag` when creating a new character tag** + +Find the block inside `add_tag` (around `app/main.py:830-840`): + +```python + if not tag: + tag = Tag(name=tag_name, kind=kind) + # For new character tags, parse and link fandom + if kind == "character": + char_part = tag_name.split(":", 1)[1] + char_name, fandom_name = _parse_character_fandom(char_part) + if fandom_name: + fandom_tag = _ensure_fandom_tag(fandom_name) + tag.fandom_id = fandom_tag.id + db.session.add(tag) + db.session.flush() +``` + +Replace with: + +```python + if not tag: + # For new character tags, parse + normalize + link fandom + if kind == "character": + char_part = tag_name.split(":", 1)[1] + char_name, fandom_name = _parse_character_fandom(char_part) + norm_char = normalize_display_name(char_name) + if fandom_name: + fandom_tag = _ensure_fandom_tag(fandom_name) + tag_name = f"character:{norm_char} ({fandom_tag.name.split(':', 1)[1]})" + else: + tag_name = f"character:{norm_char}" + elif kind == "fandom": + fandom_part = tag_name.split(":", 1)[1] + tag_name = f"fandom:{normalize_display_name(fandom_part)}" + # Look up again under the normalized name — an existing row may match. + tag = Tag.query.filter_by(name=tag_name).first() + if not tag: + tag = Tag(name=tag_name, kind=kind) + if kind == "character" and fandom_name: + tag.fandom_id = fandom_tag.id + db.session.add(tag) + db.session.flush() + elif kind == "character" and tag.fandom_id: + fandom_tag = Tag.query.get(tag.fandom_id) +``` + +*Why the second lookup:* after normalization, the rewritten `tag_name` may already exist (e.g. user typed `character:misty` but `character:Misty` is already in the DB). We re-query so we reuse that row instead of crashing on the unique constraint. The `elif` branch mirrors the existing "existing char with fandom" branch so `fandom_tag` gets populated for the auto-apply step below. + +- [ ] **Step 4: Normalize inside `add_bulk_tag`** + +Find the analogous block in `add_bulk_tag` (around `app/main.py:2240-2257`). It mirrors `add_tag` structure. Replace the create block: + +```python + tag = Tag(name=tag_name, kind=kind) + if kind == 'character': + char_part = tag_name.split(':', 1)[1] + char_name, fandom_name = _parse_character_fandom(char_part) + if fandom_name: + fandom_tag = _ensure_fandom_tag(fandom_name) + tag.fandom_id = fandom_tag.id + db.session.add(tag) + db.session.flush() + elif tag.kind == 'character' and tag.fandom_id: + fandom_tag = Tag.query.get(tag.fandom_id) +``` + +With: + +```python + if kind == 'character': + char_part = tag_name.split(':', 1)[1] + char_name, fandom_name = _parse_character_fandom(char_part) + norm_char = normalize_display_name(char_name) + if fandom_name: + fandom_tag = _ensure_fandom_tag(fandom_name) + tag_name = f"character:{norm_char} ({fandom_tag.name.split(':', 1)[1]})" + else: + tag_name = f"character:{norm_char}" + elif kind == 'fandom': + fandom_part = tag_name.split(':', 1)[1] + tag_name = f"fandom:{normalize_display_name(fandom_part)}" + tag = Tag.query.filter_by(name=tag_name).first() + if not tag: + tag = Tag(name=tag_name, kind=kind) + if kind == 'character' and fandom_name: + tag.fandom_id = fandom_tag.id + db.session.add(tag) + db.session.flush() + elif tag.kind == 'character' and tag.fandom_id: + fandom_tag = Tag.query.get(tag.fandom_id) + elif tag.kind == 'character' and tag.fandom_id: + fandom_tag = Tag.query.get(tag.fandom_id) +``` + +The outermost `elif` handles the case where `tag` was already found on the first lookup (pre-normalization) and has an existing fandom — preserved from the original code. + +- [ ] **Step 5: Normalize inside `set_tag_fandom`** + +Find (around `app/main.py:913-946`) the block that reassembles the character name: + +```python + # Get the character's base name (strip existing fandom suffix) + char_part = tag.name.split(":", 1)[1] if ":" in tag.name else tag.name + char_base, _ = _parse_character_fandom(char_part) + + if fandom_id: + fandom_tag = Tag.query.get(fandom_id) + if not fandom_tag or fandom_tag.kind != "fandom": + return jsonify(ok=False, error="Invalid fandom tag"), 400 + fandom_display = fandom_tag.name.split(":", 1)[1] if ":" in fandom_tag.name else fandom_tag.name + elif fandom_name: + fandom_tag = _ensure_fandom_tag(fandom_name) + fandom_display = fandom_name + else: + # Clear fandom + tag.fandom_id = None + tag.name = f"character:{char_base}" + db.session.commit() + return jsonify(ok=True, tag={ + "id": tag.id, + "name": tag.name, + "kind": tag.kind, + "fandom_id": None, + "fandom_name": None + }) + + # Check for name conflict before renaming + new_name = f"character:{char_base} ({fandom_display})" + conflict = Tag.query.filter(Tag.name == new_name, Tag.id != tag_id).first() + if conflict: + return jsonify(ok=False, error=f"A tag named '{new_name}' already exists"), 409 + + tag.fandom_id = fandom_tag.id + tag.name = new_name + db.session.commit() +``` + +Replace with: + +```python + # Get the character's base name (strip existing fandom suffix) and normalize it. + char_part = tag.name.split(":", 1)[1] if ":" in tag.name else tag.name + char_base, _ = _parse_character_fandom(char_part) + char_base = normalize_display_name(char_base) + + if fandom_id: + fandom_tag = Tag.query.get(fandom_id) + if not fandom_tag or fandom_tag.kind != "fandom": + return jsonify(ok=False, error="Invalid fandom tag"), 400 + fandom_display = fandom_tag.name.split(":", 1)[1] if ":" in fandom_tag.name else fandom_tag.name + elif fandom_name: + # _ensure_fandom_tag normalizes internally. + fandom_tag = _ensure_fandom_tag(fandom_name) + fandom_display = fandom_tag.name.split(":", 1)[1] + else: + # Clear fandom + tag.fandom_id = None + tag.name = f"character:{char_base}" + db.session.commit() + return jsonify(ok=True, tag={ + "id": tag.id, + "name": tag.name, + "kind": tag.kind, + "fandom_id": None, + "fandom_name": None + }) + + # Check for name conflict before renaming + new_name = f"character:{char_base} ({fandom_display})" + conflict = Tag.query.filter(Tag.name == new_name, Tag.id != tag_id).first() + if conflict: + return jsonify(ok=False, error=f"A tag named '{new_name}' already exists"), 409 + + tag.fandom_id = fandom_tag.id + tag.name = new_name + db.session.commit() +``` + +*Backfill is added in Task 3.* Task 2 only handles normalization. + +- [ ] **Step 6: Normalize inside `accept_image_suggestion`** + +Find the auto-create block (around `app/main.py:726-732`): + +```python + tag = Tag.query.filter_by(name=tag_name).first() + if tag is None: + # Auto-create in all categories. General maps to kind=None. + kind = _WD14_CATEGORY_TO_KIND.get(category) + tag = Tag(name=tag_name, kind=kind) + db.session.add(tag) + db.session.flush() # assign tag.id +``` + +Replace with: + +```python + tag = Tag.query.filter_by(name=tag_name).first() + if tag is None: + # Auto-create in all categories. General maps to kind=None. + kind = _WD14_CATEGORY_TO_KIND.get(category) + # Normalize character/fandom display names so WD14 auto-accepts stay consistent. + if kind in ('character', 'fandom') and ':' in tag_name: + prefix, rest = tag_name.split(':', 1) + tag_name = f"{prefix}:{normalize_display_name(rest)}" + # The normalized name may already exist — reuse if so. + tag = Tag.query.filter_by(name=tag_name).first() + if tag is None: + tag = Tag(name=tag_name, kind=kind) + db.session.add(tag) + db.session.flush() # assign tag.id +``` + +- [ ] **Step 7: Verify imports and sanity-check** + +```bash +docker compose exec -T web python -c " +from app import create_app +app = create_app() +print('import ok') +" +``` + +Expected: `import ok`. Any traceback means a typo — fix and rerun. + +- [ ] **Step 8: Commit** + +```bash +git add app/main.py +git commit -m "feat(tags): normalize character/fandom display names on every write path" +``` + +--- + +## Task 3: Retroactive fandom backfill on `set_tag_fandom` + +**Files:** +- Modify: `app/main.py` + +- [ ] **Step 1: Add the backfill step after the rename commit** + +Find the end of `set_tag_fandom` (the block just after the main `db.session.commit()` that finalizes the rename, around the last `return jsonify(ok=True, tag={...})` in the function body, at or near `app/main.py:948-954`): + +```python + tag.fandom_id = fandom_tag.id + tag.name = new_name + db.session.commit() + + return jsonify(ok=True, tag={ + "id": tag.id, + "name": tag.name, + "kind": tag.kind, + "fandom_id": tag.fandom_id, + "fandom_name": fandom_display + }) +``` + +Replace with: + +```python + tag.fandom_id = fandom_tag.id + tag.name = new_name + db.session.commit() + + # Retroactive backfill: every image already tagged with this character should + # also have the fandom tag attached. Runs in a second transaction so a failed + # backfill doesn't roll back the rename. + try: + db.session.execute( + db.text(""" + INSERT INTO image_tags (image_id, tag_id) + SELECT it.image_id, :fandom_id + FROM image_tags it + WHERE it.tag_id = :char_id + AND NOT EXISTS ( + SELECT 1 FROM image_tags it2 + WHERE it2.image_id = it.image_id + AND it2.tag_id = :fandom_id + ) + """), + {'char_id': tag.id, 'fandom_id': fandom_tag.id}, + ) + db.session.commit() + except Exception as e: + db.session.rollback() + # Rename already committed; log and continue. Sweep button recovers. + current_app.logger.warning( + "set_tag_fandom backfill failed for tag %s -> fandom %s: %s", + tag.id, fandom_tag.id, e, + ) + + return jsonify(ok=True, tag={ + "id": tag.id, + "name": tag.name, + "kind": tag.kind, + "fandom_id": tag.fandom_id, + "fandom_name": fandom_display + }) +``` + +- [ ] **Step 2: Verify `current_app` is importable from `app/main.py`** + +Run: + +```bash +grep -n "current_app\|from flask import" app/main.py | head -5 +``` + +If `current_app` is not already in the `from flask import (...)` tuple near the top of the file, add it. Typical existing line looks like `from flask import Blueprint, render_template, request, jsonify, abort, redirect, url_for, send_file`; append `current_app`. + +- [ ] **Step 3: Sanity-check the app still boots** + +```bash +docker compose exec -T web python -c "from app import create_app; create_app(); print('ok')" +``` + +Expected: `ok`. + +- [ ] **Step 4: Commit** + +```bash +git add app/main.py +git commit -m "feat(tags): backfill fandom to existing character-tagged images on set-fandom" +``` + +--- + +## Task 4: Create `sync_character_fandoms` Celery maintenance task + +**Files:** +- Create: `app/tasks/maintenance.py` +- Modify: `app/celery_app.py` + +- [ ] **Step 1: Create the maintenance tasks module** + +Write to `app/tasks/maintenance.py`: + +```python +"""Maintenance-queue Celery tasks. Lightweight, user-triggered, non-ML.""" +import logging +from celery import shared_task + +from app import db +from app.models import Tag + +log = logging.getLogger(__name__) + + +@shared_task( + name='app.tasks.maintenance.sync_character_fandoms', + soft_time_limit=120, + time_limit=180, +) +def sync_character_fandoms(): + """For every character tag with a non-null fandom_id, attach the fandom + tag to every image tagged with the character but not the fandom. + + Additive only — never removes fandom tags. + + Returns a summary dict with counts. + """ + chars = ( + Tag.query + .filter_by(kind='character') + .filter(Tag.fandom_id.isnot(None)) + .all() + ) + total_added = 0 + failed = 0 + for char in chars: + try: + result = db.session.execute( + db.text(""" + INSERT INTO image_tags (image_id, tag_id) + SELECT it.image_id, :fandom_id + FROM image_tags it + WHERE it.tag_id = :char_id + AND NOT EXISTS ( + SELECT 1 FROM image_tags it2 + WHERE it2.image_id = it.image_id + AND it2.tag_id = :fandom_id + ) + """), + {'char_id': char.id, 'fandom_id': char.fandom_id}, + ) + total_added += result.rowcount or 0 + db.session.commit() + except Exception as e: + db.session.rollback() + failed += 1 + log.warning( + "sync_character_fandoms: char tag %s (%s) failed: %s", + char.id, char.name, e, + ) + log.info( + "sync_character_fandoms: scanned %d characters, added %d image↔fandom links, %d failed", + len(chars), total_added, failed, + ) + return { + 'characters_scanned': len(chars), + 'links_added': total_added, + 'characters_failed': failed, + } +``` + +- [ ] **Step 2: Register the module in the Celery include list** + +Find in `app/celery_app.py` (around lines 36-43): + +```python + include=[ + 'app.tasks.scan', + 'app.tasks.import_file', + 'app.tasks.thumbnail', + 'app.tasks.sidecar', + 'app.tasks.ml', + ] +``` + +Replace with: + +```python + include=[ + 'app.tasks.scan', + 'app.tasks.import_file', + 'app.tasks.thumbnail', + 'app.tasks.sidecar', + 'app.tasks.ml', + 'app.tasks.maintenance', + ] +``` + +- [ ] **Step 3: Route the new task to the `maintenance` queue** + +Find in `app/celery_app.py` (around lines 63-81) the `task_routes` dict. The `maintenance` queue already exists. Append one entry inside the existing `'app.tasks.scan.*'` maintenance entries block, keeping the file's existing style: + +Find: + +```python + # Maintenance tasks - handled by scheduler (periodic/lightweight) + 'app.tasks.scan.recover_interrupted_tasks': {'queue': 'maintenance'}, + 'app.tasks.scan.cleanup_old_tasks': {'queue': 'maintenance'}, + 'app.tasks.scan.update_system_stats': {'queue': 'maintenance'}, + 'app.tasks.scan.update_batch_stats': {'queue': 'maintenance'}, +``` + +Replace with: + +```python + # Maintenance tasks - handled by scheduler (periodic/lightweight) + 'app.tasks.scan.recover_interrupted_tasks': {'queue': 'maintenance'}, + 'app.tasks.scan.cleanup_old_tasks': {'queue': 'maintenance'}, + 'app.tasks.scan.update_system_stats': {'queue': 'maintenance'}, + 'app.tasks.scan.update_batch_stats': {'queue': 'maintenance'}, + 'app.tasks.maintenance.sync_character_fandoms': {'queue': 'maintenance'}, +``` + +- [ ] **Step 4: Sanity-check import and registration** + +```bash +docker compose exec -T web python -c " +from app.tasks.maintenance import sync_character_fandoms +print('task name:', sync_character_fandoms.name) +" +``` + +Expected: `task name: app.tasks.maintenance.sync_character_fandoms`. + +- [ ] **Step 5: Commit** + +```bash +git add app/tasks/maintenance.py app/celery_app.py +git commit -m "feat(tasks): add sync_character_fandoms maintenance task" +``` + +--- + +## Task 5: Add sweep route + Settings maintenance button + +**Files:** +- Modify: `app/main.py` +- Modify: `app/templates/settings.html` + +- [ ] **Step 1: Add the trigger route** + +Find the existing `trigger_recompute_all_centroids` route (added in the prior feature; it sits right after `trigger_ml_backfill`). Immediately after it, add: + +```python +@main.post('/settings/maintenance/sync-character-fandoms') +def trigger_sync_character_fandoms(): + from app.tasks.maintenance import sync_character_fandoms + sync_character_fandoms.apply_async() + return redirect(url_for('main.settings', tab='maintenance')) +``` + +Note: no `queue='...'` arg; the task routes itself to `maintenance` via `celery_app.py`. + +- [ ] **Step 2: Add the Settings → Maintenance button** + +Find the existing `
`, add: + +```html + +

+ Attach each character's fandom to every image already tagged with that character. + Additive only — existing fandom tags are never removed. Run after you've set a + fandom on an existing character, or whenever you suspect drift. +

+ + +
+``` + +- [ ] **Step 3: Verify the route is registered** + +```bash +docker compose exec -T web python -c " +from app import create_app +app = create_app() +print([r for r in app.url_map.iter_rules() if 'sync-character-fandoms' in r.rule]) +" +``` + +Expected: one `Rule` entry for `/settings/maintenance/sync-character-fandoms`. + +- [ ] **Step 4: Commit** + +```bash +git add app/main.py app/templates/settings.html +git commit -m "feat(settings): Sync character fandoms maintenance button + route" +``` + +--- + +## Task 6: One-time migration — rename and merge + +**Files:** +- Create: `migrations/versions/h26041901_normalize_character_fandom_tag_names.py` + +- [ ] **Step 1: Write the migration file** + +Write to `migrations/versions/h26041901_normalize_character_fandom_tag_names.py`: + +```python +"""Normalize character and fandom tag display names; merge case-only duplicates. + +Revision ID: h26041901 +Revises: g26041901 +Create Date: 2026-04-19 + +Upgrade: title-cases the display-name portion of every character:* and fandom:* +tag. When two rows of the same kind collapse to the same name, merges image_tags, +tag.fandom_id references, and tag_reference_embedding rows into the already- +correctly-cased target ("winner") and deletes the "loser" tag row. + +Downgrade: raises. Merges are destructive; reverting would require restoring +from a pre-migration backup. +""" +from alembic import op + + +revision = 'h26041901' +down_revision = 'g26041901' +branch_labels = None +depends_on = None + + +def _normalize_display(s: str) -> str: + """Mirror of app.utils.tag_names.normalize_display_name. Inlined here so + the migration doesn't import app code (Alembic runs without the Flask app).""" + if not s: + return s + out = [] + for p in s.split(' '): + if p == '': + out.append(p) + else: + out.append(p[:1].upper() + p[1:]) + return ' '.join(out) + + +def _compute_new_name(old_name: str, kind: str) -> str: + """Given a full 'prefix:display' name, return the normalized full name.""" + if ':' not in old_name: + return old_name + prefix, rest = old_name.split(':', 1) + if kind == 'character': + # Split off "(Fandom)" suffix if present, normalize each part. + if '(' in rest and rest.rstrip().endswith(')'): + paren_idx = rest.rfind('(') + base = rest[:paren_idx].rstrip() + fandom_part = rest[paren_idx+1:-1] # strip outer parens + base = _normalize_display(base) + fandom_part = _normalize_display(fandom_part) + return f"{prefix}:{base} ({fandom_part})" + return f"{prefix}:{_normalize_display(rest)}" + elif kind == 'fandom': + return f"{prefix}:{_normalize_display(rest)}" + return old_name + + +def upgrade(): + conn = op.get_bind() + # Work on a snapshot; we mutate the tag table as we iterate. + rows = conn.execute( + op.get_context().bind.dialect.paramstyle and + __import__('sqlalchemy').text( + "SELECT id, name, kind FROM tag WHERE kind IN ('character', 'fandom') ORDER BY id" + ) + ).fetchall() + + renames = 0 + merges = 0 + for row in rows: + tag_id = row[0] + old_name = row[1] + kind = row[2] + new_name = _compute_new_name(old_name, kind) + if new_name == old_name: + continue + + # Re-check that this tag still exists (may have been deleted as a loser in + # an earlier merge within this loop). + still_exists = conn.execute( + __import__('sqlalchemy').text("SELECT id FROM tag WHERE id = :id"), + {'id': tag_id}, + ).fetchone() + if not still_exists: + continue + + # Look for a collision: same target name + same kind + different id. + target = conn.execute( + __import__('sqlalchemy').text( + "SELECT id, name FROM tag WHERE name = :new_name AND kind = :kind AND id != :tag_id" + ), + {'new_name': new_name, 'kind': kind, 'tag_id': tag_id}, + ).fetchone() + + if target is None: + # No collision — plain rename. + conn.execute( + __import__('sqlalchemy').text("UPDATE tag SET name = :new_name WHERE id = :tag_id"), + {'new_name': new_name, 'tag_id': tag_id}, + ) + renames += 1 + else: + # Merge: winner = target (already correctly cased), loser = current row. + winner_id = target[0] + winner_name = target[1] # already == new_name + loser_id = tag_id + loser_name = old_name + + # 1. Reassign image_tags, dedup into winner. + conn.execute( + __import__('sqlalchemy').text(""" + INSERT INTO image_tags (image_id, tag_id) + SELECT DISTINCT image_id, :winner_id + FROM image_tags + WHERE tag_id = :loser_id + ON CONFLICT DO NOTHING + """), + {'winner_id': winner_id, 'loser_id': loser_id}, + ) + conn.execute( + __import__('sqlalchemy').text("DELETE FROM image_tags WHERE tag_id = :loser_id"), + {'loser_id': loser_id}, + ) + + # 2. Reassign any fandom_id FKs pointing at the loser. + conn.execute( + __import__('sqlalchemy').text( + "UPDATE tag SET fandom_id = :winner_id WHERE fandom_id = :loser_id" + ), + {'winner_id': winner_id, 'loser_id': loser_id}, + ) + + # 3. tag_reference_embedding: the table is keyed by (tag_name, model_version). + # For each model_version where only the loser has a row, rename; where + # both have rows, keep the winner's and delete the loser's. + conn.execute( + __import__('sqlalchemy').text(""" + UPDATE tag_reference_embedding + SET tag_name = :winner_name + WHERE tag_name = :loser_name + AND NOT EXISTS ( + SELECT 1 FROM tag_reference_embedding w + WHERE w.tag_name = :winner_name + AND w.model_version = tag_reference_embedding.model_version + ) + """), + {'winner_name': winner_name, 'loser_name': loser_name}, + ) + conn.execute( + __import__('sqlalchemy').text( + "DELETE FROM tag_reference_embedding WHERE tag_name = :loser_name" + ), + {'loser_name': loser_name}, + ) + + # 4. Delete the loser tag row. + conn.execute( + __import__('sqlalchemy').text("DELETE FROM tag WHERE id = :loser_id"), + {'loser_id': loser_id}, + ) + merges += 1 + + print(f"normalize_character_fandom_tag_names: {renames} renames, {merges} merges") + + +def downgrade(): + raise NotImplementedError( + "h26041901 normalizes tag names and merges case-only duplicates. " + "Merges are destructive and cannot be reversed automatically. " + "Restore from a pre-migration backup if you need to roll back." + ) +``` + +*Note the repeated `__import__('sqlalchemy').text(...)` — this avoids adding a top-level `from sqlalchemy import text` that would clutter the diff. If you prefer, you can add `from sqlalchemy import text` at the top and use `text(...)` directly — the effect is identical.* + +- [ ] **Step 2: Run the migration** + +```bash +docker compose exec -T web flask db upgrade +``` + +Expected output includes a line like `normalize_character_fandom_tag_names: N renames, M merges`. Also `INFO [alembic.runtime.migration] Running upgrade g26041901 -> h26041901`. + +- [ ] **Step 3: Verify dev DB state** + +```bash +docker compose exec -T postgres psql -U imagerepo -d imagerepo -c "SELECT name, kind FROM tag WHERE kind IN ('character','fandom') ORDER BY kind, name" +``` + +Expected: every displayed name starts each whitespace-separated word with an upper-case letter. In the current dev DB, `character:emelie` should now be `character:Emelie`. + +- [ ] **Step 4: Verify no orphan references** + +```bash +docker compose exec -T postgres psql -U imagerepo -d imagerepo -c " +SELECT 'orphan image_tags' AS what, COUNT(*) FROM image_tags it LEFT JOIN tag t ON t.id = it.tag_id WHERE t.id IS NULL +UNION ALL +SELECT 'orphan fandom_id refs', COUNT(*) FROM tag WHERE fandom_id IS NOT NULL AND fandom_id NOT IN (SELECT id FROM tag) +" +``` + +Expected: both counts `0`. + +- [ ] **Step 5: Commit** + +```bash +git add migrations/versions/h26041901_normalize_character_fandom_tag_names.py +git commit -m "feat(migration): normalize character/fandom names, merge case-only duplicates" +``` + +--- + +## Task 7: Rebuild + smoke tests + +**Files:** +- None modified unless smoke tests reveal bugs. + +- [ ] **Step 1: Rebuild web + scheduler containers** + +The scheduler container holds the `maintenance` queue worker, so it must be rebuilt along with web. + +```bash +docker compose up -d --build web scheduler +``` + +Expected: both containers rebuild and start cleanly. + +- [ ] **Step 2: Verify scheduler registered the new task** + +```bash +docker compose logs scheduler --tail 80 2>&1 | grep "sync_character_fandoms" +``` + +Expected: one line ` . app.tasks.maintenance.sync_character_fandoms` in the task listing. + +- [ ] **Step 3: Verify the web container is up** + +```bash +docker compose logs web --tail 10 2>&1 | grep -iE "listening|ready|error|traceback" | head -5 +``` + +Expected: a `Listening at: http://0.0.0.0:5000` line with no errors. + +- [ ] **Step 4: Smoke-test — create a character with a lowercase fandom** + +In the browser (or via curl): + +```bash +# Pick any valid image_id from your dev DB. +docker compose exec -T postgres psql -U imagerepo -d imagerepo -tAc "SELECT id FROM image_record LIMIT 1" +# Use that id below. +curl -s -X POST "http://localhost:5000/image//tags/add" \ + -d "name=character:pikachu (pokemon)" +``` + +Then verify: + +```bash +docker compose exec -T postgres psql -U imagerepo -d imagerepo -c " +SELECT name, kind, fandom_id FROM tag WHERE name ILIKE 'character:pikachu%' OR name ILIKE 'fandom:pokemon' +" +``` + +Expected: two rows — `character:Pikachu (Pokemon)` (with `fandom_id` set) and `fandom:Pokemon` (kind='fandom'). Both capitalized despite lowercase input. + +- [ ] **Step 5: Smoke-test — retroactive backfill on `set_tag_fandom`** + +Create a character tag with no fandom, attach to 2 images, then set its fandom and verify both images get the fandom: + +```bash +# Pick two image ids. +docker compose exec -T postgres psql -U imagerepo -d imagerepo -tAc "SELECT id FROM image_record LIMIT 2" +# Call them IMG1 and IMG2. + +# Add the character to both (no fandom yet). +curl -s -X POST "http://localhost:5000/image//tags/add" -d "name=character:TestKid" +curl -s -X POST "http://localhost:5000/image//tags/add" -d "name=character:TestKid" + +# Get the character tag id. +docker compose exec -T postgres psql -U imagerepo -d imagerepo -tAc "SELECT id FROM tag WHERE name = 'character:TestKid'" +# Call it CHAR_ID. + +# Set a fandom on it. +curl -s -X POST "http://localhost:5000/api/tag//set-fandom" -d "fandom_name=TestShow" + +# Verify both images now have fandom:TestShow. +docker compose exec -T postgres psql -U imagerepo -d imagerepo -c " +SELECT i.id, t.name FROM image_record i +JOIN image_tags it ON it.image_id = i.id +JOIN tag t ON t.id = it.tag_id +WHERE i.id IN (, ) AND t.kind = 'fandom' +" +``` + +Expected: two rows, both with `fandom:TestShow`. + +- [ ] **Step 6: Smoke-test — sweep button is idempotent** + +Click "Sync character fandoms to images" from Settings → Maintenance (or POST to the URL directly). Watch the scheduler logs: + +```bash +docker compose logs -f scheduler 2>&1 | grep sync_character_fandoms +``` + +Expected: `sync_character_fandoms: scanned N characters, added 0 image↔fandom links, 0 failed` (or similar — zero links added because Step 5 already synced them). + +- [ ] **Step 7: Smoke-test — sweep fixes manual drift** + +Remove the fandom from one of the images manually: + +```bash +docker compose exec -T postgres psql -U imagerepo -d imagerepo -c " +DELETE FROM image_tags WHERE image_id = AND tag_id = (SELECT id FROM tag WHERE name = 'fandom:TestShow') +" +``` + +Click the sweep button again. Verify: + +```bash +docker compose exec -T postgres psql -U imagerepo -d imagerepo -c " +SELECT 1 FROM image_tags it JOIN tag t ON t.id = it.tag_id +WHERE it.image_id = AND t.name = 'fandom:TestShow' +" +``` + +Expected: one row (link restored). Scheduler log shows `links_added: 1`. + +- [ ] **Step 8: Clean up test data (optional)** + +```bash +docker compose exec -T postgres psql -U imagerepo -d imagerepo -c " +DELETE FROM image_tags WHERE tag_id IN (SELECT id FROM tag WHERE name IN ('character:TestKid (TestShow)', 'character:TestKid', 'fandom:TestShow', 'character:Pikachu (Pokemon)', 'fandom:Pokemon')); +DELETE FROM tag WHERE name IN ('character:TestKid (TestShow)', 'character:TestKid', 'fandom:TestShow', 'character:Pikachu (Pokemon)', 'fandom:Pokemon'); +" +``` + +- [ ] **Step 9: Commit any fix-ups from smoke tests** + +If the smoke tests uncovered bugs, fix them in the relevant task's files and commit separately. If everything passed, there's nothing to commit. + +--- + +## Self-Review Checklist + +- Spec coverage: + - [x] `normalize_display_name` helper → Task 1 + - [x] Helper applied to `_ensure_fandom_tag`, `add_tag`, `add_bulk_tag`, `set_tag_fandom`, `accept_image_suggestion` → Task 2 + - [x] Retroactive backfill on `set_tag_fandom` (second transaction, additive) → Task 3 + - [x] `sync_character_fandoms` Celery task on `maintenance` queue → Task 4 + - [x] Sweep route + Settings button → Task 5 + - [x] One-time migration with rename + merge (image_tags, fandom_id, tag_reference_embedding) → Task 6 + - [x] `downgrade()` raises `NotImplementedError` → Task 6 + - [x] Manual smoke tests → Task 7 + +- Type/name consistency: helper is `normalize_display_name` everywhere; task name `app.tasks.maintenance.sync_character_fandoms` matches between module, routing entry, trigger endpoint, and scheduler log. + +- Placeholders: none. ``, ``, ``, `` in Task 7 smoke tests are per-environment substitution values the operator fills in from the preceding query, not unspecified content. + +--- + +## Notes for the implementer + +- **Alembic migration style:** the existing migrations in this repo use `sqlalchemy.text(...)` for raw SQL. You may prefer adding `from sqlalchemy import text` at the top of the migration and using `text(...)` directly instead of the `__import__('sqlalchemy').text(...)` idiom shown above. Either works. +- **Frontend cache-busting:** hard-refresh the browser after the rebuild so the maintenance page picks up the new button. +- **Tests:** the project has no automated test suite; validation is Task 7's manual smoke-test script. diff --git a/docs/superpowers/plans/2026-04-19-tag-underscores-and-modal-polish.md b/docs/superpowers/plans/2026-04-19-tag-underscores-and-modal-polish.md new file mode 100644 index 0000000..4c17568 --- /dev/null +++ b/docs/superpowers/plans/2026-04-19-tag-underscores-and-modal-polish.md @@ -0,0 +1,1007 @@ +# Tag Underscores + Modal Polish Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Translate WD14 underscored names (`big_hair`) to spaces at display, persist, and migration layers; make suggestion-accept refresh the tag list the same way manual add does; tint accept/reject chip buttons green/red persistently. + +**Architecture:** One helper (`underscores_to_spaces`) added to `app/utils/tag_names.py`; `normalize_display_name` calls it internally so character/fandom paths collapse `misty_ketchum` → `Misty Ketchum`. WD14 names are canonicalized in two places: `app/services/tag_suggestions.py` rewrites them at emit so the modal sees `big hair`, and `app/main.py:accept_image_suggestion` re-applies the same transform before DB write. General (null-kind) gets underscore→space only; case preserved. One-time Alembic migration `i26041901` does the same rename-or-merge sweep as `h26041901`, extended to cover null-kind rows. Modal JS awaits `loadTags` before the chip animates out, mirroring manual-add timing. CSS base rule tints ✓ green / ✕ red; hover intensifies. + +**Tech Stack:** Python 3.12, Flask, SQLAlchemy, Alembic (via Flask-Migrate), Postgres 16, pgvector, vanilla JS, CSS. No JS build step. + +**Testing philosophy:** This codebase has no automated test infrastructure — the prior `h26041901` feature used manual dev-DB verification. This plan follows the same approach: each task ends with a concrete manual check against the running dev stack, then a commit. No pytest/jest scaffolding. + +--- + +## File Structure + +**Modify:** +- `app/utils/tag_names.py` — add `underscores_to_spaces`, update `normalize_display_name` to call it. +- `app/services/tag_suggestions.py` — move `_WD14_CATEGORY_TO_KIND` here, add `_canonicalize_wd14_name`, apply to `_wd14_suggestions` output + `already` set. +- `app/main.py` — drop local `_WD14_CATEGORY_TO_KIND`, import from service; rewrite `accept_image_suggestion` name-canonicalization block; add underscore→space in `add_tag` and `bulk_add_tag` user/no-prefix paths. +- `app/static/js/view-modal.js` — rewrite `acceptSuggestion` for refresh parity; simplify `animateChipOut` signature. +- `app/static/style.css` — add persistent color rules for `.sugg-accept` and `.sugg-reject`; hover intensifies. + +**Create:** +- `migrations/versions/i26041901_normalize_underscores_and_general_tags.py` — one-time rename-or-merge across character/fandom/null kinds. + +--- + +## Task 1: Extend normalization helpers + +**Files:** +- Modify: `app/utils/tag_names.py` + +- [ ] **Step 1: Update the module** + +Replace the entire contents of `app/utils/tag_names.py` with: + +```python +"""Normalization helpers for Tag display names. + +Keeps the first letter of every whitespace-separated word upper-case and +preserves internal punctuation. Called from every character/fandom write +path so typos like "misty" don't accumulate alongside "Misty". +""" + + +def underscores_to_spaces(s: str) -> str: + """Replace every underscore with a space. + + Used by general/null-kind write paths and as a pre-step inside + normalize_display_name so WD14's 'misty_ketchum' and a hand-typed + 'Misty Ketchum' collapse onto the same tag name. + + Examples: + "big_hair" -> "big hair" + "long_blue_hair" -> "long blue hair" + "" -> "" + """ + return s.replace('_', ' ') if s else s + + +def normalize_display_name(s: str) -> str: + """Title-case each whitespace-separated word. Preserves internal punctuation. + + Underscores are converted to spaces first so WD14-style names normalize + identically to hand-typed names. + + Examples: + "misty" -> "Misty" + "misty ketchum" -> "Misty Ketchum" + "misty_ketchum" -> "Misty Ketchum" + "o'brien" -> "O'brien" + "mary-kate" -> "Mary-kate" + "" -> "" + " " -> " " (only whitespace — returned unchanged) + """ + if not s: + return s + s = underscores_to_spaces(s) + parts = s.split(' ') + out: list[str] = [] + for p in parts: + if p == '': + out.append(p) + else: + out.append(p[:1].upper() + p[1:]) + return ' '.join(out) +``` + +- [ ] **Step 2: Manually verify in a Python shell** + +Run: +```bash +cd /home/bvandeusen/Nextcloud/Projects/ImageRepo +python3 -c " +from app.utils.tag_names import underscores_to_spaces, normalize_display_name +assert underscores_to_spaces('big_hair') == 'big hair' +assert underscores_to_spaces('') == '' +assert normalize_display_name('misty_ketchum') == 'Misty Ketchum' +assert normalize_display_name('misty') == 'Misty' +assert normalize_display_name('o\\'brien') == 'O\\'brien' +assert normalize_display_name('') == '' +print('OK') +" +``` +Expected: `OK` printed. + +- [ ] **Step 3: Commit** + +```bash +git add app/utils/tag_names.py +git commit -m "feat(tags): add underscores_to_spaces helper, fold into normalize_display_name" +``` + +--- + +## Task 2: Move `_WD14_CATEGORY_TO_KIND` into the service module + +**Files:** +- Modify: `app/services/tag_suggestions.py` (add constant near top) +- Modify: `app/main.py:710-713` (remove local copy, import from service) + +- [ ] **Step 1: Add the constant to `app/services/tag_suggestions.py`** + +In `app/services/tag_suggestions.py`, insert this block immediately after `ELIGIBLE_CENTROID_KINDS = ('character', 'fandom', None)` (around line 28): + +```python +# Maps the WD14 prediction category ('character', 'copyright') to the kind used +# in the Tag table. 'general' maps to None and is handled separately at accept +# time. Lives here (not in main) so the canonicalization helper below can use it +# without a circular import. +_WD14_CATEGORY_TO_KIND = { + 'character': 'character', + 'copyright': 'fandom', +} +``` + +- [ ] **Step 2: Remove the old constant from `app/main.py:710-713`** + +Delete these four lines from `app/main.py`: + +```python +_WD14_CATEGORY_TO_KIND = { + 'character': 'character', + 'copyright': 'fandom', # ImageRepo uses 'fandom' for IP/franchise +} +``` + +- [ ] **Step 3: Update the one reference in `accept_image_suggestion`** + +Find the line in `app/main.py` (around line 741): + +```python + kind = _WD14_CATEGORY_TO_KIND.get(category) +``` + +Change it to import from the service: + +```python + from app.services.tag_suggestions import _WD14_CATEGORY_TO_KIND + kind = _WD14_CATEGORY_TO_KIND.get(category) +``` + +(The inline import matches the existing pattern in this function — surrounding code already does `from app.services.tag_suggestions import ELIGIBLE_CENTROID_KINDS` inline.) + +- [ ] **Step 4: Verify the app still starts** + +Run: +```bash +cd /home/bvandeusen/Nextcloud/Projects/ImageRepo +docker compose up -d web +sleep 3 +docker compose logs --tail=20 web +``` +Expected: no ImportError; `web` reports "Running on …" or similar ready line. + +- [ ] **Step 5: Commit** + +```bash +git add app/services/tag_suggestions.py app/main.py +git commit -m "refactor(tags): move _WD14_CATEGORY_TO_KIND into tag_suggestions service" +``` + +--- + +## Task 3: WD14 name canonicalization at emit + +**Files:** +- Modify: `app/services/tag_suggestions.py` (new helper + integrate into `_wd14_suggestions` + `_existing_tag_names_for_image`) + +- [ ] **Step 1: Add the helper** + +In `app/services/tag_suggestions.py`, insert this function immediately after the `_WD14_CATEGORY_TO_KIND` constant: + +```python +def _canonicalize_wd14_name(raw: str, category: str) -> str: + """Rewrite a raw WD14 tag name to the canonical form ImageRepo persists. + + - 'character' / 'copyright': title-case the display portion. + - everything else ('general', 'meta', unknown): underscore-to-space only, + preserving the user's casing for general topics. + """ + from app.utils.tag_names import normalize_display_name, underscores_to_spaces + kind = _WD14_CATEGORY_TO_KIND.get(category) + if ':' in raw: + prefix, rest = raw.split(':', 1) + transform = normalize_display_name if kind in ('character', 'fandom') else underscores_to_spaces + return f"{prefix}:{transform(rest)}" + return normalize_display_name(raw) if kind in ('character', 'fandom') else underscores_to_spaces(raw) +``` + +- [ ] **Step 2: Canonicalize WD14 output names** + +In the same file, replace the `_wd14_suggestions` function (lines 77-97) with: + +```python +def _wd14_suggestions(image_id: int, cfg: dict, already: set[str]) -> list[dict]: + wd14_ver = cfg.get('wd14_model_version', _DEFAULTS['wd14_model_version']) + preds = ( + ImageTagPrediction.query + .filter_by(image_id=image_id, model_version=wd14_ver) + .all() + ) + out: list[dict] = [] + for p in preds: + threshold = _category_threshold(p.tag_category, cfg) + if p.confidence < threshold: + continue + canonical = _canonicalize_wd14_name(p.tag_name, p.tag_category) + if canonical in already: + continue + out.append({ + 'name': canonical, + 'category': p.tag_category, + 'confidence': p.confidence, + 'source': 'wd14', + }) + return out +``` + +Two changes: (1) name is rewritten before the `already` check and before the dict is built; (2) the `already` set is assumed to already contain canonical names — see next step. + +- [ ] **Step 3: Canonicalize the "already attached" set** + +In the same file, replace `_existing_tag_names_for_image` (lines 63-70) with: + +```python +def _existing_tag_names_for_image(image_id: int) -> set[str]: + """Names of tags already attached to this image. + + Names are returned as-stored; since every write path canonicalizes, any + row in the DB is already in canonical form. WD14 output is canonicalized + before lookup, so equality works. + """ + rows = ( + db.session.query(Tag.name) + .join(image_tags, image_tags.c.tag_id == Tag.id) + .filter(image_tags.c.image_id == image_id) + .all() + ) + return {row[0] for row in rows} +``` + +No behavioral change yet — the comment documents the invariant that Task 4 and Task 7 (migration) establish. Once the migration runs, all existing rows have canonical names and this function needs no runtime transform. + +- [ ] **Step 4: Manually verify at the running service** + +Pick one image that has WD14 predictions with underscored names. Run: + +```bash +cd /home/bvandeusen/Nextcloud/Projects/ImageRepo +docker compose exec db psql -U postgres imagerepo -c \ + "SELECT image_id, tag_name FROM image_tag_prediction WHERE tag_name LIKE '%\\_%' ESCAPE '\\' LIMIT 3" +``` +Pick an `image_id` from the output. Then: + +```bash +curl -s http://localhost:5000/image//suggestions | python3 -m json.tool | grep -E '"name"' | head -20 +``` + +Expected: no `_` characters in any `"name"` field. Names with spaces only. + +- [ ] **Step 5: Commit** + +```bash +git add app/services/tag_suggestions.py +git commit -m "feat(suggestions): canonicalize WD14 names (underscore→space) at emit" +``` + +--- + +## Task 4: Accept-endpoint canonicalization + +**Files:** +- Modify: `app/main.py:723-796` (`accept_image_suggestion`) + +- [ ] **Step 1: Rewrite the name-canonicalization block** + +In `app/main.py`, locate `accept_image_suggestion` (around line 723). The current body (lines 727-752) reads: + +```python + payload = request.get_json(force=True, silent=True) or {} + tag_name = (payload.get('tag_name') or '').strip() + source = payload.get('source') or '' + category = payload.get('category') or '' + confidence = float(payload.get('confidence') or 0.0) + if not tag_name or not source or not category: + return jsonify({'ok': False, 'error': 'missing_fields'}), 400 + + image = ImageRecord.query.get(image_id) + if image is None: + return jsonify({'ok': False, 'error': 'image_not_found'}), 404 + + tag = Tag.query.filter_by(name=tag_name).first() + if tag is None: + # Auto-create in all categories. General maps to kind=None. + kind = _WD14_CATEGORY_TO_KIND.get(category) + # Normalize character/fandom display names so WD14 auto-accepts stay consistent. + if kind in ('character', 'fandom') and ':' in tag_name: + prefix, rest = tag_name.split(':', 1) + tag_name = f"{prefix}:{normalize_display_name(rest)}" + # The normalized name may already exist — reuse if so. + tag = Tag.query.filter_by(name=tag_name).first() + if tag is None: + tag = Tag(name=tag_name, kind=kind) + db.session.add(tag) + db.session.flush() +``` + +Replace those lines with: + +```python + from app.services.tag_suggestions import _WD14_CATEGORY_TO_KIND, _canonicalize_wd14_name + payload = request.get_json(force=True, silent=True) or {} + tag_name = (payload.get('tag_name') or '').strip() + source = payload.get('source') or '' + category = payload.get('category') or '' + confidence = float(payload.get('confidence') or 0.0) + if not tag_name or not source or not category: + return jsonify({'ok': False, 'error': 'missing_fields'}), 400 + + image = ImageRecord.query.get(image_id) + if image is None: + return jsonify({'ok': False, 'error': 'image_not_found'}), 404 + + # Canonicalize before lookup so an already-attached "big hair" matches a + # freshly-POSTed "big_hair" (defensive — the client should already send canonical). + tag_name = _canonicalize_wd14_name(tag_name, category) + + tag = Tag.query.filter_by(name=tag_name).first() + if tag is None: + kind = _WD14_CATEGORY_TO_KIND.get(category) # None for 'general' + tag = Tag(name=tag_name, kind=kind) + db.session.add(tag) + db.session.flush() +``` + +Key changes: (1) inline import both helpers at function top; (2) canonicalize once, up front; (3) kind derivation moves before `Tag(...)` — no double-lookup, no redundant normalization. + +- [ ] **Step 2: Remove the now-unused inline import in the centroid-recompute block** + +Still in `accept_image_suggestion` (around line 766), the existing inline `from app.services.tag_suggestions import ELIGIBLE_CENTROID_KINDS` line stays — it's used by the centroid gate below. No change needed; just confirm it's still present after the Step 1 edit. + +- [ ] **Step 3: Test the happy path end-to-end** + +Open the modal for an image with a WD14 underscored suggestion. Click ✓. Then: + +```bash +docker compose exec db psql -U postgres imagerepo -c \ + "SELECT name, kind FROM tag ORDER BY id DESC LIMIT 5" +``` +Expected: the new row's `name` has no underscores (e.g., `big hair`, not `big_hair`). + +- [ ] **Step 4: Test idempotency — accept the same tag twice** + +In the modal, click ✓ on another underscored suggestion. Verify the chip animates out and the tag appears in the attached-tags list. Refresh the modal; the tag should not re-appear as a suggestion (because `already` canonicalization works). + +- [ ] **Step 5: Commit** + +```bash +git add app/main.py +git commit -m "feat(tags): canonicalize suggestion-accept tag names via service helper" +``` + +--- + +## Task 5: Write-path underscore stripping in `add_tag` + +**Files:** +- Modify: `app/main.py:823-910` (`add_tag`) + +- [ ] **Step 1: Inspect the current function for the insertion point** + +Open `app/main.py` at `add_tag` (line 823). The current prefix-split block (lines 837-843) reads: + +```python + if ":" in name: + kind = name.split(":", 1)[0] + tag_name = name + else: + kind = "user" + tag_name = name +``` + +- [ ] **Step 2: Add underscore stripping to the user path** + +Replace the block above with: + +```python + if ":" in name: + kind = name.split(":", 1)[0] + tag_name = name + else: + # No prefix — treat as a user-typed general tag. Strip underscores so + # WD14-style names pasted into the add box converge with our convention. + from app.utils.tag_names import underscores_to_spaces + kind = "user" + tag_name = underscores_to_spaces(name) +``` + +Note: this only affects the user/no-prefix case. Character and fandom paths are already fully covered because `normalize_display_name` now strips underscores internally (Task 1). + +- [ ] **Step 3: Manually verify** + +In the modal, type `big_hair` (no colon) into the add-tag input and click Add. Then: + +```bash +docker compose exec db psql -U postgres imagerepo -c \ + "SELECT name, kind FROM tag WHERE name ILIKE 'big%' ORDER BY id DESC LIMIT 3" +``` +Expected: a row with `name = 'big hair'`, `kind = 'user'`. No `big_hair` row. + +- [ ] **Step 4: Commit** + +```bash +git add app/main.py +git commit -m "feat(tags): strip underscores on user add-tag path" +``` + +--- + +## Task 6: Write-path underscore stripping in `bulk_add_tag` + +**Files:** +- Modify: `app/main.py:2279-2329` (`bulk_add_tag`) + +- [ ] **Step 1: Add underscore stripping to the kind-detection block** + +In `app/main.py`, locate `bulk_add_tag` (line 2279). The current block (lines 2296-2300) reads: + +```python + # Determine tag kind from name + if ':' in tag_name: + kind = tag_name.split(':', 1)[0] + else: + kind = 'user' +``` + +Replace with: + +```python + # Determine tag kind from name + if ':' in tag_name: + kind = tag_name.split(':', 1)[0] + else: + # No prefix — treat as a user-typed general tag. Strip underscores so + # WD14-style names pasted into the bulk box converge with our convention. + from app.utils.tag_names import underscores_to_spaces + kind = 'user' + tag_name = underscores_to_spaces(tag_name) +``` + +Character and fandom paths are already covered by the existing `normalize_display_name` calls (lines 2310, 2318) that now strip underscores internally. + +- [ ] **Step 2: Manually verify via the bulk endpoint** + +Pick two image IDs from the DB: + +```bash +docker compose exec db psql -U postgres imagerepo -c "SELECT id FROM image_record LIMIT 2" +``` + +Call the endpoint with an underscored name: + +```bash +curl -s -X POST http://localhost:5000/api/images/bulk-add-tag \ + -H 'Content-Type: application/json' \ + -d '{"image_ids":[,], "tag_name":"long_hair"}' +``` + +Then: + +```bash +docker compose exec db psql -U postgres imagerepo -c \ + "SELECT name, kind FROM tag WHERE name ILIKE 'long%' ORDER BY id DESC LIMIT 3" +``` +Expected: a row with `name = 'long hair'`, `kind = 'user'`. + +- [ ] **Step 3: Commit** + +```bash +git add app/main.py +git commit -m "feat(tags): strip underscores on bulk add-tag path" +``` + +--- + +## Task 7: One-time migration `i26041901` + +**Files:** +- Create: `migrations/versions/i26041901_normalize_underscores_and_general_tags.py` + +- [ ] **Step 1: Confirm the current head revision** + +```bash +cd /home/bvandeusen/Nextcloud/Projects/ImageRepo +docker compose exec web alembic heads +``` + +Expected: `h26041901 (head)`. If it differs, stop and ask the plan author — the migration's `down_revision` must match the actual current head. + +- [ ] **Step 2: Create the migration file** + +Create `migrations/versions/i26041901_normalize_underscores_and_general_tags.py` with this content: + +```python +"""Strip underscores from character/fandom/null-kind tag display names and merge +case-only duplicates that collapse onto the same canonical name. + +Revision ID: i26041901 +Revises: h26041901 +Create Date: 2026-04-19 + +Upgrade: for every row in `tag` with kind in ('character', 'fandom', NULL), +compute the canonical name (character/fandom: title-case + underscore→space; +null: underscore→space only). Rename survivors. On collision, merge image_tags, +fandom_id references, and tag_reference_embedding rows into the already- +correctly-named winner and delete the loser. + +Downgrade: raises. Merges are destructive; reverting requires a pre-migration +backup. +""" +from alembic import op +from sqlalchemy import text + + +revision = 'i26041901' +down_revision = 'h26041901' +branch_labels = None +depends_on = None + + +def _underscores_to_spaces(s: str) -> str: + """Mirror of app.utils.tag_names.underscores_to_spaces. Inlined so the + migration doesn't import app code (Alembic runs without the Flask app).""" + return s.replace('_', ' ') if s else s + + +def _normalize_display(s: str) -> str: + """Mirror of app.utils.tag_names.normalize_display_name.""" + if not s: + return s + s = _underscores_to_spaces(s) + out = [] + for p in s.split(' '): + if p == '': + out.append(p) + else: + out.append(p[:1].upper() + p[1:]) + return ' '.join(out) + + +def _compute_new_name(old_name: str, kind) -> str: + """Return the canonical full name for a given (name, kind) pair. + + kind == 'character': title-case base + (optional fandom suffix). + kind == 'fandom': title-case the display portion after the prefix. + kind IS NULL: underscore→space only; case preserved. + """ + if kind == 'character': + if ':' not in old_name: + return old_name + prefix, rest = old_name.split(':', 1) + if '(' in rest and rest.rstrip().endswith(')'): + paren_idx = rest.rfind('(') + base = rest[:paren_idx].rstrip() + fandom_part = rest[paren_idx+1:-1].strip() + return f"{prefix}:{_normalize_display(base)} ({_normalize_display(fandom_part)})" + return f"{prefix}:{_normalize_display(rest)}" + elif kind == 'fandom': + if ':' not in old_name: + return old_name + prefix, rest = old_name.split(':', 1) + return f"{prefix}:{_normalize_display(rest)}" + elif kind is None: + # Null-kind (general): keep case, just strip underscores. Split off any + # prefix defensively — historical rows may or may not carry one. + if ':' in old_name: + prefix, rest = old_name.split(':', 1) + return f"{prefix}:{_underscores_to_spaces(rest)}" + return _underscores_to_spaces(old_name) + return old_name + + +def upgrade(): + conn = op.get_bind() + rows = conn.execute( + text(""" + SELECT id, name, kind FROM tag + WHERE kind IN ('character', 'fandom') OR kind IS NULL + ORDER BY id + """) + ).fetchall() + + renames = 0 + merges = 0 + for row in rows: + tag_id = row[0] + old_name = row[1] + kind = row[2] + new_name = _compute_new_name(old_name, kind) + if new_name == old_name: + continue + + # Re-check that this tag still exists (may have been deleted as a loser + # in an earlier merge within this loop). + still_exists = conn.execute( + text("SELECT id FROM tag WHERE id = :id"), + {'id': tag_id}, + ).fetchone() + if not still_exists: + continue + + # Collision check: same target name + same kind (NULL-safe) + different id. + if kind is None: + target = conn.execute( + text("SELECT id, name FROM tag WHERE name = :new_name AND kind IS NULL AND id != :tag_id"), + {'new_name': new_name, 'tag_id': tag_id}, + ).fetchone() + else: + target = conn.execute( + text("SELECT id, name FROM tag WHERE name = :new_name AND kind = :kind AND id != :tag_id"), + {'new_name': new_name, 'kind': kind, 'tag_id': tag_id}, + ).fetchone() + + if target is None: + conn.execute( + text("UPDATE tag SET name = :new_name WHERE id = :tag_id"), + {'new_name': new_name, 'tag_id': tag_id}, + ) + renames += 1 + else: + winner_id = target[0] + winner_name = target[1] + loser_id = tag_id + loser_name = old_name + + # 1. Reassign image_tags into winner. + conn.execute( + text(""" + INSERT INTO image_tags (image_id, tag_id) + SELECT DISTINCT image_id, :winner_id + FROM image_tags + WHERE tag_id = :loser_id + ON CONFLICT DO NOTHING + """), + {'winner_id': winner_id, 'loser_id': loser_id}, + ) + conn.execute( + text("DELETE FROM image_tags WHERE tag_id = :loser_id"), + {'loser_id': loser_id}, + ) + + # 2. Reassign fandom_id FKs pointing at the loser. + conn.execute( + text("UPDATE tag SET fandom_id = :winner_id WHERE fandom_id = :loser_id"), + {'winner_id': winner_id, 'loser_id': loser_id}, + ) + + # 3. tag_reference_embedding: keep winner's row per model_version; + # otherwise rename loser's row to the winner. + conn.execute( + text(""" + UPDATE tag_reference_embedding + SET tag_name = :winner_name + WHERE tag_name = :loser_name + AND NOT EXISTS ( + SELECT 1 FROM tag_reference_embedding w + WHERE w.tag_name = :winner_name + AND w.model_version = tag_reference_embedding.model_version + ) + """), + {'winner_name': winner_name, 'loser_name': loser_name}, + ) + conn.execute( + text("DELETE FROM tag_reference_embedding WHERE tag_name = :loser_name"), + {'loser_name': loser_name}, + ) + + # 4. Delete the loser tag row. + conn.execute( + text("DELETE FROM tag WHERE id = :loser_id"), + {'loser_id': loser_id}, + ) + merges += 1 + + print(f"normalize_underscores_and_general_tags: {renames} renames, {merges} merges") + + +def downgrade(): + raise NotImplementedError( + "i26041901 canonicalizes tag names and merges duplicates that collapse. " + "Merges are destructive and cannot be reversed automatically. " + "Restore from a pre-migration backup if you need to roll back." + ) +``` + +- [ ] **Step 3: Seed test data before the migration** + +Run this to create a controlled test: one underscored-only-loser-collides-with-winner case, one pure-rename case, plus a null-kind rename: + +```bash +docker compose exec db psql -U postgres imagerepo <<'SQL' +-- Collision case: winner 'big hair' exists, loser 'big_hair' will merge in. +INSERT INTO tag (name, kind) VALUES ('big hair', NULL) ON CONFLICT DO NOTHING; +INSERT INTO tag (name, kind) VALUES ('big_hair', NULL); + +-- Pure rename case: no collision. +INSERT INTO tag (name, kind) VALUES ('fandom:trigun_stampede', 'fandom'); + +-- Null-kind pure rename: underscores survive as spaces, case preserved. +INSERT INTO tag (name, kind) VALUES ('long_blue_sky', NULL); +SQL +``` + +Record the `id`s for later verification: + +```bash +docker compose exec db psql -U postgres imagerepo -c \ + "SELECT id, name, kind FROM tag WHERE name IN ('big hair','big_hair','fandom:trigun_stampede','long_blue_sky') ORDER BY id" +``` + +Attach the loser `big_hair` to one real image so the merge has something to move. Pick any image id: + +```bash +docker compose exec db psql -U postgres imagerepo <<'SQL' +INSERT INTO image_tags (image_id, tag_id) +SELECT (SELECT id FROM image_record LIMIT 1), id +FROM tag WHERE name = 'big_hair'; +SQL +``` + +- [ ] **Step 4: Run the migration** + +```bash +docker compose exec web alembic upgrade head +``` + +Expected output includes: `normalize_underscores_and_general_tags: 2 renames, 1 merges`. (Two renames: `fandom:trigun_stampede` → `fandom:Trigun Stampede` and `long_blue_sky` → `long blue sky`. One merge: `big_hair` into `big hair`.) + +- [ ] **Step 5: Verify post-migration state** + +```bash +docker compose exec db psql -U postgres imagerepo <<'SQL' +-- Expected: exactly one 'big hair' row, no 'big_hair'. +SELECT name, kind FROM tag WHERE name ILIKE 'big%' AND kind IS NULL; +-- Expected: 'fandom:Trigun Stampede' present, 'fandom:trigun_stampede' absent. +SELECT name, kind FROM tag WHERE name ILIKE 'fandom:trigun%'; +-- Expected: 'long blue sky' present. +SELECT name, kind FROM tag WHERE name = 'long blue sky'; +-- Expected: the image previously tagged with 'big_hair' now has 'big hair'. +SELECT it.image_id, t.name FROM image_tags it JOIN tag t ON t.id = it.tag_id +WHERE t.name IN ('big hair', 'big_hair'); +SQL +``` + +- [ ] **Step 6: Commit** + +```bash +git add migrations/versions/i26041901_normalize_underscores_and_general_tags.py +git commit -m "feat(migrations): i26041901 normalize underscores + general tags with merge" +``` + +--- + +## Task 8: Modal JS — accept refresh parity + +**Files:** +- Modify: `app/static/js/view-modal.js:195-233` (`animateChipOut`, `acceptSuggestion`) + +- [ ] **Step 1: Simplify `animateChipOut`** + +In `app/static/js/view-modal.js`, replace the `animateChipOut` function (lines 195-201) with: + +```javascript + function animateChipOut(chip) { + chip.classList.add('leaving'); + setTimeout(() => { chip.remove(); }, 200); + } +``` + +(Removes the `onDone` callback — no callers need it after Task 8 completes.) + +- [ ] **Step 2: Rewrite `acceptSuggestion`** + +Still in `view-modal.js`, replace `acceptSuggestion` (lines 203-233) with: + +```javascript + async function acceptSuggestion(chip, imageId) { + if (chip.dataset.disabled === 'true') return; + chip.dataset.disabled = 'true'; + chip.style.pointerEvents = 'none'; + try { + const r = await fetch(`/image/${imageId}/suggestions/accept`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + tag_name: chip.dataset.tagName, + source: chip.dataset.source, + category: chip.dataset.category, + confidence: Number(chip.dataset.confidence) || 0, + }), + }); + const j = await r.json(); + if (!j.ok) { + chip.dataset.disabled = 'false'; + chip.style.pointerEvents = ''; + if (tagActionFeedback) { + tagActionFeedback.textContent = `Couldn't accept: ${j.error || 'error'}`; + } + return; + } + if (typeof loadTags === 'function') await loadTags(imageId); + if (tagActionFeedback) tagActionFeedback.textContent = `Added ${j.tag.name}`; + if (typeof window.refreshItemTags === 'function') window.refreshItemTags(imageId); + animateChipOut(chip); + } catch { + chip.dataset.disabled = 'false'; + chip.style.pointerEvents = ''; + } + } +``` + +Key changes vs. current code: (1) `chip.dataset.disabled` guards double-click; (2) `await loadTags(imageId)` runs before feedback text and animation, mirroring manual-add timing (`addTag` at line 601); (3) chip animation no longer gates the data refresh. + +- [ ] **Step 3: Update the one other caller of `animateChipOut` — `rejectSuggestion`** + +Still in `view-modal.js`, `rejectSuggestion` (around line 235) currently calls `animateChipOut(chip, () => { ... })`. Replace those lines (currently 247-249) with: + +```javascript + animateChipOut(chip); + if (tagActionFeedback) tagActionFeedback.textContent = `Rejected ${chip.dataset.tagName}`; +``` + +(The feedback text was the only thing the callback did. Moving it out keeps the same ordering relative to the user's perception — the chip fades while the message appears.) + +- [ ] **Step 4: Manually verify accept timing** + +In a browser, open DevTools → Network tab. Open an image modal with suggestions. Click ✓ on one. Watch the Network panel: + +Expected sequence: +1. `POST /image//suggestions/accept` → 200 +2. `GET /image//tags` → 200 (the `loadTags` call) +3. Chip's leaving animation plays and chip disappears + +The attached-tags section should show the new tag *before* the chip disappears (because `await loadTags` runs before `animateChipOut`). + +- [ ] **Step 5: Verify duplicate-click protection** + +Rapid double-click ✓. Check Network: only one `POST /accept` fires. + +- [ ] **Step 6: Verify error path** + +Temporarily edit `accept_image_suggestion` in `app/main.py` to unconditionally return `jsonify({'ok': False, 'error': 'test'}), 400` at the top. Reload the page, click ✓. Verify: the chip stays on screen, its pointer events re-enable, feedback text says "Couldn't accept: test". Then revert the temporary change — do NOT commit the test stub. + +- [ ] **Step 7: Commit** + +```bash +git add app/static/js/view-modal.js +git commit -m "feat(ui): suggestion accept awaits loadTags, mirrors manual-add timing" +``` + +--- + +## Task 9: CSS — persistent green/red chip buttons + +**Files:** +- Modify: `app/static/style.css:998-1006` + +- [ ] **Step 1: Replace the button color rules** + +In `app/static/style.css`, locate the two `:hover` blocks at lines 998-1006: + +```css +.suggestion-chip .sugg-btn.sugg-accept:hover { + color: #8be78b; + border-color: rgba(139, 231, 139, 0.6); + background: rgba(139, 231, 139, 0.08); +} +.suggestion-chip .sugg-btn.sugg-reject:hover { + color: #e78b8b; + border-color: rgba(231, 139, 139, 0.6); + background: rgba(231, 139, 139, 0.08); +} +``` + +Replace with: + +```css +.suggestion-chip .sugg-btn.sugg-accept { + color: #8be78b; + border-color: rgba(139, 231, 139, 0.35); +} +.suggestion-chip .sugg-btn.sugg-accept:hover { + background: rgba(139, 231, 139, 0.12); + border-color: rgba(139, 231, 139, 0.6); +} +.suggestion-chip .sugg-btn.sugg-reject { + color: #e78b8b; + border-color: rgba(231, 139, 139, 0.35); +} +.suggestion-chip .sugg-btn.sugg-reject:hover { + background: rgba(231, 139, 139, 0.12); + border-color: rgba(231, 139, 139, 0.6); +} +``` + +Rationale: the base `.sugg-btn` rule already covers layout + transparent background. The new accept/reject base rules add the semantic color; hover intensifies background + border. + +- [ ] **Step 2: Manually verify** + +Hard-refresh the browser (Ctrl-Shift-R) to bust CSS cache. Open an image modal with suggestions. Without hovering, verify: +- ✓ buttons show a green glyph and a soft green border. +- ✕ buttons show a red glyph and a soft red border. +- Hovering intensifies both. + +- [ ] **Step 3: Commit** + +```bash +git add app/static/style.css +git commit -m "feat(ui): persistent green/red tint on suggestion accept/reject buttons" +``` + +--- + +## Task 10: End-to-end smoke test + +- [ ] **Step 1: Full-flow test on a WD14-predicted image** + +Pick an image that has underscored WD14 predictions (any image already processed by `tag_and_embed`). In the browser: + +1. Open the modal. +2. Confirm: every suggestion chip's name is underscore-free (e.g., `long hair`, not `long_hair`). +3. Confirm: ✓ is green-tinted, ✕ is red-tinted, both without hover. +4. Click ✓ on one suggestion. +5. Confirm: attached-tags list updates before chip animation finishes; feedback text "Added " shows. +6. Confirm: refreshing the page and re-opening the modal no longer shows that suggestion. +7. In DB: `SELECT name FROM tag WHERE id = (SELECT tag_id FROM image_tags WHERE image_id = ORDER BY tag_id DESC LIMIT 1)` — expect no underscores. + +- [ ] **Step 2: Rollback of test seed data (if present from Task 7)** + +Clean up any leftover test-only tags from the Task 7 seed step. Inspect and delete as needed: + +```bash +docker compose exec db psql -U postgres imagerepo <<'SQL' +-- Only delete if they came from test seeding. Check what is attached first. +SELECT t.id, t.name, t.kind, COUNT(it.image_id) AS attached_count +FROM tag t LEFT JOIN image_tags it ON it.tag_id = t.id +WHERE t.name IN ('big hair', 'long blue sky', 'fandom:Trigun Stampede') +GROUP BY t.id, t.name, t.kind; +SQL +``` + +If any of these were only attached by the Task 7 seed (and the user doesn't want to keep them), delete: + +```bash +docker compose exec db psql -U postgres imagerepo <<'SQL' +DELETE FROM image_tags WHERE tag_id IN (SELECT id FROM tag WHERE name IN ('big hair', 'long blue sky', 'fandom:Trigun Stampede')); +DELETE FROM tag WHERE name IN ('big hair', 'long blue sky', 'fandom:Trigun Stampede'); +SQL +``` + +Skip this step entirely if the user's real library already contained any of those names pre-migration. + +- [ ] **Step 3: Final commit (if anything uncommitted remains)** + +```bash +git status +``` +If clean, nothing to do. Plan complete. + +--- + +## Self-Review Notes + +**Spec coverage check:** +- WD14 underscore→space at emit → Task 3 ✓ +- Write-path canonicalization (character/fandom via `normalize_display_name`) → Task 1 (the helper change itself covers both kinds in `add_tag`/`bulk_add_tag`/`set_tag_fandom`/`accept`) ✓ +- General (null) underscore→space on write paths → Tasks 4 (accept), 5 (add_tag), 6 (bulk_add_tag) ✓ +- Migration `i26041901` rename-or-merge across three kinds → Task 7 ✓ +- Accept refresh parity with manual add → Task 8 ✓ +- Persistent green/red chip buttons → Task 9 ✓ +- `_WD14_CATEGORY_TO_KIND` relocated to service (per spec) → Task 2 ✓ + +**No `set_tag_fandom` task** — the spec doesn't name it as a write path that needs new code. `set_tag_fandom` already routes through `normalize_display_name` (from the prior character-tag-integrity feature), which now strips underscores automatically via the Task 1 change. No explicit edit required. + +**No test files** — matches project convention (see the parent feature `h26041901` and the character-tag integrity spec, both of which used manual verification only). diff --git a/docs/superpowers/plans/2026-04-19-user-tag-embedding-suggestions.md b/docs/superpowers/plans/2026-04-19-user-tag-embedding-suggestions.md new file mode 100644 index 0000000..db21444 --- /dev/null +++ b/docs/superpowers/plans/2026-04-19-user-tag-embedding-suggestions.md @@ -0,0 +1,1052 @@ +# User-Tag Embedding Suggestions Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Extend SigLIP centroid-based tag suggestions from `character` only to `character` + `fandom` + null (general/topic) kinds, and drop `rating` from the suggestion UI entirely. + +**Architecture:** Rename `character_reference_embedding` table to `tag_reference_embedding`, add `tag_kind` column, and thread that kind through the recompute task, suggestion service, and accept endpoint. Add a batch "recompute all eligible centroids" task and expose it in Settings → Maintenance. + +**Tech Stack:** Flask + SQLAlchemy + Alembic migrations, Postgres 16 + pgvector, Celery 5 + Redis, SigLIP SO400M embeddings. + +**Spec:** `docs/superpowers/specs/2026-04-19-user-tag-embedding-suggestions-design.md` + +--- + +## File Structure + +Files created: + +- `migrations/versions/g26041901_rename_character_to_tag_reference.py` — Alembic migration (rename table/column, add `tag_kind`, rename/drop config rows). + +Files modified (one responsibility each): + +- `app/models.py:250-256` — rename `CharacterReferenceEmbedding` class → `TagReferenceEmbedding`, rename column `character_tag` → `tag_name`, add `tag_kind`. +- `app/services/tag_suggestions.py` — defines `ELIGIBLE_CENTROID_KINDS`, rewrites `_embedding_character_suggestions` as `_embedding_tag_suggestions`, drops `rating` from grouped output, renames config keys. +- `app/tasks/ml.py:146-189` — rewrite `recompute_centroid` to accept any eligible kind, add new `recompute_all_centroids` batch task. +- `app/main.py:691-772` — drop `rating` from `_WD14_CATEGORY_TO_KIND`, extend centroid trigger to all eligible kinds, add `trigger_recompute_all_centroids` route. +- `app/static/js/view-modal.js:88-117` — drop `rating` from `SUGGESTION_CATEGORY_ORDER` / `SUGGESTION_CATEGORY_LABEL`. +- `app/templates/settings.html:440-449` — add "Recompute all centroids" button next to existing ML backfill form. + +--- + +## Task 1: Database migration — rename table, add tag_kind, clean config + +**Files:** +- Create: `migrations/versions/g26041901_rename_character_to_tag_reference.py` +- Reference: `migrations/versions/0017871d2221_add_tag_suggestions.py` (prior migration, down_revision target) + +Check current alembic head first to confirm the `down_revision`: + +- [ ] **Step 1: Confirm the alembic revision chain** + +```bash +docker compose exec web alembic heads +``` + +Expected: `0017871d2221 (head)` (this is the tag-suggestions migration that created the table we're renaming). + +- [ ] **Step 2: Create the migration file** + +Write `migrations/versions/g26041901_rename_character_to_tag_reference.py`: + +```python +"""rename character_reference_embedding to tag_reference_embedding and add tag_kind + +Revision ID: g26041901 +Revises: 0017871d2221 +Create Date: 2026-04-19 + +""" +from alembic import op +import sqlalchemy as sa + + +revision = 'g26041901' +down_revision = '0017871d2221' +branch_labels = None +depends_on = None + + +def upgrade(): + # Rename table + op.rename_table('character_reference_embedding', 'tag_reference_embedding') + + # Rename column + op.alter_column('tag_reference_embedding', 'character_tag', new_column_name='tag_name') + + # Add tag_kind column (nullable — null = general/topic tag) + op.add_column( + 'tag_reference_embedding', + sa.Column('tag_kind', sa.Text(), nullable=True), + ) + + # Every pre-existing row was written by the character-only path, so tag_kind='character'. + op.execute("UPDATE tag_reference_embedding SET tag_kind = 'character'") + + # Rename config keys. Postgres-safe single UPDATE per key. + op.execute( + "UPDATE tag_suggestion_config " + "SET key = 'threshold_embedding' " + "WHERE key = 'threshold_embedding_character'" + ) + op.execute( + "UPDATE tag_suggestion_config " + "SET key = 'min_reference_images' " + "WHERE key = 'min_reference_images_per_character'" + ) + + # Drop the rating threshold — rating is no longer surfaced in the UI. + op.execute("DELETE FROM tag_suggestion_config WHERE key = 'threshold_rating'") + + +def downgrade(): + # Restore threshold_rating seed with original default. + op.execute( + "INSERT INTO tag_suggestion_config (key, value, description) " + "VALUES ('threshold_rating', '0.5', 'Min confidence for rating tags') " + "ON CONFLICT (key) DO NOTHING" + ) + + # Revert config key renames. + op.execute( + "UPDATE tag_suggestion_config " + "SET key = 'min_reference_images_per_character' " + "WHERE key = 'min_reference_images'" + ) + op.execute( + "UPDATE tag_suggestion_config " + "SET key = 'threshold_embedding_character' " + "WHERE key = 'threshold_embedding'" + ) + + # Drop tag_kind column. + op.drop_column('tag_reference_embedding', 'tag_kind') + + # Revert column and table renames. + op.alter_column('tag_reference_embedding', 'tag_name', new_column_name='character_tag') + op.rename_table('tag_reference_embedding', 'character_reference_embedding') +``` + +- [ ] **Step 3: Run the migration against the dev DB** + +```bash +docker compose exec web alembic upgrade head +``` + +Expected output: `Running upgrade 0017871d2221 -> g26041901, rename character_reference_embedding to tag_reference_embedding and add tag_kind`. + +- [ ] **Step 4: Verify the schema changed** + +```bash +docker compose exec -T postgres psql -U imagerepo -d imagerepo -c "\d tag_reference_embedding" +``` + +Expected: table `tag_reference_embedding` with columns `tag_name`, `tag_kind`, `model_version`, `centroid`, `reference_count`, `computed_at`. Primary key `(tag_name, model_version)`. + +```bash +docker compose exec -T postgres psql -U imagerepo -d imagerepo -c "SELECT tag_name, tag_kind FROM tag_reference_embedding LIMIT 5" +``` + +Expected: any pre-existing rows show `tag_kind = 'character'`. + +```bash +docker compose exec -T postgres psql -U imagerepo -d imagerepo -c "SELECT key FROM tag_suggestion_config ORDER BY key" +``` + +Expected: includes `threshold_embedding`, `min_reference_images`; does **not** include `threshold_embedding_character`, `min_reference_images_per_character`, or `threshold_rating`. + +- [ ] **Step 5: Commit** + +```bash +git add migrations/versions/g26041901_rename_character_to_tag_reference.py +git commit -m "feat(db): rename character_reference_embedding → tag_reference_embedding + tag_kind" +``` + +--- + +## Task 2: Update the SQLAlchemy model + +**Files:** +- Modify: `app/models.py:250-256` + +- [ ] **Step 1: Replace the model class** + +Find this block in `app/models.py`: + +```python +class CharacterReferenceEmbedding(db.Model): + __tablename__ = "character_reference_embedding" + character_tag = db.Column(db.Text, primary_key=True) + model_version = db.Column(db.Text, primary_key=True) + centroid = db.Column(Vector(1152), nullable=False) + reference_count = db.Column(db.Integer, nullable=False) + computed_at = db.Column(db.DateTime(timezone=True), server_default=db.func.now(), nullable=False) +``` + +Replace with: + +```python +class TagReferenceEmbedding(db.Model): + __tablename__ = "tag_reference_embedding" + tag_name = db.Column(db.Text, primary_key=True) + model_version = db.Column(db.Text, primary_key=True) + tag_kind = db.Column(db.Text, nullable=True) + centroid = db.Column(Vector(1152), nullable=False) + reference_count = db.Column(db.Integer, nullable=False) + computed_at = db.Column(db.DateTime(timezone=True), server_default=db.func.now(), nullable=False) +``` + +- [ ] **Step 2: Verify no other import references the old class name** + +```bash +grep -rn "CharacterReferenceEmbedding" app/ +``` + +Expected output: references only from `app/models.py` (the class definition, now renamed) and these files which we'll update in later tasks: +- `app/tasks/ml.py` (Task 3) +- `app/services/tag_suggestions.py` (Task 4) +- `app/main.py` (Task 5) + +If any **other** file references `CharacterReferenceEmbedding`, stop and add it to the fix list. + +- [ ] **Step 3: Commit** + +```bash +git add app/models.py +git commit -m "feat(models): rename CharacterReferenceEmbedding → TagReferenceEmbedding" +``` + +--- + +## Task 3: Rewrite `recompute_centroid` task for any eligible kind + add batch task + +**Files:** +- Modify: `app/tasks/ml.py:146-189` (replace entire `recompute_centroid` function) +- Modify: `app/tasks/ml.py` (add new `recompute_all_centroids` task at end of file) + +- [ ] **Step 1: Update imports** + +At the top of `app/tasks/ml.py`, find: + +```python +from app.models import ( + ImageRecord, + Tag, + ImageTagPrediction, + ImageEmbedding, + CharacterReferenceEmbedding, + TagSuggestionConfig, + image_tags, +) +``` + +Replace `CharacterReferenceEmbedding` with `TagReferenceEmbedding`: + +```python +from app.models import ( + ImageRecord, + Tag, + ImageTagPrediction, + ImageEmbedding, + TagReferenceEmbedding, + TagSuggestionConfig, + image_tags, +) +``` + +- [ ] **Step 2: Replace `recompute_centroid` with the kind-agnostic version** + +Find the existing function (lines ~146-189) and replace with: + +```python +@celery.task(bind=True, name='app.tasks.ml.recompute_centroid', + soft_time_limit=60, time_limit=120) +def recompute_centroid(self, tag_name: str): + """Recompute the mean embedding for an eligible tag from its currently-associated images. + + Eligible tag kinds are defined by ELIGIBLE_CENTROID_KINDS in + app.services.tag_suggestions. Tags of any other kind return early with + status='ineligible_kind' and do not write a centroid row. + """ + from app.ml.siglip import MODEL_VERSION as SIGLIP_VER + from app.services.tag_suggestions import ELIGIBLE_CENTROID_KINDS + + tag = Tag.query.filter_by(name=tag_name).first() + if tag is None: + log.warning(f"recompute_centroid: tag {tag_name!r} not found") + return {'status': 'missing_tag'} + + if tag.kind not in ELIGIBLE_CENTROID_KINDS: + log.info(f"recompute_centroid: tag {tag_name!r} has ineligible kind {tag.kind!r}") + return {'status': 'ineligible_kind'} + + rows = ( + db.session.query(ImageEmbedding.embedding) + .join(image_tags, image_tags.c.image_id == ImageEmbedding.image_id) + .filter(image_tags.c.tag_id == tag.id) + .filter(ImageEmbedding.model_version == SIGLIP_VER) + .all() + ) + if not rows: + log.info(f"recompute_centroid: no embeddings yet for {tag_name}") + return {'status': 'no_embeddings'} + + vectors = np.array([np.array(r[0], dtype=np.float32) for r in rows]) + centroid = vectors.mean(axis=0) + + stmt = pg_insert(TagReferenceEmbedding).values( + tag_name=tag_name, + tag_kind=tag.kind, + model_version=SIGLIP_VER, + centroid=centroid.tolist(), + reference_count=len(rows), + computed_at=func.now(), + ) + stmt = stmt.on_conflict_do_update( + index_elements=['tag_name', 'model_version'], + set_={ + 'tag_kind': stmt.excluded.tag_kind, + 'centroid': stmt.excluded.centroid, + 'reference_count': stmt.excluded.reference_count, + 'computed_at': func.now(), + }, + ) + db.session.execute(stmt) + db.session.commit() + log.info(f"recompute_centroid: {tag_name} (kind={tag.kind}) -> n={len(rows)}") + return {'status': 'ok', 'reference_count': len(rows), 'tag_kind': tag.kind} +``` + +- [ ] **Step 3: Add the batch task at end of `app/tasks/ml.py`** + +Append: + +```python +@celery.task(bind=True, name='app.tasks.ml.recompute_all_centroids', + soft_time_limit=None, time_limit=None) +def recompute_all_centroids(self): + """Enqueue recompute_centroid for every eligible tag with enough reference images. + + Uses a single aggregate query to find tags with >= min_reference_images applied + images, then enqueues one recompute_centroid task per tag on the ml queue. + """ + from app.services.tag_suggestions import ELIGIBLE_CENTROID_KINDS + + min_refs_row = TagSuggestionConfig.query.filter_by(key='min_reference_images').first() + min_refs = int(min_refs_row.value) if min_refs_row else 5 + + # Build an IS NULL / IN filter that covers ELIGIBLE_CENTROID_KINDS including None. + kinds_not_null = [k for k in ELIGIBLE_CENTROID_KINDS if k is not None] + allow_null = None in ELIGIBLE_CENTROID_KINDS + + kind_filter = Tag.kind.in_(kinds_not_null) + if allow_null: + kind_filter = kind_filter | Tag.kind.is_(None) + + rows = ( + db.session.query(Tag.name, func.count(image_tags.c.image_id).label('n')) + .join(image_tags, image_tags.c.tag_id == Tag.id) + .filter(kind_filter) + .group_by(Tag.name) + .having(func.count(image_tags.c.image_id) >= min_refs) + .all() + ) + + enqueued = 0 + for tag_name, n in rows: + recompute_centroid.apply_async(args=[tag_name], queue='ml') + enqueued += 1 + log.info(f"recompute_all_centroids: enqueued {enqueued} tags (min_refs={min_refs})") + return {'status': 'ok', 'enqueued': enqueued, 'min_refs': min_refs} +``` + +- [ ] **Step 4: Register the new task in celery_app.py includes** + +Check `app/celery_app.py` has `'app.tasks.ml'` in the `include=` list. It should already be present from the prior feature — if missing, add it. Run: + +```bash +grep -n "app.tasks.ml" app/celery_app.py +``` + +Expected: at least one line showing `'app.tasks.ml'` in the `include=[...]` tuple. If absent, add it. + +- [ ] **Step 5: Rebuild the ml-worker so the new task registers** + +```bash +docker compose up -d --build ml-worker +``` + +- [ ] **Step 6: Verify new task appears in the worker task list** + +```bash +docker compose logs ml-worker --tail 50 2>&1 | grep -E "recompute_all_centroids|recompute_centroid" +``` + +Expected: both `app.tasks.ml.recompute_centroid` and `app.tasks.ml.recompute_all_centroids` appear in the `[tasks]` listing. + +- [ ] **Step 7: Commit** + +```bash +git add app/tasks/ml.py +git commit -m "feat(ml): kind-aware recompute_centroid + recompute_all_centroids batch task" +``` + +--- + +## Task 4: Update suggestion service — eligible kinds, drop rating group + +**Files:** +- Modify: `app/services/tag_suggestions.py` + +- [ ] **Step 1: Update imports + add eligible-kinds constant at the top** + +Find the imports near the top of `app/services/tag_suggestions.py`: + +```python +from app.models import ( + ImageTagPrediction, + ImageEmbedding, + CharacterReferenceEmbedding, + TagSuggestionConfig, + ImageRecord, + Tag, + image_tags, +) +``` + +Replace with (swap class name): + +```python +from app.models import ( + ImageTagPrediction, + ImageEmbedding, + TagReferenceEmbedding, + TagSuggestionConfig, + ImageRecord, + Tag, + image_tags, +) +``` + +Then below the imports, add the constant: + +```python +# Tag kinds that receive embedding-similarity suggestions. None = general/topic tags. +# Adding 'artist' or 'series' here enables them with no other code changes. +ELIGIBLE_CENTROID_KINDS = ('character', 'fandom', None) +``` + +- [ ] **Step 2: Rename config keys in `_DEFAULTS`** + +Find: + +```python +_DEFAULTS = { + 'threshold_general': 0.35, + 'threshold_character_wd14': 0.75, + 'threshold_copyright': 0.5, + 'threshold_rating': 0.5, + 'threshold_meta': 0.5, + 'threshold_embedding_character': 0.85, + 'min_reference_images_per_character': 5.0, + 'wd14_model_version': 'wd-eva02-large-tagger-v3', + 'siglip_model_version': 'siglip-so400m-patch14-384', +} +``` + +Replace with: + +```python +_DEFAULTS = { + 'threshold_general': 0.35, + 'threshold_character_wd14': 0.75, + 'threshold_copyright': 0.5, + 'threshold_meta': 0.5, + 'threshold_embedding': 0.85, + 'min_reference_images': 5.0, + 'wd14_model_version': 'wd-eva02-large-tagger-v3', + 'siglip_model_version': 'siglip-so400m-patch14-384', +} +``` + +- [ ] **Step 3: Replace `_embedding_character_suggestions` with `_embedding_tag_suggestions`** + +Find the existing function: + +```python +def _embedding_character_suggestions(image_id: int, cfg: dict, already: set[str]) -> list[dict]: + siglip_ver = cfg.get('siglip_model_version', _DEFAULTS['siglip_model_version']) + min_refs = int(cfg.get('min_reference_images_per_character', 5)) + threshold = cfg.get('threshold_embedding_character', 0.85) + + image_emb = ( + ImageEmbedding.query + .filter_by(image_id=image_id, model_version=siglip_ver) + .first() + ) + if image_emb is None: + return [] + + # pgvector cosine distance; similarity = 1 - distance + distance = CharacterReferenceEmbedding.centroid.cosine_distance(image_emb.embedding) + rows = ( + db.session.query( + CharacterReferenceEmbedding.character_tag, + CharacterReferenceEmbedding.reference_count, + distance.label('distance'), + ) + .filter(CharacterReferenceEmbedding.model_version == siglip_ver) + .filter(CharacterReferenceEmbedding.reference_count >= min_refs) + .order_by(distance) + .limit(20) + .all() + ) + out: list[dict] = [] + for tag_name, ref_count, dist in rows: + similarity = 1.0 - float(dist) + if similarity < threshold: + continue + if tag_name in already: + continue + out.append({ + 'name': tag_name, + 'category': 'character', + 'confidence': similarity, + 'source': 'embedding_similarity', + }) + return out +``` + +Replace with: + +```python +# Maps tag kind (as stored on Tag and TagReferenceEmbedding) to the display +# category used in the modal UI's grouped suggestions. +_KIND_TO_DISPLAY_CATEGORY = { + 'character': 'character', + 'fandom': 'copyright', + None: 'general', +} + + +def _embedding_tag_suggestions(image_id: int, cfg: dict, already: set[str]) -> list[dict]: + siglip_ver = cfg.get('siglip_model_version', _DEFAULTS['siglip_model_version']) + min_refs = int(cfg.get('min_reference_images', 5)) + threshold = cfg.get('threshold_embedding', 0.85) + + image_emb = ( + ImageEmbedding.query + .filter_by(image_id=image_id, model_version=siglip_ver) + .first() + ) + if image_emb is None: + return [] + + # pgvector cosine distance; similarity = 1 - distance + distance = TagReferenceEmbedding.centroid.cosine_distance(image_emb.embedding) + rows = ( + db.session.query( + TagReferenceEmbedding.tag_name, + TagReferenceEmbedding.tag_kind, + TagReferenceEmbedding.reference_count, + distance.label('distance'), + ) + .filter(TagReferenceEmbedding.model_version == siglip_ver) + .filter(TagReferenceEmbedding.reference_count >= min_refs) + .order_by(distance) + .limit(40) + .all() + ) + out: list[dict] = [] + for tag_name, tag_kind, ref_count, dist in rows: + similarity = 1.0 - float(dist) + if similarity < threshold: + continue + if tag_name in already: + continue + category = _KIND_TO_DISPLAY_CATEGORY.get(tag_kind) + if category is None and tag_kind is not None: + # Unknown kind — skip defensively. (Should never happen because + # recompute_centroid only writes eligible kinds.) + continue + if tag_kind is None: + category = 'general' + out.append({ + 'name': tag_name, + 'category': category, + 'confidence': similarity, + 'source': 'embedding_similarity', + }) + return out +``` + +- [ ] **Step 4: Update `get_suggestions` call site + drop `rating` group** + +Find: + +```python +def get_suggestions(image_id: int, top_k_per_category: int = 10) -> dict: + """Return suggestions grouped by category for one image. + + Shape: + { + 'character': [{name, confidence, source, exists_in_db}, ...], + 'copyright': [...], + 'rating': [...], + 'general': [...], + } + Ordered by confidence desc within each category. 'meta' is omitted. + """ + if ImageRecord.query.get(image_id) is None: + return {'character': [], 'copyright': [], 'rating': [], 'general': []} + + cfg = _config() + already = _existing_tag_names_for_image(image_id) + + merged = _merge( + _wd14_suggestions(image_id, cfg, already), + _embedding_character_suggestions(image_id, cfg, already), + ) + + existing_all = _existing_tag_names() + + grouped: dict[str, list[dict]] = {'character': [], 'copyright': [], 'rating': [], 'general': []} + for s in merged: + if s['category'] not in grouped: + continue # meta / artist / unknown are dropped + s['exists_in_db'] = s['name'] in existing_all + grouped[s['category']].append(s) + + for cat in grouped: + grouped[cat].sort(key=lambda x: x['confidence'], reverse=True) + grouped[cat] = grouped[cat][:top_k_per_category] + + return grouped +``` + +Replace with: + +```python +def get_suggestions(image_id: int, top_k_per_category: int = 10) -> dict: + """Return suggestions grouped by category for one image. + + Shape: + { + 'character': [{name, confidence, source, exists_in_db}, ...], + 'copyright': [...], + 'general': [...], + } + Ordered by confidence desc within each category. 'meta' and 'rating' are omitted. + """ + if ImageRecord.query.get(image_id) is None: + return {'character': [], 'copyright': [], 'general': []} + + cfg = _config() + already = _existing_tag_names_for_image(image_id) + + merged = _merge( + _wd14_suggestions(image_id, cfg, already), + _embedding_tag_suggestions(image_id, cfg, already), + ) + + existing_all = _existing_tag_names() + + grouped: dict[str, list[dict]] = {'character': [], 'copyright': [], 'general': []} + for s in merged: + if s['category'] not in grouped: + continue # meta / rating / artist / unknown are dropped + s['exists_in_db'] = s['name'] in existing_all + grouped[s['category']].append(s) + + for cat in grouped: + grouped[cat].sort(key=lambda x: x['confidence'], reverse=True) + grouped[cat] = grouped[cat][:top_k_per_category] + + return grouped +``` + +- [ ] **Step 5: Update `_category_threshold` to drop rating** + +Find: + +```python +def _category_threshold(category: str, cfg: dict) -> float: + return { + 'general': cfg['threshold_general'], + 'character': cfg['threshold_character_wd14'], + 'copyright': cfg['threshold_copyright'], + 'rating': cfg['threshold_rating'], + 'meta': cfg['threshold_meta'], + }.get(category, 1.01) # unknown → effectively disabled +``` + +Replace with: + +```python +def _category_threshold(category: str, cfg: dict) -> float: + return { + 'general': cfg['threshold_general'], + 'character': cfg['threshold_character_wd14'], + 'copyright': cfg['threshold_copyright'], + 'meta': cfg['threshold_meta'], + }.get(category, 1.01) # unknown/rating → effectively disabled (not surfaced) +``` + +- [ ] **Step 6: Verify no stale `_embedding_character_suggestions` / `CharacterReferenceEmbedding` references** + +```bash +grep -n "_embedding_character_suggestions\|CharacterReferenceEmbedding" app/services/tag_suggestions.py +``` + +Expected: no output. + +- [ ] **Step 7: Commit** + +```bash +git add app/services/tag_suggestions.py +git commit -m "feat(suggestions): kind-aware embedding suggestions, drop rating group" +``` + +--- + +## Task 5: Update accept endpoint — drop rating mapping, extend centroid trigger + +**Files:** +- Modify: `app/main.py:691-772` + +- [ ] **Step 1: Update `_WD14_CATEGORY_TO_KIND` — drop rating** + +Find: + +```python +_WD14_CATEGORY_TO_KIND = { + 'character': 'character', + 'copyright': 'fandom', # ImageRepo uses 'fandom' for IP/franchise + 'rating': 'rating', +} +``` + +Replace with: + +```python +_WD14_CATEGORY_TO_KIND = { + 'character': 'character', + 'copyright': 'fandom', # ImageRepo uses 'fandom' for IP/franchise +} +``` + +- [ ] **Step 2: Extend the centroid-recompute trigger to all eligible kinds** + +Find this block inside `accept_image_suggestion`: + +```python + # Schedule centroid recompute if this was a character and delta is exceeded. + if tag.kind == 'character': + from app.models import CharacterReferenceEmbedding, TagSuggestionConfig + from app.tasks.ml import recompute_centroid + from app.ml.siglip import MODEL_VERSION as SIGLIP_VER + + delta_cfg = TagSuggestionConfig.query.filter_by(key='centroid_recompute_delta').first() + delta_threshold = int(delta_cfg.value) if delta_cfg else 3 + + current_count = ( + db.session.query(db.func.count(image_tags.c.image_id)) + .filter(image_tags.c.tag_id == tag.id) + .scalar() + ) or 0 + ref = ( + CharacterReferenceEmbedding.query + .filter_by(character_tag=tag_name, model_version=SIGLIP_VER) + .first() + ) + last_count = ref.reference_count if ref else 0 + if current_count - last_count >= delta_threshold: + try: + recompute_centroid.apply_async(args=[tag_name], queue='ml') + except Exception: + # Don't fail the accept if enqueue fails + pass +``` + +Replace with: + +```python + # Schedule centroid recompute if this tag's kind is eligible and delta is exceeded. + from app.services.tag_suggestions import ELIGIBLE_CENTROID_KINDS + if tag.kind in ELIGIBLE_CENTROID_KINDS: + from app.models import TagReferenceEmbedding, TagSuggestionConfig + from app.tasks.ml import recompute_centroid + from app.ml.siglip import MODEL_VERSION as SIGLIP_VER + + delta_cfg = TagSuggestionConfig.query.filter_by(key='centroid_recompute_delta').first() + delta_threshold = int(delta_cfg.value) if delta_cfg else 3 + + current_count = ( + db.session.query(db.func.count(image_tags.c.image_id)) + .filter(image_tags.c.tag_id == tag.id) + .scalar() + ) or 0 + ref = ( + TagReferenceEmbedding.query + .filter_by(tag_name=tag_name, model_version=SIGLIP_VER) + .first() + ) + last_count = ref.reference_count if ref else 0 + if current_count - last_count >= delta_threshold: + try: + recompute_centroid.apply_async(args=[tag_name], queue='ml') + except Exception: + # Don't fail the accept if enqueue fails + pass +``` + +- [ ] **Step 3: Add the `trigger_recompute_all_centroids` route** + +Find the existing `trigger_ml_backfill` function (around line 604): + +```python +@main.post('/settings/maintenance/ml-backfill') +def trigger_ml_backfill(): + from app.tasks.ml import backfill + backfill.apply_async(queue='ml') + return redirect(url_for('main.settings', tab='maintenance')) +``` + +Immediately after it, add: + +```python +@main.post('/settings/maintenance/recompute-centroids') +def trigger_recompute_all_centroids(): + from app.tasks.ml import recompute_all_centroids + recompute_all_centroids.apply_async(queue='ml') + return redirect(url_for('main.settings', tab='maintenance')) +``` + +- [ ] **Step 4: Verify no stale references** + +```bash +grep -n "CharacterReferenceEmbedding\|'rating':\s*'rating'" app/main.py +``` + +Expected: no output. + +- [ ] **Step 5: Commit** + +```bash +git add app/main.py +git commit -m "feat(accept): extend centroid trigger to eligible kinds, drop rating mapping" +``` + +--- + +## Task 6: Update modal UI — drop rating group + +**Files:** +- Modify: `app/static/js/view-modal.js:88-117` + +- [ ] **Step 1: Find and update `SUGGESTION_CATEGORY_ORDER` and `SUGGESTION_CATEGORY_LABEL`** + +Find: + +```javascript + const SUGGESTION_CATEGORY_ORDER = ['character', 'copyright', 'rating', 'general']; + const SUGGESTION_CATEGORY_LABEL = { + character: 'Characters', + copyright: 'Fandoms', + rating: 'Rating', + general: 'General', + }; +``` + +Replace with: + +```javascript + const SUGGESTION_CATEGORY_ORDER = ['character', 'copyright', 'general']; + const SUGGESTION_CATEGORY_LABEL = { + character: 'Characters', + copyright: 'Fandoms', + general: 'General', + }; +``` + +(If the exact labels in your file differ, keep the existing labels and only drop the `rating` entry + its position in the order array.) + +- [ ] **Step 2: Verify no other rating references remain in the JS** + +```bash +grep -n "rating" app/static/js/view-modal.js +``` + +Expected: no output. If any references remain (e.g. styling hooks), they should be removed too since the category will never arrive from the server after Task 4. + +- [ ] **Step 3: Commit** + +```bash +git add app/static/js/view-modal.js +git commit -m "feat(modal): drop rating group from suggestions UI" +``` + +--- + +## Task 7: Add Settings → Maintenance button for recompute-all-centroids + +**Files:** +- Modify: `app/templates/settings.html:440-449` + +- [ ] **Step 1: Add the new form next to the existing backfill form** + +Find: + +```html +
+

ML tag suggestions

+

+ Run inference on all images missing predictions or embeddings for the current model versions. + Resumable and safe to re-run. Backfill enqueues work on the ml queue; monitor progress in the ml-worker logs. +

+
+ +
+
+``` + +Replace the entire section with: + +```html +
+

ML tag suggestions

+

+ Run inference on all images missing predictions or embeddings for the current model versions. + Resumable and safe to re-run. Backfill enqueues work on the ml queue; monitor progress in the ml-worker logs. +

+
+ +
+ +

+ Recompute centroids for all eligible tags (character, fandom, and general/topic) that have at least + the minimum number of reference images. Run this after the initial backfill, or any time you've + applied many tags manually and want the suggestions to catch up. +

+
+ +
+
+``` + +- [ ] **Step 2: Commit** + +```bash +git add app/templates/settings.html +git commit -m "feat(settings): add Recompute all centroids maintenance button" +``` + +--- + +## Task 8: Rebuild web container and run smoke tests + +**Files:** +- None modified. This task validates the whole chain end-to-end. + +- [ ] **Step 1: Rebuild web + ml-worker** + +```bash +docker compose up -d --build web ml-worker +``` + +Expected: both containers rebuild and start successfully. + +- [ ] **Step 2: Verify the web container is up** + +```bash +docker compose logs web --tail 5 2>&1 | grep -iE "listening|ready|error" +``` + +Expected: a "listening" or similar line, no errors. If error: stop and fix. + +- [ ] **Step 3: Verify the ml-worker task listing includes the new task** + +```bash +docker compose logs ml-worker --tail 60 2>&1 | grep "recompute_all_centroids" +``` + +Expected: one line ` . app.tasks.ml.recompute_all_centroids`. + +- [ ] **Step 4: Smoke-test: load a gallery image modal, request suggestions** + +In the browser: +1. Open a gallery page, click any image to open the modal. +2. Hard refresh (Ctrl+Shift+R) to pick up the new JS. +3. Confirm the Suggestions section shows three groups max: Characters, Fandoms, General. **No Rating group should appear.** + +If Rating appears: check that Task 4 actually dropped it from the `grouped` dict, rebuild web. + +- [ ] **Step 5: Smoke-test: click "Recompute all centroids"** + +1. Navigate to Settings → Maintenance. +2. Click "Recompute all centroids", confirm the prompt. +3. Tail ml-worker logs: + +```bash +docker compose logs -f ml-worker +``` + +Expected: `recompute_all_centroids: enqueued N tags (min_refs=5)` where N is the count of distinct tags (character/fandom/null-kind) currently attached to at least 5 images. Followed by `recompute_centroid: (kind=...) -> n=M` lines. + +- [ ] **Step 6: Smoke-test: verify a fandom or general centroid gets written** + +```bash +docker compose exec -T postgres psql -U imagerepo -d imagerepo -tAc "SELECT tag_name, tag_kind, reference_count FROM tag_reference_embedding WHERE tag_kind IS NULL OR tag_kind = 'fandom' ORDER BY tag_kind, tag_name" +``` + +Expected: one row per fandom tag with ≥5 images, one row per null-kind (general/topic) tag with ≥5 images. Each row has a non-null `tag_kind` (either 'fandom', 'character', or NULL for general) and the expected reference count. + +- [ ] **Step 7: Smoke-test: open a new image modal and confirm a user general tag surfaces** + +1. Pick an image that is visually similar to ones you've tagged with a general/topic tag you've applied to 5+ images. +2. Open its modal, look at the Suggestions → General group. +3. Confirm the suggestion appears with `source=embedding_similarity` (visible via DOM inspection: `data-source="embedding_similarity"` on the chip). If it does, the full chain is working. + +If nothing appears: possible causes, in order of likelihood: +- No centroid row exists yet for that tag (re-run Step 5 or recompute that specific tag). +- Similarity below threshold (0.85); try a visually closer image or lower the `threshold_embedding` config row temporarily. +- Tag is already applied to this image (filtered out by `_existing_tag_names_for_image`). + +- [ ] **Step 8: Commit any fix-ups that surfaced during smoke tests** + +If the smoke tests uncovered issues, fix them in the relevant task's files and commit separately. If everything passed, there's nothing to commit for this task. + +--- + +## Self-Review Checklist (run before handing off) + +- Spec coverage: + - [x] Table/column/column rename + tag_kind add → Task 1 + - [x] Model class rename → Task 2 + - [x] `ELIGIBLE_CENTROID_KINDS` constant → Task 4 Step 1 + - [x] Kind-agnostic `recompute_centroid` + `ineligible_kind` guard → Task 3 Step 2 + - [x] Batch `recompute_all_centroids` task → Task 3 Step 3 + - [x] `_embedding_tag_suggestions` query-all-centroids + display-category mapping → Task 4 Step 3 + - [x] `get_suggestions` drops `rating` group → Task 4 Step 4 + - [x] `_category_threshold` drops rating → Task 4 Step 5 + - [x] `_WD14_CATEGORY_TO_KIND` drops rating → Task 5 Step 1 + - [x] Centroid-recompute trigger extended to all eligible kinds → Task 5 Step 2 + - [x] Settings maintenance route → Task 5 Step 3 + - [x] Modal JS drops rating group → Task 6 + - [x] Settings button → Task 7 + - [x] Config rename of `threshold_embedding_character` → `threshold_embedding` and `min_reference_images_per_character` → `min_reference_images` → Task 1 (migration) + Task 4 Step 2 (`_DEFAULTS` dict) + - [x] `threshold_rating` row dropped from config → Task 1 (migration) + - [x] End-to-end validation → Task 8 + +- Type/name consistency: `TagReferenceEmbedding.tag_name`, `TagReferenceEmbedding.tag_kind`, `ELIGIBLE_CENTROID_KINDS` — used consistently across Tasks 2–5. + +- Placeholders: none. All code blocks are complete. + +--- + +## Notes for the implementer + +- **Tests:** the base tag-suggestions feature has no automated tests, and the spec explicitly defers new tests. Validation is the Task 8 smoke-test script. If the implementer wants to add tests anyway, fine — but don't block commits on them. +- **Migration target:** `down_revision = '0017871d2221'`. If by the time you run this someone else has added migrations between then and now, rebase the `down_revision` onto whatever is `alembic current` says is head. +- **Frontend cache-busting:** hard-refresh the browser after rebuilding; otherwise the old JS with the rating group in `SUGGESTION_CATEGORY_ORDER` is still in memory. +- **Manual recompute after migration:** the Task 1 migration fills `tag_kind='character'` on existing rows, but does **not** create centroids for fandom/general tags that may have existed before this feature. Task 8 Step 5 (the "Recompute all centroids" button) handles this the first time the user clicks it. diff --git a/docs/superpowers/specs/2026-04-18-clickable-post-tag-design.md b/docs/superpowers/specs/2026-04-18-clickable-post-tag-design.md new file mode 100644 index 0000000..3cc7375 --- /dev/null +++ b/docs/superpowers/specs/2026-04-18-clickable-post-tag-design.md @@ -0,0 +1,100 @@ +# Clickable Post Tag in Gallery Modal + +**Status:** Approved, ready for implementation planning +**Date:** 2026-04-18 +**Scope:** Gallery/showcase image view modal (`_gallery_modal.html`) + +## Problem + +When viewing an image in the gallery/showcase modal, the user frequently wants to revisit the original post (Patreon/SubscribeStar/HentaiFoundry) the image came from, to re-read context. Today the post tag is rendered as a plain, non-clickable `` in the editable tag list — there is no path from the modal to the post. + +Secondary concern: the post tag is system-generated and shouldn't appear inside the editable tag area, where the `×` remove button invites deletion of metadata the user didn't author. + +## Goal + +Make the post tag in the image modal a clickable chip that navigates to the gallery filtered by that post tag. The filtered gallery view already surfaces the post's title, description, and source URL (`main.py:143-157`), so click-to-filter is the shortest path to the original post. + +Move the post tag out of the editable tag list into a dedicated "provenance" area so it reads as metadata, not a user tag. + +## Non-goals + +- Opening the external post URL directly from the modal. Navigation goes to the filtered gallery, which is where `source_url` is already presented. +- Making `source` (🌐) or `archive` (🗜️) tags clickable in the modal. Will reconsider after the post-tag treatment is validated. +- Changes to `tag-editor.js` / the tags-list page's edit modal. Different modal. +- New backend endpoints. + +## Design + +### Placement + +A new section `modal-provenance-section` is inserted into the modal sidebar **between** the existing Series section and Tags section. + +- Section name is forward-looking: "provenance" accommodates source/archive later without renaming. +- The section is rendered only when the current image has at least one post tag. When empty, it is fully hidden — no placeholder, no heading. + +### Chip content + +One chip per post tag (in practice zero or one). Chip structure: + +- 📌 icon prefix +- Display label — see rules below +- Trailing `↗` affordance to signal the chip navigates away from the modal + +**Display label rules.** `getTagDisplayName` today only strips prefixes for `artist|character|series|fandom|rating`; post tags fall through and yield `platform:artist:id`, which is ugly. Label resolution for the provenance chip: + +1. **Before metadata resolves:** show the raw tag name split once on `:` (the `platform:artist:id` portion). Ugly but functional placeholder. +2. **After metadata resolves:** prefer `pm.artist`; if absent, fall back to `pm.title`; if both absent, keep the placeholder. + +The upgrade from placeholder to metadata-driven label happens in the same render pass that applies the platform accent color. + +### Chip behavior + +- Click target: the whole chip, as an `` +- Navigation: same tab (consistent with other tag chips in the app) +- No `×` remove button — system tag, not user-editable + +### Chip styling — platform-tinted ghost chip + +Visually distinct from the standard editable tag chip (which is light-on-light with a `×` on hover). The provenance chip is a ghost chip with a platform-colored accent. + +- **Base:** transparent background, 1px border `rgba(255, 255, 255, 0.15)`, light text +- **Platform accent:** a 2px colored left-border driven by `PostMetadata.platform`: + - `patreon` → Patreon orange (`#f96854`) + - `subscribestar` → teal (`#009cde`) + - `hentaifoundry` → red (`#c8232c`) + - Unknown / missing metadata → neutral fallback, `rgba(255, 255, 255, 0.4)` +- **Hover:** border brightens to `rgba(255, 255, 255, 0.35)`, background `rgba(255, 255, 255, 0.04)` +- Platform color is applied via a `data-platform=""` attribute on the chip plus CSS attribute selectors — no inline styles. + +### Data flow + +1. `view-modal.js` already calls `loadTags(imageId)` when the modal opens and when navigating between images. +2. Extend the tag load: partition the returned tags into + - `postTags` — `tag.kind === 'post'` + - `editableTags` — everything else +3. Render `editableTags` into the existing `#modalTagList` container (unchanged behavior, minus the post chip). +4. Render `postTags` into a new `#modalProvenanceList` container inside `modal-provenance-section`. Show the section iff `postTags.length > 0`; hide it otherwise. +5. For each post tag, resolve its platform by calling the existing endpoint `/api/post-metadata/by-tag-name/`, which returns `{ platform, source_url, ... }`. The chip renders immediately with the neutral fallback accent, then upgrades to the platform-tinted accent when metadata resolves. This keeps the modal responsive even on a slow metadata fetch. +6. Clean up provenance rendering on modal close and on image navigation, the same way the tag list is cleaned up today. + +### Components to touch + +- `app/templates/_gallery_modal.html` — add the `modal-provenance-section` with a `#modalProvenanceList` container between the Series and Tags sections. +- `app/static/js/view-modal.js` — partition tags on load, render provenance chips, fetch platform metadata per post tag, handle show/hide of the section, reset on close/navigate. +- `app/static/style.css` — `.modal-provenance-section`, `.provenance-chip`, platform-specific `[data-platform="..."]` accent rules, hover state. + +### Error handling + +- Metadata endpoint fails, returns 404, or returns no metadata record → chip renders with neutral fallback accent. The chip remains clickable; the filter-by-tag navigation works without platform data. +- Post tag exists but its name is malformed → chip still renders with `getTagDisplayName` output and fallback accent; click still navigates to `/gallery?tag=`. + +## Risks and considerations + +- Tag-kind drift: if another part of the codebase ever assumes every tag in the modal is user-editable (has a `×` button), splitting the list could surprise it. Mitigation: grep for `#modalTagList` consumers and confirm only view-modal.js reads it. +- Duplicate clicks during metadata fetch: chip is clickable before metadata arrives — fine, because click navigation doesn't depend on metadata. +- Future extension to source/archive: section and class names are generic (`provenance`), so adding them later is additive and doesn't require renames. + +## Future work (deferred) + +- Consider moving `source` (🌐) and `archive` (🗜️) tags into the same provenance section once the post-tag treatment is polished. +- Consider a direct "open original post" action (using `source_url`) as a secondary affordance — only if click-to-filter proves insufficient. diff --git a/docs/superpowers/specs/2026-04-18-tag-suggestions-integration.md b/docs/superpowers/specs/2026-04-18-tag-suggestions-integration.md new file mode 100644 index 0000000..b3658c8 --- /dev/null +++ b/docs/superpowers/specs/2026-04-18-tag-suggestions-integration.md @@ -0,0 +1,252 @@ +# Tag Suggestions — ImageRepo Integration Spec + +**Parent spec:** `docs/superpowers/tag-suggestion-system-spec.md` +**Status:** Design approved 2026-04-18. Ready for implementation planning. +**Scope this phase:** Scan pipeline + modal suggestion UI. Multi-edit / gallery bulk suggestions deferred. + +This document layers ImageRepo-specific integration decisions on top of the parent spec. Read the parent spec first for model choice, schema rationale, threshold config, and model versioning; this spec assumes all of that is still in effect. + +--- + +## 1. System layout + +A new Docker Swarm service joins the existing stack: + +- `ml-worker` — Celery worker on a new `ml` queue. Shares the existing Redis broker and Postgres database. Runs **CPU inference only** (Swarm does not expose GPUs cleanly and this is a single-user system; per-upload latency is async, backfill runs once overnight). + +Existing services are unchanged. The `worker` container stays on its current queues (`import`, `thumbnail`, `sidecar`, `default`); the `scheduler` keeps `maintenance` and `scan`. + +### Image + +`ml-worker` uses its own Dockerfile (`Dockerfile.ml` or similar) so the main `worker` image stays lean: + +- Base: `python:3.12-slim` +- System deps: `libgl1`, `libglib2.0-0` (for Pillow / OpenCV if used for preprocessing) +- Python deps (new `requirements-ml.txt`): `onnxruntime` (CPU build), `numpy`, `Pillow`, `huggingface_hub` (to pull models at build time), plus everything in the base `requirements.txt` that the Celery app imports +- Models baked into the image at build time under `/models/` (WD14 ONNX + SigLIP-SO400M ONNX). Fetching at build time avoids cold-start latency and network dependency at runtime. Image size will be ~2-3 GB; acceptable for a persistent service. + +### Resource sizing + +- CPU: at least 4 cores pinned via `deploy.resources.limits.cpus` +- RAM: 6-8 GB (ONNX graph + model weights + image batch preprocessing headroom) +- Celery concurrency: **1**. ONNX Runtime will use multiple CPU threads internally via `intra_op_num_threads`; multi-worker concurrency just fights for the same cores. + +--- + +## 2. Database + +Follows the parent spec's schema, adapted to ImageRepo conventions: + +- Primary keys use `Integer`, not `BIGSERIAL` (consistent with `ImageRecord.id`, `Tag.id`). +- FKs target the actual ImageRepo table names: `image_record.id`, `tag.id`. The parent spec's `images(id)` references become `image_record(id)`. +- Table names stay singular per ImageRepo convention: `image_tag_prediction`, `image_embedding`, `character_reference_embedding`, `suggestion_feedback`, `tag_suggestion_config`. +- `tag_suggestion_config` could be folded into the existing `AppSettings` key/value table, but the parent spec's dedicated table keeps threshold tuning separate from general settings and makes the thresholds discoverable. Stick with the dedicated table. + +### Alembic migration + +A new revision in `migrations/versions/` does: + +1. `CREATE EXTENSION IF NOT EXISTS vector;` +2. Create the five new tables with SQLAlchemy-`pgvector` column types. +3. Create the HNSW index on `image_embedding.embedding` using `vector_cosine_ops`. +4. Seed `tag_suggestion_config` with the initial threshold rows from the parent spec. + +The `pgvector` Python package must be added to the base `requirements.txt` (not just ml-worker's) so the web process can read embeddings for suggestion generation. + +### Embedding dimension + +Locked at **1152** (SigLIP-SO400M). If the model is swapped later, a new `model_version` coexists in the same tables per the parent spec; the schema does not change. + +--- + +## 3. Task flow + +New Celery module `app/tasks/ml.py`, tasks routed to the `ml` queue. + +### Per-upload: chained from the import pipeline + +`app.tasks.import_file.import_media_file` currently ends by committing an `ImageRecord` row. After commit, it enqueues: + +```python +from app.tasks.ml import tag_and_embed +tag_and_embed.apply_async(args=[image_id], queue='ml') +``` + +`tag_and_embed(image_id)`: +1. Load the image from disk (or from the path stored on `ImageRecord.filepath`) +2. Run WD14 preprocessing → WD14 inference → write rows into `image_tag_prediction` +3. Run SigLIP preprocessing → SigLIP inference → write row into `image_embedding` +4. Commit both in one DB transaction + +If either model fails the task raises and Celery will retry per its default policy. A failure does not block the upload — the image is already in the DB and visible in the gallery; suggestions simply will not appear for it until the task succeeds. + +### Backfill: resumable batch job + +`app.tasks.ml.backfill(batch_size=50)`: + +1. Query images missing predictions **or** embeddings for the current model versions +2. For each, enqueue `tag_and_embed.apply_async(args=[image_id], queue='ml')` +3. Process in batches so the enqueue side doesn't dump 62k messages into Redis at once — sleep/yield between batches +4. Track progress via a row in the existing `ImportBatch` / `Task` pattern (whichever is conventional for long-running scan jobs — follow `scan_directory` for precedent) + +Resumability comes from the query itself: it only enqueues images without current rows. Re-running after a crash picks up exactly where it left off. + +An admin trigger for `backfill` is exposed via a new button on the Settings → Maintenance tab (the tab already exists from the UI/UX redesign). No automated scheduling — the user kicks it off manually once after deploy, and again after a model version change. + +### On accept: centroid recompute + +When a user accepts a `character`-kind suggestion on an image: + +1. Apply the tag as normal (existing tag-add endpoint flow) +2. Write a `suggestion_feedback` row with `decision='accepted'` +3. Check `character_reference_embedding.reference_count` for this tag. If the current count of confirmed images minus `reference_count` at last computation exceeds `centroid_recompute_delta` (default 3), enqueue `recompute_centroid.apply_async(args=[character_tag_name], queue='ml')` + +`recompute_centroid(tag_name)` fetches all embeddings for images that currently carry that character tag, computes the mean vector, upserts `character_reference_embedding` with the new centroid and the current reference count. + +This runs on the `ml` queue because it touches the pgvector tables; it does **not** need the ONNX models, so it completes in milliseconds. + +### On reject: feedback only + +Rejecting a suggestion writes a `suggestion_feedback` row with `decision='rejected'`. No other side effect. The rejected tag will reappear on the next modal open — intentional, per the parent spec's rationale that feedback is informational and we should not lose tags the user might later want. + +--- + +## 4. Web integration + +### Flask endpoints + +All under the existing image blueprint. Permissions inherit from however the modal's existing `/image//tags` endpoints gate access. + +**`GET /image//suggestions`** + +Returns the merged suggestion list grouped by category. Body: + +```json +{ + "ok": true, + "suggestions": { + "character": [ + {"name": "", "confidence": 0.91, "source": "embedding_similarity", "exists_in_db": true} + ], + "copyright": [ ... ], + "rating": [ ... ], + "general": [ ... ] + } +} +``` + +Suggestion generation logic lives in a new `app/services/tag_suggestions.py` module and is invoked by the route. It: + +1. Fetches this image's `image_tag_prediction` rows for the current WD14 model version, filters by per-category thresholds from `tag_suggestion_config`, drops tags already applied to the image. +2. Fetches this image's `image_embedding` for the current SigLIP model version, computes cosine similarity against every `character_reference_embedding` whose `reference_count >= min_reference_images_per_character`, keeps matches above `threshold_embedding_character`, drops characters already applied. +3. Merges: WD14-character and embedding-character lists dedupe by tag name; keep the higher normalized confidence and record both sources if desired (for now, single `source` is fine). +4. Annotates each suggestion with `exists_in_db: true|false` by looking up each name in the `tag` table. +5. Groups by category, sorts by confidence descending within each group. + +Categories returned in the response map 1:1 to WD14's categories. `meta` is omitted from the modal for now (noisy, low value). + +**`POST /image//suggestions/accept`** + +Body: `{"tag_name": "...", "source": "wd14"|"embedding_similarity", "category": "general"|"character"|"copyright"|"rating"}`. + +Server applies the hybrid vocabulary policy: + +- `character`, `copyright`, `rating` → if the tag doesn't exist in the `tag` table, create it with the appropriate `kind`. Then associate with the image. Kind mapping: + - WD14 `character` → `kind='character'` + - WD14 `copyright` → `kind='fandom'` (ImageRepo's IP/franchise kind — series is reserved for sequential readers) + - WD14 `rating` → `kind='rating'` +- `general` → if the tag exists, associate with the image. If not, return `{"ok": false, "error": "unknown_general_tag"}` (the UI renders this branch as a disabled chip in advance; this error is the safety net for race conditions). + +After applying, writes `suggestion_feedback` with `decision='accepted'` and triggers centroid recompute if the accepted tag is a character. + +**`POST /image//suggestions/reject`** + +Body: `{"tag_name": "...", "source": "..."}`. Writes `suggestion_feedback` with `decision='rejected'`. Returns `{"ok": true}`. + +### Modal UI + +New section in the gallery modal sidebar (`app/templates/_gallery_modal.html`) **below** the existing Tags section, with id `modalSuggestionsSection` and a `display: none` inline style that JavaScript toggles (matches the project's inline-style-over-class convention per commit `e37ce27`). + +Structure: + +```html + +``` + +Frontend logic goes in `app/static/js/view-modal.js` next to the existing `loadTags` pattern. A new `loadSuggestions(imageId)` fetches `/image//suggestions`, renders a flat DOM structure with subtle category-heading dividers between the groups, and wires up the accept / reject click handlers. + +**Chip rendering:** + +- Layout: `[+ label · XX%]` with a hover-revealed `✕` on the right. +- Accept click: disable chip, `POST .../accept`, on success animate-out and call the existing `refreshItemTags(imageId)` hook so the gallery thumbnail overlay updates. Also append the new tag to the editable tag list in the modal. +- Reject click: disable chip, `POST .../reject`, animate-out. No other side effect. +- `exists_in_db: false` and policy says general-must-exist: render the chip dimmed, accept button disabled, tooltip "create this tag first in Tags admin." +- Platform-style accents: not needed here. Keep chips visually distinct from the editable tags (maybe a dashed border or slightly lower opacity) so users don't confuse suggestions with applied tags. + +**Feedback:** + +Reuse the `tagActionFeedback` element the modal already owns for "applied" / "rejected" / error messages. No toast system. + +**Lifecycle:** + +- Open modal → `loadSuggestions(imageId)` called alongside `loadTags` / `renderProvenance`. +- Close modal → clear the suggestions list (the existing `closeModal` hook). +- Arrow-key navigation between images re-runs `loadSuggestions` per image. + +--- + +## 5. Tag vocabulary policy (hybrid) + +Confirmed from brainstorming: + +- **Character / copyright / rating** suggestions → auto-create the tag on accept if it doesn't exist in the DB. The `tag.kind` is set as follows: + - WD14 `character` → `kind='character'` + - WD14 `copyright` → `kind='fandom'` (ImageRepo's IP/franchise kind — `series` is reserved for sequential readers, not franchises) + - WD14 `rating` → `kind='rating'` +- **General** suggestions → filter to tags that already exist in the DB. Unknown general tags are rendered dimmed (disabled) in the UI; the accept endpoint rejects them as a safety net. +- **Embedding-similarity** suggestions are always characters; they suggest only characters that already have a reference set, so vocabulary never grows via that path. + +--- + +## 6. Scope boundaries + +### In scope this phase + +- `ml-worker` Docker service + image + requirements +- Alembic migration for pgvector extension, five tables, HNSW index, seed config rows +- SQLAlchemy models for all five tables (in `app/models.py` or a dedicated `app/models_ml.py` — follow whichever the codebase prefers for locality) +- `app/tasks/ml.py` with `tag_and_embed`, `backfill`, `recompute_centroid` +- Hook into `import_media_file` to chain `tag_and_embed` after commit +- `app/services/tag_suggestions.py` suggestion-generation module +- Three new Flask endpoints under the existing image blueprint +- Modal UI additions: template block, CSS, `view-modal.js` suggestion logic +- Settings → Maintenance tab: "Run ML backfill" button +- Threshold values read from `tag_suggestion_config` (DB-editable via psql for now) + +### Deferred + +- Bulk-edit gallery multi-select suggestions (explicit next phase) +- Admin UI for editing `tag_suggestion_config` thresholds +- Admin panel showing character reference-count stats +- Reranker / feedback-driven threshold tuning +- Automatic backfill scheduling (manual trigger only for now) +- Multi-centroid-per-character for characters with high intra-class variance (see parent spec §"Why centroids instead of querying all reference images") + +--- + +## 7. Verification + +Per the ImageRepo project norm, there is no automated test harness. Verification at implementation time is a manual browser check: + +- Upload a new image; confirm `image_tag_prediction` and `image_embedding` rows appear after a few seconds. +- Open the modal on an image that has predictions; confirm suggestions appear grouped by category with plausible confidences. +- Accept a character suggestion; confirm the tag is applied, a `suggestion_feedback` row is written, and (if count delta is met) a `recompute_centroid` task runs and `character_reference_embedding` is updated. +- Reject a suggestion; confirm a feedback row is written and the suggestion reappears on modal reopen. +- Kick off a backfill from Settings → Maintenance; confirm it processes images in batches, is resumable after a restart, and does not stall the ml-worker. +- Accept a general tag that doesn't exist in the DB (forcing the disabled-chip path); confirm the UI prevents submission and the endpoint returns `unknown_general_tag` if attempted. diff --git a/docs/superpowers/specs/2026-04-19-character-tag-integrity-design.md b/docs/superpowers/specs/2026-04-19-character-tag-integrity-design.md new file mode 100644 index 0000000..a699dc2 --- /dev/null +++ b/docs/superpowers/specs/2026-04-19-character-tag-integrity-design.md @@ -0,0 +1,177 @@ +# Character-Tag Integrity Design + +**Date:** 2026-04-19 +**Branch:** `feat/tag-suggestions` (same branch as the tag-suggestion feature; this is a separate concern but bundles naturally). + +## Problem + +Two related defects erode the integrity of the `Tag` table's character/fandom data: + +1. **Stale image↔fandom associations.** When a character tag gains a `fandom_id` via `POST /api/tag//set-fandom` (or via a rename that introduces `(Fandom)`), existing images already tagged with that character do not receive the fandom tag. Only *future* attachments through `add_tag` / `add_bulk_tag` auto-apply the fandom, because those paths explicitly append it. The result: images tagged before the association is set miss the fandom and won't show up under the fandom in filters or suggestions. + +2. **Inconsistent casing on character/fandom display names.** The user (solo dev) introduces typos like `character:misty` alongside `character:Misty`. There is no normalization on create or rename, so the DB accumulates duplicates that diverge by case. + +## Goals + +- Every image tagged with a character whose tag has a `fandom_id` also has that fandom tag attached. +- Character and fandom tag display names always start each whitespace-separated word with an upper-case letter. +- Existing lowercase names are renamed one-time on migration, merging into any already-correctly-cased tag of the same kind. + +## Non-Goals + +- Subtractive reconciliation (removing fandom tags from images when the character's fandom changes or is cleared). The DB does not record *why* a fandom was attached, so removal risks deleting user-intended tags. +- Smart handling of apostrophes, hyphens, or Roman numerals in names. Users who need `"O'Brien"` or `"Mary-Kate"` can still type the exact casing via `set-fandom`; the normalizer only guarantees first-letter-of-each-word. +- Normalization for other kinds (`artist`, `series`, `user`, `post`, `rating`). Scope stays on the two kinds the user asked about. +- Retroactive removal of image↔fandom links. Only additive. + +## Architecture + +### Normalization helper + +New module `app/utils/tag_names.py`: + +```python +def normalize_display_name(s: str) -> str: + """Title-case each whitespace-separated word. Preserves internal punctuation. + + Examples: + "misty" -> "Misty" + "misty ketchum" -> "Misty Ketchum" + "o'brien" -> "O'brien" + "mary-kate" -> "Mary-kate" + """ + return ' '.join(w[:1].upper() + w[1:] for w in s.split(' ') if w != '') \ + if s else s +``` + +Called from every character/fandom write path: + +- `add_tag` (`/image//tags/add`) — after parsing `(Fandom)` suffix, normalize both `char_name` and `fandom_name` before storing. +- `add_bulk_tag` (in `app/main.py` around line 2240) — same. +- `set_tag_fandom` (`/api/tag//set-fandom`) — normalize `fandom_name` when the `fandom_name` path is taken, normalize the character display base before reassembling the tag name. +- `accept_image_suggestion` (WD14 accept path in `app/main.py`) — when auto-creating a new character or fandom tag from a WD14 suggestion, normalize first. +- `_ensure_fandom_tag` (helper in `app/main.py`) — normalize internally so every caller benefits. +- `_parse_character_fandom` (helper in `app/main.py`) — unchanged; parsing is orthogonal to normalization. + +### Retroactive fandom backfill on `set_tag_fandom` + +After the existing commit that renames the character tag and sets `fandom_id`, the endpoint runs a second transaction that attaches the fandom tag to every image already tagged with the character. The operation is a single SQL statement on the `image_tags` association table: + +```sql +INSERT INTO image_tags (image_id, tag_id) +SELECT image_id, :fandom_tag_id +FROM image_tags +WHERE tag_id = :character_tag_id + AND NOT EXISTS ( + SELECT 1 FROM image_tags it2 + WHERE it2.image_id = image_tags.image_id + AND it2.tag_id = :fandom_tag_id + ); +``` + +Running it in a second transaction after the rename commits means a failed backfill never rolls back the rename. A failed backfill leaves stale state that the sweep button will correct. + +### "Sync character fandoms to images" sweep + +New Celery task `sync_character_fandoms` in `app/tasks/maintenance.py` (new module) on the default queue: + +```python +@celery.task(name='app.tasks.maintenance.sync_character_fandoms', + soft_time_limit=120, time_limit=180) +def sync_character_fandoms(): + """For every character tag with fandom_id, attach fandom to images + that have the character but not the fandom.""" + with app.app_context(): + total_added = 0 + chars = Tag.query.filter_by(kind='character').filter(Tag.fandom_id.isnot(None)).all() + for char in chars: + inserted = _attach_fandom_to_images(char.id, char.fandom_id) + total_added += inserted + db.session.commit() + log.info("sync_character_fandoms: added %d image↔fandom links across %d characters", + total_added, len(chars)) + return {'characters_scanned': len(chars), 'links_added': total_added} +``` + +New route `POST /settings/maintenance/sync-character-fandoms` that calls `sync_character_fandoms.apply_async()` (default queue, no `queue='ml'`). + +Settings → Maintenance gets a third button "Sync character fandoms to images". + +### One-time migration: rename + merge + +New Alembic migration `h26041901_normalize_character_fandom_tag_names.py` (revision id follows the existing date-coded pattern; adjust `down_revision` to whatever `alembic current` is at deploy time). + +Upgrade algorithm, entirely in raw SQL within a single transaction: + +1. Build a working set: `SELECT id, name, kind, fandom_id FROM tag WHERE kind IN ('character', 'fandom')`. +2. For each row, compute `normalized_name` by: + - Splitting the name at the first `:` into `prefix` + `rest`. + - For character tags: further split `rest` at ` (` to separate `char_base` from `fandom_part)`. + - Normalize `char_base`. + - If a fandom part exists, normalize it too and reassemble `{norm_char} ({norm_fandom})`. + - For fandom tags: normalize `rest` directly. + - Reassemble `{prefix}:{normalized_rest}`. +3. For each `(id, old_name, new_name)` where `old_name != new_name`: + - Check for a target row with `name = new_name AND kind = current_row.kind AND id != current_row.id`. + - **No collision:** `UPDATE tag SET name = :new_name WHERE id = :id`. + - **Collision (merge):** + - Winner: the existing correctly-cased target (the one the current row would collide with). + - Loser: the row being renamed. + - `INSERT INTO image_tags(image_id, tag_id) SELECT DISTINCT image_id, :winner_id FROM image_tags WHERE tag_id = :loser_id ON CONFLICT DO NOTHING`. + - `DELETE FROM image_tags WHERE tag_id = :loser_id`. + - `UPDATE tag SET fandom_id = :winner_id WHERE fandom_id = :loser_id`. + - `UPDATE tag_reference_embedding SET tag_name = :winner_name WHERE tag_name = :loser_name AND NOT EXISTS (SELECT 1 FROM tag_reference_embedding WHERE tag_name = :winner_name AND model_version = tag_reference_embedding.model_version)` — keeps winner's centroid if both have one. + - `DELETE FROM tag_reference_embedding WHERE tag_name = :loser_name`. + - `DELETE FROM tag WHERE id = :loser_id`. +4. Log counts of renames vs merges to migration output. + +Downgrade: no-op for merges (destructive), plain name-revert for pure renames is possible but not worth the bookkeeping. The downgrade function raises `NotImplementedError` with an explanatory message. + +## Data flow — summary + +``` +Write paths (add_tag, add_bulk_tag, accept_suggestion, set_tag_fandom) + → normalize_display_name(…) + → store {prefix}:{Normalized Name} + → attach to image (and auto-attach fandom if character has one) + +set_tag_fandom (specifically) + → normalize + rename + set fandom_id (commit #1) + → SELECT image_ids tagged with this character + → INSERT (image_id, fandom_tag_id) for rows missing the fandom (commit #2) + → return JSON + +Sweep button + → enqueue sync_character_fandoms on default queue + → iterate character tags with fandom_id + → per-tag: batch-insert missing links, commit per tag + → log summary + +Migration (one-time) + → enumerate character:* and fandom:* tags + → compute normalized names + → rename survivors; merge collisions (reassign image_tags, fandom_id, tag_reference_embedding; delete loser) + → one transaction; no downgrade for merges +``` + +## Error handling + +- **Normalization of empty or whitespace-only string:** returns the input unchanged; callers already strip and reject empty via `abort(400)`. +- **`set_tag_fandom` rename collision with differently-cased target:** post-normalization, both names converge, so the 409 check at `app/main.py:940` fires correctly. No behavior change. +- **`set_tag_fandom` backfill DB error:** rename commits first (commit #1); backfill fails (commit #2 rolled back). Response still returns 200 with the renamed tag. Sweep button later recovers. +- **Sweep task per-tag error:** try/except around each tag; log `sync_character_fandoms: tag %s failed: %s`, continue, return partial summary. +- **Migration mid-flight error:** full rollback. User investigates via migration log. + +## Testing plan + +Manual verification on dev DB: + +1. **Migration:** dev DB has `character:emelie`. Run migration → verify tag is renamed to `character:Emelie`, row count in `tag` unchanged, all `image_tags` links preserved. +2. **Create with normalization:** via add-tag, submit `character:misty (pokemon)`. Verify tag lands as `character:Misty (Pokemon)`; fandom tag `fandom:Pokemon` is auto-created; both attached to the image. +3. **Retroactive backfill on `set-fandom`:** pick an existing character with no fandom, attach it to 3 images. Call `set-fandom` with a new fandom_name. Verify all 3 images now have the fandom tag (even though `set-fandom` was the trigger). +4. **Merge on migration:** manually seed `character:foo` and `character:Foo` with overlapping and distinct image links. Run migration. Verify one `character:Foo` survives with the union of image links; `character:foo` row is gone; `tag_reference_embedding` has one row (if either did); `fandom_id` references pointing at either row now point at the survivor. +5. **Sweep button:** manually remove a fandom link from an image whose character tag has a fandom (simulating drift). Click "Sync character fandoms to images". Verify the fandom is re-attached and the task log shows `links_added: 1`. + +## Open items + +None. Design confirmed via brainstorming: additive-only backfill, title-case normalization via simple first-letter-per-word rule, covers character + fandom kinds, one-time migration with merge, sweep available as manual maintenance action. diff --git a/docs/superpowers/specs/2026-04-19-tag-underscores-and-modal-polish-design.md b/docs/superpowers/specs/2026-04-19-tag-underscores-and-modal-polish-design.md new file mode 100644 index 0000000..ae1ba01 --- /dev/null +++ b/docs/superpowers/specs/2026-04-19-tag-underscores-and-modal-polish-design.md @@ -0,0 +1,269 @@ +## Tag Underscores + Modal Polish Design + +**Date:** 2026-04-19 +**Branch:** `feat/tag-suggestions` +**Builds on:** `2026-04-19-character-tag-integrity-design.md` (same branch, same normalizer module) + +## Problem + +Three related pain points around WD14-sourced suggestions and the suggestion modal: + +1. **WD14 uses Danbooru-style underscored names.** Raw WD14 tags surface as `big_hair`, `long_hair`, `blue_sky`. The user's library uses spaces (`big hair`). Accepting a WD14 suggestion today writes the underscored form into the `Tag` table and attaches it to the image, so the library diverges from the user's naming convention. +2. **Accepting a suggestion refreshes the attached-tags list later and more loosely than the manual "Add" button.** Manual add awaits `loadTags(imageId)` before showing feedback; accept fires `loadTags` un-awaited inside a 200 ms animation callback. The visible result: the modal's attached-tags list looks briefly stale after accept, and reject/accept ordering can race. +3. **Accept (✓) and reject (✕) buttons have no persistent color cue.** Green/red only appear on hover, so the buttons read as decorative glyphs until the user hovers. + +## Goals + +- WD14 suggestion chips display and persist as space-separated names (`big hair`, not `big_hair`), regardless of kind. +- Character and fandom names continue to be title-cased on write (`character:Misty`); general (null) names preserve the user's casing but gain underscore→space. +- A one-time migration scrubs any already-persisted underscored names in the `Tag` table, merging into any existing space-separated duplicate. +- Accepting a suggestion refreshes the modal's attached-tags list with the same timing semantics as manual add — the data refresh is awaited and not gated by animation. +- Accept and reject buttons are persistently green and red, not only on hover. + +## Non-Goals + +- Changing the WD14 model, thresholds, or suggestion logic. The transformation is cosmetic/naming. +- Touching kinds beyond character, fandom, and null (general). Artist, post, series, rating, user, source, archive keep their current behavior. +- Stripping other Danbooru-style punctuation (parentheses, backslashes). Underscore→space only. +- Changing the reject endpoint behavior. It already records feedback without creating a tag; nothing to adjust on the backend. +- Redesigning the chip layout. Only the button colors change. + +## Architecture + +### Normalization helpers + +`app/utils/tag_names.py` grows two additions: + +```python +def underscores_to_spaces(s: str) -> str: + """Replace underscores with spaces. Used for general/null-kind names and + as a pre-step inside normalize_display_name for character/fandom.""" + return s.replace('_', ' ') if s else s + + +def normalize_display_name(s: str) -> str: + """Title-case each whitespace-separated word. Underscores become spaces + first so 'misty_ketchum' and 'Misty Ketchum' collapse onto the same name.""" + if not s: + return s + s = underscores_to_spaces(s) + parts = s.split(' ') + out: list[str] = [] + for p in parts: + if p == '': + out.append(p) + else: + out.append(p[:1].upper() + p[1:]) + return ' '.join(out) +``` + +No new helper for "general" — write paths call `underscores_to_spaces` directly when handling null-kind. That keeps the module small (one concept per helper) and avoids a second name that does nothing more than call `.replace`. + +### Write-path integration + +#### `accept_image_suggestion` (`app/main.py:723`) + +Accept is called only for suggestion categories `character`, `copyright`, and `general`. Kind mapping: `character → character`, `copyright → fandom`, `general → None`. Change: + +- Before the `Tag.query.filter_by(name=tag_name)` lookup, canonicalize based on category: + - `character` / `copyright`: split `prefix:rest`, apply `normalize_display_name` to `rest`, reassemble. + - `general`: apply `underscores_to_spaces` to the whole name (no prefix expected). +- Use the canonicalized `tag_name` for both the existence check and the new `Tag(...)` row. The existing character/fandom normalization block inside the "if tag is None" branch becomes redundant and is removed. + +The net effect: `fandom:trigun_stampede` → `fandom:Trigun Stampede`; `big_hair` → `big hair`; `character:misty_ketchum` → `character:Misty Ketchum`. Accepting a WD14 chip produces the same Tag row the user would get by typing the clean name manually. + +#### `add_tag` (`app/main.py:823`) and `bulk_add_tag` (`app/main.py:2279`) + +These already call `normalize_display_name` on character/fandom. With `normalize_display_name` now stripping underscores internally, character and fandom paths are already covered. Add: + +- For the no-prefix / `kind == "user"` path: `tag_name = underscores_to_spaces(tag_name)` before the Tag lookup. + +Kinds that are **not** transformed: `artist`, `post`, `source`, `archive`, `series`, `rating`. Artists sometimes have `_` in their canonical identifier (e.g., `redjuice_redjuice`); post tags use `:` as the separator and may contain underscored platform IDs; source/archive/series are path- or identifier-like where underscores may be meaningful. Those paths pass their raw name through to the DB unchanged, matching current behavior. + +Rule of thumb: the transform applies to exactly three kinds — `character`, `fandom`, and null (general) — the same set the migration touches. + +### Suggestion-service transform + +`app/services/tag_suggestions.py` — WD14 names are emitted to the modal still carrying underscores. Two choices: + +- **Transform at emit time** (chosen): after `_wd14_suggestions` builds a row, rewrite `p.tag_name` using the same rule as the accept path. The chip displays `big hair` and the accept endpoint receives `big hair` in the POST body. +- ~~Transform at paint time (in `view-modal.js`)~~ rejected — it would mean the display name and the POST body diverge, forcing the accept endpoint to re-derive the canonical form. + +Implementation: + +```python +from app.utils.tag_names import normalize_display_name, underscores_to_spaces + +_WD14_CATEGORY_TO_KIND = { + 'character': 'character', + 'copyright': 'fandom', +} + +def _canonicalize_wd14_name(raw: str, category: str) -> str: + kind = _WD14_CATEGORY_TO_KIND.get(category) + if ':' in raw: + prefix, rest = raw.split(':', 1) + transform = normalize_display_name if kind in ('character', 'fandom') else underscores_to_spaces + return f"{prefix}:{transform(rest)}" + return normalize_display_name(raw) if kind in ('character', 'fandom') else underscores_to_spaces(raw) +``` + +The existing `already` set (tags already attached to the image) is also canonicalized — `{underscores_to_spaces(n) for n in already}` — so that an already-attached `big hair` correctly shadows a WD14 `big_hair` suggestion. + +`_WD14_CATEGORY_TO_KIND` moves from `app/main.py` to the service module to avoid circular imports; `main.py` imports from the service. + +### Migration `i26041901_normalize_underscores_and_general_tags.py` + +One-time rename-or-merge covering every existing row in `tag` where the name contains `_` in the portion after the first `:` (or the whole name if there is no `:`). The down_revision is `h26041901` (the previous character/fandom normalization migration). + +Algorithm (raw SQL, single transaction): + +1. `SELECT id, name, kind, fandom_id FROM tag WHERE kind IN ('character', 'fandom') OR kind IS NULL` — widen the net to any kind that the write-paths now transform. +2. For each row, compute `new_name`: + - Split on first `:` into `prefix`, `rest` (or `rest = name` if no prefix). + - For character: split `rest` at ` (` to isolate `char_base` and `fandom_part)`; run `normalize_display_name` on each; reassemble. + - For fandom: `normalize_display_name(rest)`. + - For null (general): `underscores_to_spaces(rest)`. + - Reassemble with prefix (character/fandom always have prefixes; general usually has none). +3. For each `(id, old_name, new_name)` where they differ: + - Look for a `tag` row with `name = new_name AND kind = current.kind AND id != current.id`. + - **No collision:** `UPDATE tag SET name = :new_name WHERE id = :id`. + - **Collision (merge):** winner = existing correctly-named row, loser = current row. + - `INSERT INTO image_tags(image_id, tag_id) SELECT DISTINCT image_id, :winner_id FROM image_tags WHERE tag_id = :loser_id ON CONFLICT DO NOTHING` + - `DELETE FROM image_tags WHERE tag_id = :loser_id` + - `UPDATE tag SET fandom_id = :winner_id WHERE fandom_id = :loser_id` + - `UPDATE tag_reference_embedding SET tag_name = :winner_name WHERE tag_name = :loser_name AND NOT EXISTS (SELECT 1 FROM tag_reference_embedding t2 WHERE t2.tag_name = :winner_name AND t2.model_version = tag_reference_embedding.model_version)` + - `DELETE FROM tag_reference_embedding WHERE tag_name = :loser_name` + - `DELETE FROM tag WHERE id = :loser_id` +4. Log rename count and merge count. + +Downgrade: raises `NotImplementedError` (merges are destructive, same rationale as `h26041901`). + +Code duplication note: this migration and `h26041901` share the "compute new name / rename-or-merge" shape but not the exact transform. Migrations are historical artifacts — they run once and are never edited — so repeating the scaffolding is preferred over extracting a shared helper module that the migrations would need to import. + +### Modal JS: accept-path refresh parity + +`app/static/js/view-modal.js:203-233` (`acceptSuggestion`): + +```javascript +async function acceptSuggestion(chip, imageId) { + if (chip.dataset.disabled === 'true') return; + chip.dataset.disabled = 'true'; + chip.style.pointerEvents = 'none'; + try { + const r = await fetch(`/image/${imageId}/suggestions/accept`, { /* ...body as before... */ }); + const j = await r.json(); + if (!j.ok) { + chip.dataset.disabled = 'false'; + chip.style.pointerEvents = ''; + if (tagActionFeedback) { + tagActionFeedback.textContent = `Couldn't accept: ${j.error || 'error'}`; + } + return; + } + if (typeof loadTags === 'function') await loadTags(imageId); + if (tagActionFeedback) tagActionFeedback.textContent = `Added ${j.tag.name}`; + if (typeof window.refreshItemTags === 'function') window.refreshItemTags(imageId); + animateChipOut(chip); // fire-and-forget; data is already refreshed + } catch { + chip.dataset.disabled = 'false'; + chip.style.pointerEvents = ''; + } +} +``` + +Key changes: +- `await loadTags(imageId)` before setting feedback and before the chip animation runs. Matches the manual `addTag` ordering at `view-modal.js:600-602`. +- `animateChipOut` no longer takes a completion callback (it's become a pure visual-only helper); its `setTimeout(...)` just removes the node. The signature becomes `animateChipOut(chip)` — callers that still pass a callback lose the callback argument cleanly. +- The chip is disabled via `dataset.disabled` for duplicate-click protection. + +### Modal CSS: persistent green/red chip buttons + +`app/static/style.css:980-1006`: + +```css +.suggestion-chip .sugg-btn.sugg-accept { + color: #8be78b; + border-color: rgba(139, 231, 139, 0.35); +} +.suggestion-chip .sugg-btn.sugg-accept:hover { + background: rgba(139, 231, 139, 0.12); + border-color: rgba(139, 231, 139, 0.6); +} +.suggestion-chip .sugg-btn.sugg-reject { + color: #e78b8b; + border-color: rgba(231, 139, 139, 0.35); +} +.suggestion-chip .sugg-btn.sugg-reject:hover { + background: rgba(231, 139, 139, 0.12); + border-color: rgba(231, 139, 139, 0.6); +} +``` + +The base `.sugg-btn` rule keeps `color: inherit` for anything that isn't accept/reject — safe-by-default if a future button gets added. + +## Data flow — summary + +``` +Suggestion emit: + WD14 prediction ("big_hair", "character:misty_ketchum") + → _wd14_suggestions builds row + → _canonicalize_wd14_name rewrites name ("big hair", "character:Misty Ketchum") + → merged with embedding rows (already canonical) + → returned to modal + +Accept: + POST /image//suggestions/accept {tag_name: "big hair", category: "general"} + → normalize tag_name (underscores_to_spaces on general, normalize_display_name on character/fandom) + → Tag.query.filter_by(name=...) — single lookup + → if missing, create with kind = _WD14_CATEGORY_TO_KIND[category] or None + → attach to image, record feedback, commit + → return {ok: true, tag} + Modal: + → await loadTags(imageId) — tag list refreshes (same timing as manual add) + → set feedback text + → refresh gallery thumbnail overlay + → animate chip out (visual only) + +Migration: + enumerate kind in (character, fandom, NULL) tag rows + compute new name per kind + rename survivors; merge collisions (image_tags union, fandom_id reassign, embedding keep-winner, delete loser) + one transaction, no downgrade +``` + +## Error handling + +- **Canonicalization of empty string:** returns input unchanged; caller's existing `if not tag_name` check short-circuits. +- **Accept with name already canonical:** transforms are idempotent; second call is a no-op on the name. +- **Migration mid-flight error:** full transaction rollback. User investigates via migration log. +- **WD14 emits a name with no underscores but mixed case (e.g., `Copyright:OnePiece`):** `normalize_display_name` applies to character/fandom regardless of underscores; result is `copyright:Onepiece`. The user can still rename via `set-fandom` later if needed. General with no underscore: no change. +- **Accept fetch rejects or throws:** chip is re-enabled via `chip.dataset.disabled = 'false'`; user can retry. + +## Testing plan + +Manual verification on dev DB: + +1. **Migration:** seed `tag` rows `big_hair` (kind=NULL), `fandom:trigun_stampede`, and a pre-existing `big hair` (kind=NULL) to force a merge. Run migration. Verify: + - `big_hair` is gone, `big hair` remains with the union of image links. + - `fandom:trigun_stampede` is renamed to `fandom:Trigun Stampede` (rename, not merge — no collision). + - `tag_reference_embedding` rows for merged losers are deleted. +2. **WD14 chip display:** open an image modal whose WD14 predictions include `long_hair`. Verify the chip shows "long hair" (not "long_hair"). +3. **Accept refresh parity:** click the ✓ on a suggestion. Observe: attached-tags list updates *before* the chip animation completes (no stale frame). Feedback text matches manual-add timing. +4. **Accept canonicalization roundtrip:** click ✓ on a `big_hair` suggestion. Verify: + - `Tag.query.filter_by(name='big hair').first()` returns a row. + - No `big_hair` row exists in `tag`. + - The image's tags include `big hair`. +5. **Button color:** without hovering, verify ✓ is green-tinted and ✕ is red-tinted on all chips. +6. **Duplicate-click protection:** rapid double-click on ✓. Verify only one accept POST fires. +7. **Accept error path:** force the backend to return `{ok: false}`; verify chip re-enables, feedback shows the error, chip stays on screen. + +## Open items + +None. Design confirmed via brainstorming: +- WD14 is the only source introducing underscores. +- Canonicalize at one layer (service for display, accept for persistence — matching transforms). +- General keeps case; only character/fandom title-case. +- Migration covers character + fandom + null kinds with merge-on-collision. +- Modal accept mirrors manual-add refresh ordering. +- Chip buttons persist green/red tint; hover intensifies rather than introduces. diff --git a/docs/superpowers/specs/2026-04-19-user-tag-embedding-suggestions-design.md b/docs/superpowers/specs/2026-04-19-user-tag-embedding-suggestions-design.md new file mode 100644 index 0000000..65d5492 --- /dev/null +++ b/docs/superpowers/specs/2026-04-19-user-tag-embedding-suggestions-design.md @@ -0,0 +1,157 @@ +# User-Tag Embedding-Similarity Suggestions Design + +**Date:** 2026-04-19 +**Branch:** `feat/tag-suggestions` (same branch as the base feature; this is an incremental extension) +**Supersedes:** Partially — extends the character-only centroid mechanism documented in `docs/superpowers/tag-suggestion-system-spec.md`. + +## Problem + +WD14's tag vocabulary is Danbooru-style (`1girl`, `long_hair`, `blue_sky`). The user's library uses their own naming conventions. Today, WD14 is the only source of general/copyright suggestions, so users rarely see their own tags surfaced even when an image visually matches others they've already tagged. + +Character tags already bypass this problem because SigLIP centroid similarity suggests the user's character tags based on visual resemblance to previously-tagged images. This design extends the same mechanism to the other tag kinds the user actually uses: `fandom` (copyright/IP) and null (general/topic). + +## Goals + +- Extend embedding-similarity suggestions from `character` only to `character` + `fandom` + null. +- Keep WD14 alongside — it still runs, still produces suggestions, merges with embedding suggestions via name dedup. +- Drop the `rating` kind from the UI and accept path entirely (the user doesn't need content gating). + +## Non-Goals + +- Replacing WD14. WD14 suggestions continue to surface for the three kinds in scope. +- Per-kind tuning of thresholds or min-refs. Start with shared defaults; add knobs only if real usage shows a need. +- Extending to `artist`, `series`, `post`, or `rating`. Scope stays minimal. +- Changing the accept/reject UX. + +## Architecture + +### Storage + +The existing `character_reference_embedding` table becomes `tag_reference_embedding`: + +- Rename table: `character_reference_embedding` → `tag_reference_embedding`. +- Rename column: `character_tag` → `tag_name`. +- Add column: `tag_kind TEXT NULL` (populated by recompute, nullable for general/topic). +- Unique index migrates with the table: `(tag_name, model_version)`. +- pgvector ivfflat index on `centroid` migrates with the table. + +The SQLAlchemy model class renames from `CharacterReferenceEmbedding` to `TagReferenceEmbedding`. + +### Eligible kinds + +Embedding centroids are computed only for tags whose `kind` is in `{'character', 'fandom', None}`. A single module-level constant `ELIGIBLE_CENTROID_KINDS` lives in `app/services/tag_suggestions.py` and is imported where needed (recompute task, accept endpoint). Adding `artist` or `series` later is a one-tuple change. + +`rating`, `post`, and any other kinds are skipped. + +### Recompute task + +`recompute_centroid(tag_name)` in `app/tasks/ml.py`: + +1. Look up the Tag row to get its `kind`. +2. If `kind` is not in `ELIGIBLE_CENTROID_KINDS`, log and return `{'status': 'ineligible_kind'}`. +3. Query all embeddings for images tagged with `tag_name`. If empty, return `{'status': 'no_embeddings'}`. +4. Compute mean vector. +5. Upsert `TagReferenceEmbedding(tag_name=..., tag_kind=..., model_version=..., centroid=..., reference_count=...)` on the `(tag_name, model_version)` conflict. + +### Batch recompute task + +New task `recompute_all_centroids()`: + +1. For each Tag where `kind IN (character, fandom, NULL)`, count attached images. +2. For each tag with `count >= min_reference_images`, enqueue `recompute_centroid(tag_name)` on the `ml` queue. +3. Logs progress per batch. + +Exposed as a Settings → Maintenance button next to the existing "Run ML backfill" button. + +### Suggestion service + +In `app/services/tag_suggestions.py`: + +- `_embedding_character_suggestions` renames to `_embedding_tag_suggestions`. +- Query drops the character-only filter. Returns suggestions for all centroids above the similarity threshold with `reference_count >= min_reference_images`. +- For each returned row, map `tag_kind` to a display category: + - `character` → `character` + - `fandom` → `copyright` + - `None` → `general` + +The existing `_merge` step dedups by name. If WD14 and embedding similarity both suggest the same tag, the higher-confidence one wins. + +`get_suggestions` returns three groups: `character`, `copyright`, `general`. The `rating` group is removed. + +### Accept endpoint + +In `app/main.py`: + +- `_WD14_CATEGORY_TO_KIND` drops the `'rating': 'rating'` entry. It becomes `{'character': 'character', 'copyright': 'fandom'}`. +- The centroid-recompute trigger block currently guarded by `if tag.kind == 'character'` changes to `if tag.kind in ELIGIBLE_CENTROID_KINDS`. Because `ELIGIBLE_CENTROID_KINDS` includes `None`, null-kind general tags are covered by the same check. +- Delta-triggered recompute stays at `centroid_recompute_delta = 3` globally. + +### Config cleanup + +The existing `tag_suggestion_config` rows are renamed/removed: + +- `threshold_embedding_character` → `threshold_embedding` (same default: 0.85) +- `min_reference_images_per_character` → `min_reference_images` (same default: 5) +- `threshold_rating` → deleted +- Other thresholds (`threshold_general`, `threshold_character_wd14`, `threshold_copyright`) stay unchanged since they still gate WD14 suggestions. + +### UI + +Modal template (`app/templates/_gallery_modal.html`) and `view-modal.js`: + +- Suggestion category order becomes `['character', 'copyright', 'general']` — `rating` is removed. +- Category label map drops the `rating` entry. +- No visual changes to the chip layout (already updated in a prior change). + +## Data flow on accept + +``` +user clicks ✓ + → POST /image//suggestions/accept + → tag created if missing (kind = WD14 category map, or None) + → tag attached to image + → if tag.kind in ELIGIBLE_CENTROID_KINDS or None, check delta: + current_image_count - last_reference_count >= 3 + → enqueue recompute_centroid(tag_name) on ml queue + → ml-worker runs recompute_centroid + → mean-pools embeddings, upserts tag_reference_embedding row + → next image's suggestion request benefits from the updated centroid +``` + +## Migration plan for existing data + +The database already has one row in `character_reference_embedding` per character tag touched so far. The migration: + +1. `ALTER TABLE character_reference_embedding RENAME TO tag_reference_embedding`. +2. `ALTER TABLE tag_reference_embedding RENAME COLUMN character_tag TO tag_name`. +3. `ALTER TABLE tag_reference_embedding ADD COLUMN tag_kind TEXT NULL`. +4. Backfill existing rows: `UPDATE tag_reference_embedding SET tag_kind = 'character'` (safe because every pre-existing row was written by the old character-only path). +5. Drop `threshold_rating` row from `tag_suggestion_config`. +6. Rename `threshold_embedding_character` → `threshold_embedding`. +7. Rename `min_reference_images_per_character` → `min_reference_images`. + +After migration, the user clicks "Recompute all centroids" once from Settings → Maintenance to populate centroids for any fandom/general tags they've already applied to 5+ images. + +### Existing rating data is preserved + +The migration does not delete any `Tag` rows with `kind='rating'` or their image-tag associations. WD14 continues to store rating predictions in `image_tag_prediction` (category='rating') on every tag_and_embed run. This data is simply no longer surfaced in the suggestions UI or accepted via the new endpoint paths. If the user ever wants rating gating back, re-adding it is a UI-only change. + +## Error handling + +- `recompute_centroid` for an ineligible kind: logs and returns `{'status': 'ineligible_kind'}`. No retry. +- `recompute_centroid` for a tag with 0 images (e.g., all images that used this tag were deleted): logs and returns `{'status': 'no_embeddings'}`. No retry. +- `recompute_all_centroids` failures in individual enqueues: logged, loop continues. +- Accept endpoint centroid enqueue failure: swallowed (existing behavior — don't fail the accept). + +## Testing plan + +Manual verification after the implementation lands: + +1. Migration runs cleanly on the dev DB. Existing character centroid survives with `tag_kind='character'`. +2. Fresh suggestion request for an image returns three groups (no `rating`). +3. Tag a fandom across 5 images, trigger recompute, verify new image shows that fandom as a suggestion. +4. Tag a null-kind general topic across 5 images, trigger recompute, verify the general group now includes it. +5. Accept a general-kind suggestion — verify `Tag(kind=None)` is created, image gets it, centroid recompute is enqueued after the delta threshold is met. +6. Settings → Maintenance → "Recompute all centroids" button enqueues recomputes only for eligible-kind tags with ≥ 5 images. + +No automated tests in this spec — the base feature didn't have any either, and the manual test plan in the parent spec still covers the integration surface. diff --git a/docs/superpowers/tag-suggestion-system-spec.md b/docs/superpowers/tag-suggestion-system-spec.md new file mode 100644 index 0000000..eb64405 --- /dev/null +++ b/docs/superpowers/tag-suggestion-system-spec.md @@ -0,0 +1,265 @@ +# Tag Suggestion System - Technical Spec + +## Overview + +Add an ML-powered tag suggestion system to the existing gallery application. The system suggests tags for images in two ways: + +1. **Direct classification via WD14** for general tags, ratings, popular characters, and series +2. **Embedding similarity** for OC/niche character identification based on previously-tagged reference images + +The system produces *suggestions only* - never auto-applies tags. All suggestions require user confirmation. Artists are explicitly out of scope; they are set at import time from source metadata. + +## Scale & Context + +- ~62,000 existing images (anime/drawn art exclusively) +- ~300 tagged characters currently, expected to grow +- Single-user or small-user tagging workflow +- Target hardware: NVIDIA Tesla P4 GPU (8GB VRAM, compute-only, no NVDEC needed for this workload) +- Existing app is homegrown; this is an addition, not a rewrite + +## Out of Scope + +- Artist identification (handled at import from source metadata) +- Auto-applying tags without user confirmation +- Training or fine-tuning models (inference only; "learning" happens via reference set growth) +- Multi-user permission models for suggestions + +## Models + +### Tagger: WD14 (SmilingWolf EVA02-Large variant) + +- Model: `SmilingWolf/wd-eva02-large-tagger-v3` (or current best available) +- Purpose: Produces confidence-scored tags across categories (general, character, copyright/series, rating, meta) +- Runs on GPU via ONNX Runtime with CUDA execution provider +- Output: tag name + category + confidence score (0.0-1.0) + +### Embedding Model: SigLIP or OpenCLIP + +- Recommended: `google/siglip-so400m-patch14-384` (strong performance, reasonable size) +- Alternative: `laion/CLIP-ViT-H-14-laion2B-s32B-b79K` if SigLIP integration is problematic +- Output: Fixed-dimension embedding vector per image (SigLIP SO400M: 1152-dim) +- Purpose: Enables similarity search for character matching against reference set + +### Model Versioning + +Store the model identifier (name + version) alongside every output. Schema must support multiple model versions coexisting to allow migration without requiring immediate reprocessing of all 62K images. + +## Database Schema Changes + +Assumes Postgres. If the existing app uses a different database, adapt accordingly - `pgvector` is strongly preferred for embedding storage at this scale. + +### New extension + +```sql +CREATE EXTENSION IF NOT EXISTS vector; +``` + +### New tables + +```sql +-- Raw WD14 tagger output, one row per (image, tag, model_version) +CREATE TABLE image_tag_predictions ( + id BIGSERIAL PRIMARY KEY, + image_id BIGINT NOT NULL REFERENCES images(id) ON DELETE CASCADE, + tag_name TEXT NOT NULL, + tag_category TEXT NOT NULL, -- 'general', 'character', 'copyright', 'rating', 'meta' + confidence REAL NOT NULL, + model_version TEXT NOT NULL, -- e.g. 'wd-eva02-large-tagger-v3' + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_tag_predictions_image ON image_tag_predictions(image_id); +CREATE INDEX idx_tag_predictions_tag ON image_tag_predictions(tag_name); +CREATE INDEX idx_tag_predictions_model ON image_tag_predictions(model_version); + +-- Image embeddings for similarity search +CREATE TABLE image_embeddings ( + image_id BIGINT NOT NULL REFERENCES images(id) ON DELETE CASCADE, + model_version TEXT NOT NULL, + embedding vector(1152) NOT NULL, -- adjust dimension to match chosen model + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (image_id, model_version) +); + +-- HNSW index for fast nearest-neighbor search +CREATE INDEX idx_embeddings_hnsw ON image_embeddings + USING hnsw (embedding vector_cosine_ops); + +-- Cached centroid embeddings per character, for fast suggestion queries +CREATE TABLE character_reference_embeddings ( + character_tag TEXT NOT NULL, + model_version TEXT NOT NULL, + centroid vector(1152) NOT NULL, + reference_count INTEGER NOT NULL, -- how many confirmed images contributed + computed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (character_tag, model_version) +); + +-- Tracks user decisions on suggestions for future tuning/reranking +CREATE TABLE suggestion_feedback ( + id BIGSERIAL PRIMARY KEY, + image_id BIGINT NOT NULL REFERENCES images(id) ON DELETE CASCADE, + tag_name TEXT NOT NULL, + suggestion_source TEXT NOT NULL, -- 'wd14' or 'embedding_similarity' + confidence REAL NOT NULL, + decision TEXT NOT NULL, -- 'accepted' or 'rejected' + decided_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_feedback_image ON suggestion_feedback(image_id); +CREATE INDEX idx_feedback_tag ON suggestion_feedback(tag_name); +``` + +### Configuration table (or use existing config system) + +Store these as tunable values, not hardcoded: + +```sql +CREATE TABLE tag_suggestion_config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + description TEXT +); + +-- Initial values (thresholds will need tuning) +INSERT INTO tag_suggestion_config VALUES + ('threshold_general', '0.35', 'Min confidence for general tag suggestions'), + ('threshold_character_wd14', '0.75', 'Min confidence for WD14 character tags'), + ('threshold_copyright', '0.5', 'Min confidence for copyright/series tags'), + ('threshold_rating', '0.5', 'Min confidence for rating tags'), + ('threshold_meta', '0.5', 'Min confidence for meta tags'), + ('threshold_embedding_character', '0.85', 'Min cosine similarity for character via embeddings'), + ('min_reference_images_per_character', '5', 'Characters with fewer confirmed refs are excluded from embedding suggestions'), + ('centroid_recompute_delta', '3', 'Recompute character centroid after N new confirmed references'); +``` + +## Inference Pipeline + +### On upload (per-image, near-realtime) + +1. WD14 inference → insert rows into `image_tag_predictions` +2. Embedding model inference → insert row into `image_embeddings` +3. Suggestions are generated on-demand at view time (see below); no storage of suggestions themselves needed + +### Initial backfill (one-time, for existing 62K images) + +- Background job that iterates all images without predictions/embeddings +- Batched GPU inference (WD14 supports batch sizes; 8-16 is a reasonable starting point on P4) +- Expect several hours of runtime; run overnight +- Must be resumable (track last-processed image_id, handle failures gracefully) +- Progress reporting (log every N images, estimated time remaining) + +### On confirmed character tag (user accepts a character tag on an image) + +- Increment a counter for that character's reference set +- If delta since last centroid computation exceeds `centroid_recompute_delta`, schedule centroid recomputation for that character +- Recomputation: fetch all embeddings for images confirmed to have this character tag, compute mean (or medoid if desired), update `character_reference_embeddings` + +### Suggestion generation (at view time or via background job) + +When generating suggestions for an image: + +**From WD14 predictions:** +- Query `image_tag_predictions` for this image +- Filter by per-category confidence thresholds +- Exclude tags already confirmed on the image + +**From embedding similarity (for characters only):** +- Fetch this image's embedding +- Query `character_reference_embeddings` for all characters with `reference_count >= min_reference_images_per_character` +- Compute cosine similarity against each character's centroid +- Filter by `threshold_embedding_character` +- Exclude characters already confirmed on the image +- Return top K (suggest K=5 initially) + +**Merging:** +- WD14 character suggestions and embedding-based character suggestions may overlap; dedupe by tag name, prefer whichever has higher normalized confidence +- Return merged suggestion list grouped by category, sorted by confidence within each category + +## Background Jobs + +Three scheduled/triggered jobs: + +### 1. Backfill job (one-time initially, resumable) + +- Process any image missing WD14 predictions or embeddings for current model versions +- Can be re-run after model version change to update stored data + +### 2. Centroid recomputation (triggered or nightly) + +- For each character whose `reference_count` has changed by more than `centroid_recompute_delta` since last computation, recompute centroid +- Fast operation; can run frequently + +### 3. Upload processing (per-image, on upload) + +- Runs WD14 + embedding inference on newly uploaded images +- Should not block the upload response; queue for async processing + +## API / Integration Points + +The spec doesn't prescribe exact endpoints since it depends on the existing app's conventions, but the following operations must be exposed: + +- `get_suggestions(image_id) -> list of suggested tags grouped by category` - called when viewing an image's tag panel +- `accept_suggestion(image_id, tag_name, source)` - records feedback, applies tag to image, triggers centroid recomputation if character +- `reject_suggestion(image_id, tag_name, source)` - records feedback only +- `queue_backfill()` - admin operation to process any unprocessed images +- `get_character_reference_stats() -> list of (character, reference_count)` - admin visibility into which characters have enough references to participate in embedding suggestions + +## Implementation Notes & Decisions + +### Why not store suggestions in the database? + +Suggestions are a function of (predictions + embeddings + current confirmed tags + current thresholds + current centroids). Any of these can change. Storing suggestions creates a cache invalidation problem. Instead, compute them on read - the operations are cheap once predictions and embeddings exist. + +### Why store raw WD14 predictions at all? + +Two reasons: +1. Threshold tuning without re-running inference. Change thresholds, suggestions update immediately. +2. Model upgrade path. When moving from model v3 to v4, old predictions remain queryable until new ones are generated, avoiding a blackout period. + +### Why centroids instead of querying all reference images? + +At 300 characters with potentially thousands of references each, querying per-image embeddings for every character at suggestion time gets expensive. Centroids reduce it to one vector per character. Trade-off: loses some nuance (a character with multiple distinct outfits is represented as one "average" point). If this becomes a problem, options include: +- Multiple centroids per character via clustering (k-medoids with small k) +- Fallback to top-K raw embedding search for characters with high intra-class variance + +Not worth building upfront. Revisit if suggestion quality suffers. + +### Why track rejection feedback? + +Stored feedback enables: +- Per-tag or per-category threshold tuning based on acceptance rates +- Future reranker training if suggestion quality plateaus +- Diagnosing which characters consistently produce bad suggestions + +Low cost to collect; high value if needed later. + +### Embedding dimension + +Adjust `vector(1152)` in schema to match whatever model is actually selected. SigLIP SO400M is 1152-dim; ViT-H/14 CLIP is 1024-dim; some other variants differ. Mismatch will cause pgvector errors. + +### GPU considerations + +- P4 is compute-only (no NVDEC needed for this workload - we're doing ML inference, not video decode) +- WD14 ONNX model fits comfortably in 8GB VRAM +- SigLIP SO400M inference fits in 8GB with room to spare +- If co-located with other GPU workloads (e.g., Ollama), use `CUDA_VISIBLE_DEVICES` to pin to a specific GPU and avoid contention + +## Tuning Phase (post-implementation) + +After initial deployment, expect to spend time tuning: + +1. Per-category confidence thresholds (collect feedback data first, then analyze acceptance rate vs. confidence curves) +2. Per-character embedding thresholds (some characters are distinctive, some generic-looking) +3. Minimum reference count before a character enters the suggestion pool +4. Centroid recomputation cadence + +Build thresholds as config values (see `tag_suggestion_config` table), not constants. Expose them via whatever admin interface the existing app has. + +## Open Questions for Implementation + +- Which ONNX Runtime integration approach fits the existing app's stack (Python FastAPI, Rust, Node, etc.)? +- Is there an existing background job system in the app, or does one need to be added? +- What's the existing app's approach to config management - env vars, DB table, config file? + +These should be answered by inspecting the existing codebase before implementation begins. diff --git a/migrations/versions/0017871d2221_add_tag_suggestions.py b/migrations/versions/0017871d2221_add_tag_suggestions.py new file mode 100644 index 0000000..65059f6 --- /dev/null +++ b/migrations/versions/0017871d2221_add_tag_suggestions.py @@ -0,0 +1,121 @@ +"""add tag suggestions + +Revision ID: 0017871d2221 +Revises: f26021101 +Create Date: 2026-04-18 20:29:43.837834 + +""" +from alembic import op +import sqlalchemy as sa +from pgvector.sqlalchemy import Vector + + +# revision identifiers, used by Alembic. +revision = '0017871d2221' +down_revision = 'f26021101' +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute("CREATE EXTENSION IF NOT EXISTS vector") + + op.create_table( + 'image_tag_prediction', + sa.Column('id', sa.Integer(), primary_key=True), + sa.Column('image_id', sa.Integer(), nullable=False), + sa.Column('tag_name', sa.Text(), nullable=False), + sa.Column('tag_category', sa.Text(), nullable=False), + sa.Column('confidence', sa.Float(), nullable=False), + sa.Column('model_version', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.ForeignKeyConstraint(['image_id'], ['image_record.id'], ondelete='CASCADE', + name='fk_image_tag_prediction_image_id'), + ) + op.create_index('idx_tag_predictions_image', 'image_tag_prediction', ['image_id']) + op.create_index('idx_tag_predictions_tag', 'image_tag_prediction', ['tag_name']) + op.create_index('idx_tag_predictions_model', 'image_tag_prediction', ['model_version']) + + op.create_table( + 'image_embedding', + sa.Column('image_id', sa.Integer(), nullable=False), + sa.Column('model_version', sa.Text(), nullable=False), + sa.Column('embedding', Vector(1152), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.PrimaryKeyConstraint('image_id', 'model_version'), + sa.ForeignKeyConstraint(['image_id'], ['image_record.id'], ondelete='CASCADE', + name='fk_image_embedding_image_id'), + ) + op.execute( + "CREATE INDEX idx_embeddings_hnsw ON image_embedding " + "USING hnsw (embedding vector_cosine_ops)" + ) + + op.create_table( + 'character_reference_embedding', + sa.Column('character_tag', sa.Text(), nullable=False), + sa.Column('model_version', sa.Text(), nullable=False), + sa.Column('centroid', Vector(1152), nullable=False), + sa.Column('reference_count', sa.Integer(), nullable=False), + sa.Column('computed_at', sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.PrimaryKeyConstraint('character_tag', 'model_version'), + ) + + op.create_table( + 'suggestion_feedback', + sa.Column('id', sa.Integer(), primary_key=True), + sa.Column('image_id', sa.Integer(), nullable=False), + sa.Column('tag_name', sa.Text(), nullable=False), + sa.Column('suggestion_source', sa.Text(), nullable=False), + sa.Column('confidence', sa.Float(), nullable=False), + sa.Column('decision', sa.Text(), nullable=False), + sa.Column('decided_at', sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False), + sa.ForeignKeyConstraint(['image_id'], ['image_record.id'], ondelete='CASCADE', + name='fk_suggestion_feedback_image_id'), + ) + op.create_index('idx_feedback_image', 'suggestion_feedback', ['image_id']) + op.create_index('idx_feedback_tag', 'suggestion_feedback', ['tag_name']) + + op.create_table( + 'tag_suggestion_config', + sa.Column('key', sa.Text(), primary_key=True), + sa.Column('value', sa.Text(), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + ) + + # Seed default thresholds + op.bulk_insert( + sa.table( + 'tag_suggestion_config', + sa.column('key', sa.Text()), + sa.column('value', sa.Text()), + sa.column('description', sa.Text()), + ), + [ + {'key': 'threshold_general', 'value': '0.35', 'description': 'Min confidence for general tag suggestions'}, + {'key': 'threshold_character_wd14', 'value': '0.75', 'description': 'Min confidence for WD14 character tags'}, + {'key': 'threshold_copyright', 'value': '0.5', 'description': 'Min confidence for copyright/fandom tags'}, + {'key': 'threshold_rating', 'value': '0.5', 'description': 'Min confidence for rating tags'}, + {'key': 'threshold_meta', 'value': '0.5', 'description': 'Min confidence for meta tags (omitted from modal)'}, + {'key': 'threshold_embedding_character', 'value': '0.85', 'description': 'Min cosine similarity for character via embeddings'}, + {'key': 'min_reference_images_per_character', 'value': '5', 'description': 'Characters with fewer confirmed refs are excluded from embedding suggestions'}, + {'key': 'centroid_recompute_delta', 'value': '3', 'description': 'Recompute character centroid after N new confirmed references'}, + {'key': 'wd14_model_version', 'value': 'wd-eva02-large-tagger-v3', 'description': 'Current WD14 model version'}, + {'key': 'siglip_model_version', 'value': 'siglip-so400m-patch14-384', 'description': 'Current SigLIP embedding model version'}, + ], + ) + + +def downgrade(): + op.drop_table('tag_suggestion_config', if_exists=True) + op.drop_index('idx_feedback_tag', table_name='suggestion_feedback', if_exists=True) + op.drop_index('idx_feedback_image', table_name='suggestion_feedback', if_exists=True) + op.drop_table('suggestion_feedback', if_exists=True) + op.drop_table('character_reference_embedding', if_exists=True) + op.execute("DROP INDEX IF EXISTS idx_embeddings_hnsw") + op.drop_table('image_embedding', if_exists=True) + op.drop_index('idx_tag_predictions_model', table_name='image_tag_prediction', if_exists=True) + op.drop_index('idx_tag_predictions_tag', table_name='image_tag_prediction', if_exists=True) + op.drop_index('idx_tag_predictions_image', table_name='image_tag_prediction', if_exists=True) + op.drop_table('image_tag_prediction', if_exists=True) + op.execute("DROP EXTENSION IF EXISTS vector") diff --git a/migrations/versions/g26041901_rename_character_to_tag_reference.py b/migrations/versions/g26041901_rename_character_to_tag_reference.py new file mode 100644 index 0000000..ea02741 --- /dev/null +++ b/migrations/versions/g26041901_rename_character_to_tag_reference.py @@ -0,0 +1,75 @@ +"""rename character_reference_embedding to tag_reference_embedding and add tag_kind + +Revision ID: g26041901 +Revises: 0017871d2221 +Create Date: 2026-04-19 + +""" +from alembic import op +import sqlalchemy as sa + + +revision = 'g26041901' +down_revision = '0017871d2221' +branch_labels = None +depends_on = None + + +def upgrade(): + # Rename table + op.rename_table('character_reference_embedding', 'tag_reference_embedding') + + # Rename column + op.alter_column('tag_reference_embedding', 'character_tag', new_column_name='tag_name') + + # Add tag_kind column (nullable — null = general/topic tag) + op.add_column( + 'tag_reference_embedding', + sa.Column('tag_kind', sa.Text(), nullable=True), + ) + + # Every pre-existing row was written by the character-only path, so tag_kind='character'. + op.execute("UPDATE tag_reference_embedding SET tag_kind = 'character'") + + # Rename config keys. Postgres-safe single UPDATE per key. + op.execute( + "UPDATE tag_suggestion_config " + "SET key = 'threshold_embedding' " + "WHERE key = 'threshold_embedding_character'" + ) + op.execute( + "UPDATE tag_suggestion_config " + "SET key = 'min_reference_images' " + "WHERE key = 'min_reference_images_per_character'" + ) + + # Drop the rating threshold — rating is no longer surfaced in the UI. + op.execute("DELETE FROM tag_suggestion_config WHERE key = 'threshold_rating'") + + +def downgrade(): + # Restore threshold_rating seed with original default. + op.execute( + "INSERT INTO tag_suggestion_config (key, value, description) " + "VALUES ('threshold_rating', '0.5', 'Min confidence for rating tags') " + "ON CONFLICT (key) DO NOTHING" + ) + + # Revert config key renames. + op.execute( + "UPDATE tag_suggestion_config " + "SET key = 'min_reference_images_per_character' " + "WHERE key = 'min_reference_images'" + ) + op.execute( + "UPDATE tag_suggestion_config " + "SET key = 'threshold_embedding_character' " + "WHERE key = 'threshold_embedding'" + ) + + # Drop tag_kind column. + op.drop_column('tag_reference_embedding', 'tag_kind') + + # Revert column and table renames. + op.alter_column('tag_reference_embedding', 'tag_name', new_column_name='character_tag') + op.rename_table('tag_reference_embedding', 'character_reference_embedding') diff --git a/migrations/versions/h26041901_normalize_character_fandom_tag_names.py b/migrations/versions/h26041901_normalize_character_fandom_tag_names.py new file mode 100644 index 0000000..035eab1 --- /dev/null +++ b/migrations/versions/h26041901_normalize_character_fandom_tag_names.py @@ -0,0 +1,163 @@ +"""Normalize character and fandom tag display names; merge case-only duplicates. + +Revision ID: h26041901 +Revises: g26041901 +Create Date: 2026-04-19 + +Upgrade: title-cases the display-name portion of every character:* and fandom:* +tag. When two rows of the same kind collapse to the same name, merges image_tags, +tag.fandom_id references, and tag_reference_embedding rows into the already- +correctly-cased target ("winner") and deletes the "loser" tag row. + +Downgrade: raises. Merges are destructive; reverting would require restoring +from a pre-migration backup. +""" +from alembic import op +from sqlalchemy import text + + +revision = 'h26041901' +down_revision = 'g26041901' +branch_labels = None +depends_on = None + + +def _normalize_display(s: str) -> str: + """Mirror of app.utils.tag_names.normalize_display_name. Inlined here so + the migration doesn't import app code (Alembic runs without the Flask app).""" + if not s: + return s + out = [] + for p in s.split(' '): + if p == '': + out.append(p) + else: + out.append(p[:1].upper() + p[1:]) + return ' '.join(out) + + +def _compute_new_name(old_name: str, kind: str) -> str: + """Given a full 'prefix:display' name, return the normalized full name.""" + if ':' not in old_name: + return old_name + prefix, rest = old_name.split(':', 1) + if kind == 'character': + # Split off "(Fandom)" suffix if present, normalize each part. + if '(' in rest and rest.rstrip().endswith(')'): + paren_idx = rest.rfind('(') + base = rest[:paren_idx].rstrip() + fandom_part = rest[paren_idx+1:-1].strip() # strip outer parens + any interior whitespace + base = _normalize_display(base) + fandom_part = _normalize_display(fandom_part) + return f"{prefix}:{base} ({fandom_part})" + return f"{prefix}:{_normalize_display(rest)}" + elif kind == 'fandom': + return f"{prefix}:{_normalize_display(rest)}" + return old_name + + +def upgrade(): + conn = op.get_bind() + # Work on a snapshot; we mutate the tag table as we iterate. + rows = conn.execute( + text("SELECT id, name, kind FROM tag WHERE kind IN ('character', 'fandom') ORDER BY id") + ).fetchall() + + renames = 0 + merges = 0 + for row in rows: + tag_id = row[0] + old_name = row[1] + kind = row[2] + new_name = _compute_new_name(old_name, kind) + if new_name == old_name: + continue + + # Re-check that this tag still exists (may have been deleted as a loser in + # an earlier merge within this loop). + still_exists = conn.execute( + text("SELECT id FROM tag WHERE id = :id"), + {'id': tag_id}, + ).fetchone() + if not still_exists: + continue + + # Look for a collision: same target name + same kind + different id. + target = conn.execute( + text("SELECT id, name FROM tag WHERE name = :new_name AND kind = :kind AND id != :tag_id"), + {'new_name': new_name, 'kind': kind, 'tag_id': tag_id}, + ).fetchone() + + if target is None: + # No collision — plain rename. + conn.execute( + text("UPDATE tag SET name = :new_name WHERE id = :tag_id"), + {'new_name': new_name, 'tag_id': tag_id}, + ) + renames += 1 + else: + # Merge: winner = target (already correctly cased), loser = current row. + winner_id = target[0] + winner_name = target[1] # already == new_name + loser_id = tag_id + loser_name = old_name + + # 1. Reassign image_tags, dedup into winner. + conn.execute( + text(""" + INSERT INTO image_tags (image_id, tag_id) + SELECT DISTINCT image_id, :winner_id + FROM image_tags + WHERE tag_id = :loser_id + ON CONFLICT DO NOTHING + """), + {'winner_id': winner_id, 'loser_id': loser_id}, + ) + conn.execute( + text("DELETE FROM image_tags WHERE tag_id = :loser_id"), + {'loser_id': loser_id}, + ) + + # 2. Reassign any fandom_id FKs pointing at the loser. + conn.execute( + text("UPDATE tag SET fandom_id = :winner_id WHERE fandom_id = :loser_id"), + {'winner_id': winner_id, 'loser_id': loser_id}, + ) + + # 3. tag_reference_embedding: the table is keyed by (tag_name, model_version). + # For each model_version where only the loser has a row, rename; where + # both have rows, keep the winner's and delete the loser's. + conn.execute( + text(""" + UPDATE tag_reference_embedding + SET tag_name = :winner_name + WHERE tag_name = :loser_name + AND NOT EXISTS ( + SELECT 1 FROM tag_reference_embedding w + WHERE w.tag_name = :winner_name + AND w.model_version = tag_reference_embedding.model_version + ) + """), + {'winner_name': winner_name, 'loser_name': loser_name}, + ) + conn.execute( + text("DELETE FROM tag_reference_embedding WHERE tag_name = :loser_name"), + {'loser_name': loser_name}, + ) + + # 4. Delete the loser tag row. + conn.execute( + text("DELETE FROM tag WHERE id = :loser_id"), + {'loser_id': loser_id}, + ) + merges += 1 + + print(f"normalize_character_fandom_tag_names: {renames} renames, {merges} merges") + + +def downgrade(): + raise NotImplementedError( + "h26041901 normalizes tag names and merges case-only duplicates. " + "Merges are destructive and cannot be reversed automatically. " + "Restore from a pre-migration backup if you need to roll back." + ) diff --git a/migrations/versions/i26041901_normalize_underscores_and_general_tags.py b/migrations/versions/i26041901_normalize_underscores_and_general_tags.py new file mode 100644 index 0000000..89d70f5 --- /dev/null +++ b/migrations/versions/i26041901_normalize_underscores_and_general_tags.py @@ -0,0 +1,189 @@ +"""Strip underscores from character/fandom/null-kind tag display names and merge +case-only duplicates that collapse onto the same canonical name. + +Revision ID: i26041901 +Revises: h26041901 +Create Date: 2026-04-19 + +Upgrade: for every row in `tag` with kind in ('character', 'fandom', NULL), +compute the canonical name (character/fandom: title-case + underscore→space; +null: underscore→space only). Rename survivors. On collision, merge image_tags, +fandom_id references, and tag_reference_embedding rows into the already- +correctly-named winner and delete the loser. + +Downgrade: raises. Merges are destructive; reverting requires a pre-migration +backup. +""" +from alembic import op +from sqlalchemy import text + + +revision = 'i26041901' +down_revision = 'h26041901' +branch_labels = None +depends_on = None + + +def _underscores_to_spaces(s: str) -> str: + """Mirror of app.utils.tag_names.underscores_to_spaces. Inlined so the + migration doesn't import app code (Alembic runs without the Flask app).""" + return s.replace('_', ' ') if s else s + + +def _normalize_display(s: str) -> str: + """Mirror of app.utils.tag_names.normalize_display_name.""" + if not s: + return s + s = _underscores_to_spaces(s) + out = [] + for p in s.split(' '): + if p == '': + out.append(p) + else: + out.append(p[:1].upper() + p[1:]) + return ' '.join(out) + + +def _compute_new_name(old_name: str, kind) -> str: + """Return the canonical full name for a given (name, kind) pair. + + kind == 'character': title-case base + (optional fandom suffix). + kind == 'fandom': title-case the display portion after the prefix. + kind IS NULL: underscore→space only; case preserved. + """ + if kind == 'character': + if ':' not in old_name: + return old_name + prefix, rest = old_name.split(':', 1) + if '(' in rest and rest.rstrip().endswith(')'): + paren_idx = rest.rfind('(') + base = rest[:paren_idx].rstrip() + fandom_part = rest[paren_idx+1:-1].strip() + return f"{prefix}:{_normalize_display(base)} ({_normalize_display(fandom_part)})" + return f"{prefix}:{_normalize_display(rest)}" + elif kind == 'fandom': + if ':' not in old_name: + return old_name + prefix, rest = old_name.split(':', 1) + return f"{prefix}:{_normalize_display(rest)}" + elif kind is None: + # Null-kind (general): keep case, just strip underscores. Split off any + # prefix defensively — historical rows may or may not carry one. + if ':' in old_name: + prefix, rest = old_name.split(':', 1) + return f"{prefix}:{_underscores_to_spaces(rest)}" + return _underscores_to_spaces(old_name) + return old_name + + +def upgrade(): + conn = op.get_bind() + rows = conn.execute( + text(""" + SELECT id, name, kind FROM tag + WHERE kind IN ('character', 'fandom') OR kind IS NULL + ORDER BY id + """) + ).fetchall() + + renames = 0 + merges = 0 + for row in rows: + tag_id = row[0] + old_name = row[1] + kind = row[2] + new_name = _compute_new_name(old_name, kind) + if new_name == old_name: + continue + + # Re-check that this tag still exists (may have been deleted as a loser + # in an earlier merge within this loop). + still_exists = conn.execute( + text("SELECT id FROM tag WHERE id = :id"), + {'id': tag_id}, + ).fetchone() + if not still_exists: + continue + + # Collision check: same target name + same kind (NULL-safe) + different id. + if kind is None: + target = conn.execute( + text("SELECT id, name FROM tag WHERE name = :new_name AND kind IS NULL AND id != :tag_id"), + {'new_name': new_name, 'tag_id': tag_id}, + ).fetchone() + else: + target = conn.execute( + text("SELECT id, name FROM tag WHERE name = :new_name AND kind = :kind AND id != :tag_id"), + {'new_name': new_name, 'kind': kind, 'tag_id': tag_id}, + ).fetchone() + + if target is None: + conn.execute( + text("UPDATE tag SET name = :new_name WHERE id = :tag_id"), + {'new_name': new_name, 'tag_id': tag_id}, + ) + renames += 1 + else: + winner_id = target[0] + winner_name = target[1] + loser_id = tag_id + loser_name = old_name + + # 1. Reassign image_tags into winner. + conn.execute( + text(""" + INSERT INTO image_tags (image_id, tag_id) + SELECT DISTINCT image_id, :winner_id + FROM image_tags + WHERE tag_id = :loser_id + ON CONFLICT DO NOTHING + """), + {'winner_id': winner_id, 'loser_id': loser_id}, + ) + conn.execute( + text("DELETE FROM image_tags WHERE tag_id = :loser_id"), + {'loser_id': loser_id}, + ) + + # 2. Reassign fandom_id FKs pointing at the loser. + conn.execute( + text("UPDATE tag SET fandom_id = :winner_id WHERE fandom_id = :loser_id"), + {'winner_id': winner_id, 'loser_id': loser_id}, + ) + + # 3. tag_reference_embedding: keep winner's row per model_version; + # otherwise rename loser's row to the winner. + conn.execute( + text(""" + UPDATE tag_reference_embedding + SET tag_name = :winner_name + WHERE tag_name = :loser_name + AND NOT EXISTS ( + SELECT 1 FROM tag_reference_embedding w + WHERE w.tag_name = :winner_name + AND w.model_version = tag_reference_embedding.model_version + ) + """), + {'winner_name': winner_name, 'loser_name': loser_name}, + ) + conn.execute( + text("DELETE FROM tag_reference_embedding WHERE tag_name = :loser_name"), + {'loser_name': loser_name}, + ) + + # 4. Delete the loser tag row. + conn.execute( + text("DELETE FROM tag WHERE id = :loser_id"), + {'loser_id': loser_id}, + ) + merges += 1 + + print(f"normalize_underscores_and_general_tags: {renames} renames, {merges} merges") + + +def downgrade(): + raise NotImplementedError( + "i26041901 canonicalizes tag names and merges duplicates that collapse. " + "Merges are destructive and cannot be reversed automatically. " + "Restore from a pre-migration backup if you need to roll back." + ) diff --git a/requirements-ml.txt b/requirements-ml.txt new file mode 100644 index 0000000..40ebbae --- /dev/null +++ b/requirements-ml.txt @@ -0,0 +1,21 @@ +# Base app deps (mirrors requirements.txt so the ml-worker can import app code). +# Floors reflect recent-stable baselines — update them proactively when you bump the main requirements.txt +# so a future minor release of any dep can't pull the ml-worker below a version we've actually tested. +Flask>=3.0 +Flask-SQLAlchemy>=3.1 +Flask-Migrate>=4.0 +psycopg2-binary>=2.9 +pillow>=10.4 +exifread>=3.0 +imagehash>=4.3 +gunicorn>=22.0 +celery[redis]>=5.4.0 +redis>=5.1.0 +pgvector>=0.3.0 + +# ML-specific deps (CPU-only) +numpy>=1.26 # transformers 4.45+/torch 2.4+ accept numpy 2.x +onnxruntime>=1.19 # CPU build; do not install onnxruntime-gpu here +transformers>=4.45 +huggingface_hub>=0.25 +sentencepiece>=0.2.0 # required by SiglipTokenizer diff --git a/requirements.txt b/requirements.txt index 02f3596..3c04a1f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ gunicorn # Celery task queue celery[redis]>=5.3.0 redis>=5.0.0 +pgvector>=0.2.5 diff --git a/scripts/download_models.py b/scripts/download_models.py new file mode 100644 index 0000000..c0bb123 --- /dev/null +++ b/scripts/download_models.py @@ -0,0 +1,45 @@ +"""Pre-fetch WD14 and SigLIP model files into $ML_MODEL_DIR at image build time.""" +from __future__ import annotations +import os +import sys + +from huggingface_hub import hf_hub_download, snapshot_download + +MODEL_DIR = os.environ.get('ML_MODEL_DIR', '/models') +WD14_REPO = os.environ.get('WD14_REPO', 'SmilingWolf/wd-eva02-large-tagger-v3') +SIGLIP_REPO = os.environ.get('SIGLIP_REPO', 'google/siglip-so400m-patch14-384') + + +def download_wd14() -> None: + target = os.path.join(MODEL_DIR, 'wd14') + os.makedirs(target, exist_ok=True) + hf_hub_download(repo_id=WD14_REPO, filename='model.onnx', local_dir=target, local_dir_use_symlinks=False) + hf_hub_download(repo_id=WD14_REPO, filename='selected_tags.csv', local_dir=target, local_dir_use_symlinks=False) + print(f"WD14 downloaded to {target}") + + +def download_siglip() -> None: + target = os.path.join(MODEL_DIR, 'siglip') + os.makedirs(target, exist_ok=True) + snapshot_download( + repo_id=SIGLIP_REPO, + local_dir=target, + local_dir_use_symlinks=False, + allow_patterns=[ + 'config.json', + 'preprocessor_config.json', + 'tokenizer_config.json', + 'tokenizer.json', + 'special_tokens_map.json', + 'spiece.model', + 'model.safetensors', + 'pytorch_model.bin', + ], + ) + print(f"SigLIP downloaded to {target}") + + +if __name__ == '__main__': + download_wd14() + download_siglip() + sys.exit(0)