Merge pull request 'fix(agent): flatten transparency onto white before RGB' (#148) from dev into main

pg_dump + zstd in runtime image, lift Quart body cap to 1 GiB. See PR #2.

.forgejo/workflows/ci.yml

@@ -9,15 +9,10 @@ name: CI
 on:
   push:
     branches: [dev, main]
-  # Renovate opens PRs from `renovate/*` branches into `dev`. Those branches
-  # never push to dev/main, so the push trigger above gives them NO pre-merge
-  # CI — a bump could only be validated after it was already merged. This
-  # pull_request trigger (base `dev` only) validates Renovate PRs before merge.
-  # It deliberately does NOT fire on dev→main PRs (base `main`), which still
-  # rely on the dev push run — so no duplicate runs. FC has no fork PRs
-  # (single-operator Forgejo repo), so secrets-on-PR is not a concern.
-  pull_request:
-    branches: [dev]
+  # pull_request trigger intentionally absent — with branches: [dev, main]
+  # above, every PR commit already fires CI via the push event on dev. Adding
+  # pull_request would duplicate runs on dev→main PRs. FC has no fork PRs
+  # (single-operator Forgejo repo) so push coverage is complete.
 
 jobs:
   # Fast-fail lint lane. ruff is pre-installed in the ci-python image, so
@@ -32,14 +27,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Ruff lint
-        # agent/ included so the GPU-agent is linted before its image is built
-        # (build.yml only `docker build`s it — this is where it gets checked).
-        run: ruff check backend/ tests/ alembic/ agent/
-      - name: Agent syntax check
-        # The agent's runtime deps (torch/transformers/ultralytics) aren't in the
-        # CI image, so we can't import it — but compileall parses every module,
-        # catching syntax errors before the image build.
-        run: python -m compileall -q agent/fc_agent
+        run: ruff check backend/ tests/ alembic/
 
   backend-lint-and-test:
     runs-on: python-ci
@@ -97,10 +85,10 @@ jobs:
       # If we want strict lockfile-based reproducibility later, commit a
       # package-lock.json and flip this back to `npm ci`.
       - run: npm install --no-audit --no-fund
-      # No type-check step: the frontend is pure JS (no .ts files, no JSDoc),
-      # so a type-checker has nothing to do. The vue-tsc devDep + its `check`
-      # script were dropped 2026-07-11 rather than bumped to v3. If we add
-      # TS/JSDoc later, re-add a tsconfig.json + vue-tsc + a type-check step.
+      # `npm run check` (vue-tsc --noEmit) skipped: the frontend is pure JS
+      # with no .ts files and no JSDoc annotations, so vue-tsc has nothing
+      # to type-check. Re-enable once we add a tsconfig.json and either
+      # convert to TS or add JSDoc.
       - run: npm run test:unit
       - run: npm run build
 

.forgejo/workflows/extension.yml

@@ -20,7 +20,7 @@ jobs:
   lint:
     runs-on: python-ci
     container:
-      image: node:24-bookworm-slim
+      image: node:22-bookworm-slim
     steps:
       - uses: actions/checkout@v4
       - name: Install web-ext

Dockerfile

@@ -1,6 +1,6 @@
-# syntax=docker/dockerfile:1.25
+# syntax=docker/dockerfile:1.7
 
-FROM node:24-alpine AS frontend-builder
+FROM node:22-alpine AS frontend-builder
 WORKDIR /build
 COPY frontend/package.json frontend/package-lock.json* ./
 # No package-lock.json is tracked yet (we don't run npm locally per

Dockerfile.ml

@@ -1,4 +1,4 @@
-# syntax=docker/dockerfile:1.25
+# syntax=docker/dockerfile:1.7
 
 FROM python:3.14-slim
 ENV PYTHONUNBUFFERED=1 \

agent/Dockerfile

@@ -1,31 +1,20 @@
 # FabledCurator GPU agent — runs on the desktop with the GPU.
-# CUDA 12.9 + cuDNN 9 runtime so onnxruntime-gpu can use the card (it needs
-# cuDNN 9 — the plain -runtime image lacks it: "libcudnn.so.9: cannot open
-# shared object file"); ffmpeg for video frames. Ubuntu 24.04 → Python 3.12.
-# Stays on the CUDA-12 / cuDNN-9 line the default onnxruntime-gpu + torch are
-# built against (CUDA 13 has only nascent ONNX Runtime support).
-FROM nvidia/cuda:12.9.2-cudnn-runtime-ubuntu24.04
+# CUDA + cuDNN runtime so onnxruntime-gpu can use the card (it needs cuDNN 9 —
+# the plain -runtime image lacks it: "libcudnn.so.9: cannot open shared object
+# file"); ffmpeg for video frames.
+FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
 
-# PIP_BREAK_SYSTEM_PACKAGES: Ubuntu 24.04 marks its system Python as externally
-# managed (PEP 668), so a global `pip install` errors without this. It's a
-# single-purpose container — we own the whole environment, so installing into
-# the system site-packages is fine (and simplest — no venv on PATH to manage).
-ENV DEBIAN_FRONTEND=noninteractive PYTHONUNBUFFERED=1 PIP_BREAK_SYSTEM_PACKAGES=1
+ENV DEBIAN_FRONTEND=noninteractive PYTHONUNBUFFERED=1
 RUN apt-get update \
     && apt-get install -y --no-install-recommends python3 python3-pip ffmpeg \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
-# torch from the CUDA-12.4 wheel index; its wheels bundle their own CUDA + cuDNN
-# so they run on the 12.9 base and coexist with onnxruntime-gpu. Installed first
-# + separately so the GPU build of torch is deterministic and layer-cached.
-RUN pip3 install --no-cache-dir torch==2.6.0 --index-url https://download.pytorch.org/whl/cu124
 COPY requirements.txt .
 RUN pip3 install --no-cache-dir -r requirements.txt
 COPY fc_agent ./fc_agent
 
-# imgutils ONNX models + the transformers SigLIP weights both cache here; mount
-# a volume to persist them across restarts (the SigLIP download is ~3.5 GB once).
+# imgutils caches downloaded ONNX models here; mount a volume to persist them.
 ENV HF_HOME=/models
 EXPOSE 8770
 

agent/docker-compose.yml

@@ -10,13 +10,6 @@
 #   4. Open http://localhost:8770 → Start. Pause/Stop hands the GPU back.
 #      docker compose down       to stop the container entirely.
 #
-# Surviving a curator redeploy (you're away, can't touch the agent):
-#   - A running agent rides out curator being unreachable on its own — it retries
-#     leasing with capped backoff and resumes when the server is back. In-flight
-#     work is handed back (not failed), so a redeploy never poisons good jobs.
-#   - AUTO_START=1 (below) also resumes the worker if the AGENT container itself
-#     restarts (host reboot / crash via `restart: unless-stopped`) — no click.
-#
 # Needs the NVIDIA Container Toolkit installed on the host for --gpus.
 
 services:
@@ -31,32 +24,6 @@ services:
       CCIP_MODEL: ${CCIP_MODEL:-}
       DETECTOR_LEVEL: ${DETECTOR_LEVEL:-m}
       BATCH_SIZE: ${BATCH_SIZE:-4}
-      # Resume the worker automatically on container start (survive a reboot /
-      # crash-restart while you're away). Set to 0 to require a manual Start.
-      AUTO_START: ${AUTO_START:-1}
-      # Autoscale the worker count (throughput hill-climb that finds the sweet
-      # spot + backs off under VRAM pressure). On by default; toggle live in the
-      # control UI. Set to 0 to start in manual mode.
-      AUTO_SCALE: ${AUTO_SCALE:-1}
-      # Aggregate download cap in MB/s (stills + video streams combined), so the
-      # agent can't saturate the desktop's network and wreck browsing — WiFi
-      # especially. 0 = unlimited; tunable live in the control UI.
-      BANDWIDTH_LIMIT_MB_S: ${BANDWIDTH_LIMIT_MB_S:-8}
-      # Crop embedder (SigLIP concept bag): float16 keeps VRAM low on a shared
-      # desktop GPU; the model itself is announced by the server.
-      SIGLIP_DTYPE: ${SIGLIP_DTYPE:-float16}
-      # Crop PROPOSERS (extra YOLO detectors → more/better concept crops). Each
-      # downloads its weights once (cached on the models volume) and self-disables
-      # if the download/load fails. Blank any one to turn it off.
-      #   PERSON_WEIGHTS: general COCO person detector (Western/realistic figures),
-      #     merged with the anime detector. yolo11n.pt (~6 MB, auto-downloaded).
-      #   ANATOMY_WEIGHTS: booru_yolo anime/furry/NSFW components (~40 MB). NB the
-      #     repo states no license — fine for private use. yolov8n_as01.pt is the
-      #     6 MB nano if you want lighter than yolov11m_aa22.pt.
-      #   PANEL_WEIGHTS: mosesb comic-panel detector (Apache-2.0), "hf_repo::file".
-      PERSON_WEIGHTS: ${PERSON_WEIGHTS:-yolo11n.pt}
-      ANATOMY_WEIGHTS: ${ANATOMY_WEIGHTS:-https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt}
-      PANEL_WEIGHTS: ${PANEL_WEIGHTS:-mosesb/best-comic-panel-detection::best.pt}
     volumes:
       # Persist the downloaded ONNX models so restarts are fast.
       - fc-agent-models:/models

agent/fc_agent/app.py

@@ -1,70 +1,34 @@
 """FastAPI control surface for the agent (served on localhost).
 
-Start / stop the download→GPU pipeline, tune the downloader count live (the
-workload is download-bound, so downloaders are the dial that trades desktop
-bandwidth for throughput), and watch GPU load + buffer occupancy + progress +
-the server-side queue. Config is env-seeded; the downloader count is adjustable
-here on the fly (GPU consumers autoscale between 1 and 2 on their own).
+Start / stop the worker pool, tune the worker count live (trades desktop
+responsiveness for throughput), and watch GPU load + progress + the server-side
+queue. Config is env-seeded; the worker count is adjustable here on the fly.
 """
-import logging
-
 from fastapi import FastAPI, Request
 from fastapi.responses import HTMLResponse, JSONResponse
 
-from . import logbuf
 from .config import Config
 from .gpu import read_gpu
 from .worker import Worker
 
-log = logging.getLogger("fc_agent.app")
-
-# Bump on every agent change. The page embeds this and /status reports it; the UI
-# warns to reload when they differ — so a stale browser-cached page can't be
-# mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.)
-VERSION = "2026-07-02.6 · sleep mode: an empty queue sheds to one downloader and backs the lease poll off to 15 min"
-
-logbuf.install()
 cfg = Config.from_env()
 worker = Worker(cfg)
 app = FastAPI(title="FabledCurator GPU agent")
 
 
-@app.middleware("http")
-async def _no_store(request, call_next):
-    # The control page is a static string and the status/gpu/logs polls are
-    # live data — never let the browser cache either, or a freshly-pulled agent
-    # image still shows the OLD UI until a hard refresh (operator-flagged
-    # 2026-06-30).
-    resp = await call_next(request)
-    resp.headers["Cache-Control"] = "no-store"
-    return resp
-
-
-@app.on_event("startup")
-def _maybe_autostart() -> None:
-    # With AUTO_START set, a container restart (host reboot, or `restart:
-    # unless-stopped` after a crash) resumes the worker on its own — the slots
-    # then ride out a still-down curator via lease backoff. Lets the agent
-    # survive a redeploy with nobody at the desktop to click Start.
-    if cfg.auto_start and cfg.token:
-        worker.start()
-
-
 @app.get("/", response_class=HTMLResponse)
 def index() -> str:
-    return _PAGE.replace("__BUILD__", VERSION)
+    return _PAGE
 
 
 @app.post("/start")
 def start():
-    log.info("UI: Start button pressed")   # the press; worker logs the transition
     worker.start()
     return JSONResponse(worker.status())
 
 
 @app.post("/stop")
 def stop():
-    log.info("UI: Stop button pressed")
     worker.stop()
     return JSONResponse(worker.status())
 
@@ -76,329 +40,71 @@ async def concurrency(request: Request):
     return JSONResponse(worker.status())
 
 
-@app.post("/auto")
-async def auto(request: Request):
-    body = await request.json()
-    worker.set_auto(bool(body.get("value", True)))
-    return JSONResponse(worker.status())
-
-
-@app.post("/bandwidth")
-async def bandwidth(request: Request):
-    body = await request.json()
-    worker.set_bandwidth(float(body.get("value", 0)))
-    return JSONResponse(worker.status())
-
-
-@app.get("/gpu")
-def gpu():
-    # GPU meters poll this on their own fast cadence. It only reads local
-    # nvidia-smi — no curator round-trip — so the util/VRAM bars stay live even
-    # when /status is slow waiting on the (sometimes busy) curator queue call.
-    g = read_gpu() or {}
-    us = worker.util_smooth()
-    if us is not None:
-        g["util_smooth"] = round(us, 1)   # autoscaler's EWMA — the UI bar tracks this
-    return JSONResponse(g)
-
-
-@app.get("/logs")
-def logs():
-    return JSONResponse({"lines": list(logbuf.LINES)})
-
-
 @app.get("/status")
 def status():
-    # Pure in-memory read: worker.status() is lock-free and the queue snapshot is
-    # kept fresh by a background poller — NO inline curator call, so this can't
-    # stall the status view when curator is buried under a big backlog.
-    worker.note_ui()          # a browser is watching → keep the queue snapshot warm
     s = worker.status()
     s["fc_url"] = cfg.fc_url
     s["configured"] = bool(cfg.token)
-    s["queue"] = worker.latest_queue()
-    s["build"] = VERSION
+    s["gpu"] = read_gpu()
+    try:
+        s["queue"] = worker.client.queue_status()
+    except Exception:
+        s["queue"] = None
     return JSONResponse(s)
 
 
 _PAGE = """<!doctype html><html><head><meta charset=utf-8>
-<meta name=viewport content="width=device-width,initial-scale=1">
-<title>FabledCurator · GPU agent</title>
+<title>FabledCurator GPU agent</title>
 <style>
- :root{--bg:#0f1216;--panel:#181c22;--panel2:#1e232b;--bd:#2a313b;--fg:#e9edf2;
-  --mut:#8b97a6;--acc:#e8923a;--grn:#46c46a;--red:#e8584d;--amb:#e8b23a}
- *{box-sizing:border-box}
- body{font:14px/1.5 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;margin:0;
-  background:radial-gradient(1200px 600px at 50% -10%,#1a2029,#0f1216);color:var(--fg)}
- .wrap{max-width:820px;margin:0 auto;padding:28px 20px 28px;height:100vh;
-  box-sizing:border-box;overflow:hidden;display:flex;flex-direction:column}
- header{display:flex;align-items:center;justify-content:space-between;margin-bottom:4px}
- .brand{display:flex;align-items:center;gap:10px;font-size:19px;font-weight:700;letter-spacing:.2px}
- .logo{color:var(--acc);font-size:20px}
- .brand .sub{color:var(--mut);font-weight:600;font-size:13px;text-transform:uppercase;letter-spacing:.12em}
- .conn{display:flex;align-items:center;gap:8px;color:var(--mut);font-size:13px;font-weight:600}
- .dot{width:9px;height:9px;border-radius:50%;background:var(--mut);box-shadow:0 0 0 0 rgba(0,0,0,0)}
- .dot.green{background:var(--grn);box-shadow:0 0 10px 1px rgba(70,196,106,.5)}
- .dot.amber{background:var(--amb)} .dot.red{background:var(--red)}
- .meta{color:var(--mut);margin:0 0 18px;font-size:13px}
- code{background:#11151a;border:1px solid var(--bd);padding:2px 7px;border-radius:6px;
-  font:12px ui-monospace,SFMono-Regular,Menlo,monospace;color:#cdd6e0}
- .card{background:linear-gradient(180deg,var(--panel),var(--panel2));border:1px solid var(--bd);
-  border-radius:14px;padding:16px 18px;margin-bottom:14px;box-shadow:0 1px 0 rgba(255,255,255,.02) inset}
- .card-h{font-size:11px;font-weight:800;letter-spacing:.12em;text-transform:uppercase;
-  color:var(--mut);margin-bottom:14px}
- .controls{display:flex;align-items:center;gap:10px;flex-wrap:wrap}
- .spacer{flex:1}
- .btn{font:600 14px system-ui;padding:.5rem 1rem;border:1px solid transparent;border-radius:9px;
-  cursor:pointer;color:#fff;transition:.12s}
- .btn:hover{transform:translateY(-1px)}
- .btn[disabled]{opacity:.45;pointer-events:none;transform:none}
- @keyframes pulse{0%,100%{opacity:1}50%{opacity:.4}}
- .tile .n.busy{color:var(--acc);animation:pulse 1s ease-in-out infinite}
- .btn.start{background:linear-gradient(180deg,#2f9c4c,#247a3c)}
- .btn.stop{background:linear-gradient(180deg,#3a3f48,#2a2f37);color:#e9edf2;border-color:var(--bd)}
- .switch{display:inline-flex;align-items:center;gap:8px;cursor:pointer;font-weight:600;user-select:none}
- .switch input{display:none}
- .switch .track{width:38px;height:22px;border-radius:11px;background:#2a313b;position:relative;transition:.15s}
- .switch .track:after{content:"";position:absolute;top:2px;left:2px;width:18px;height:18px;border-radius:50%;
-  background:#cdd6e0;transition:.15s}
- .switch input:checked+.track{background:var(--acc)}
- .switch input:checked+.track:after{transform:translateX(16px);background:#fff}
- .stepper{display:inline-flex;align-items:center;gap:6px}
- .step{background:#262c34;color:var(--fg);border:1px solid var(--bd);border-radius:8px;
-  width:30px;height:32px;font:700 16px system-ui;cursor:pointer}
- .step:hover{border-color:var(--acc)}
- #conc,#bw{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a;
-  color:var(--fg);border:1px solid var(--bd);border-radius:8px}
- .unit{color:var(--mut);font-size:12px;font-weight:600}
- .hint{color:var(--mut);font-size:12px;margin-top:12px}
- .tiles{display:grid;grid-template-columns:repeat(6,1fr);gap:8px;margin-bottom:16px}
- .tile{background:#13171d;border:1px solid var(--bd);border-radius:10px;padding:12px 8px;text-align:center}
- .tile .n{font:800 22px ui-monospace,monospace;line-height:1.1}
- .tile .n.warn{color:var(--red)} .tile .n.ok{color:var(--grn)}
- .tile .l{font-size:10px;text-transform:uppercase;letter-spacing:.06em;color:var(--mut);margin-top:4px}
- .meters{display:flex;flex-direction:column;gap:10px;margin-bottom:14px}
- .meter-h{display:flex;justify-content:space-between;font-size:12px;color:var(--mut);margin-bottom:4px}
- .meter-h b{color:var(--fg);font-variant-numeric:tabular-nums}
- .bar{height:9px;border-radius:5px;background:#11151a;border:1px solid var(--bd);overflow:hidden}
- .bar>i{display:block;height:100%;width:0;background:linear-gradient(90deg,#3a7d57,var(--grn));transition:width .4s}
- #utilbar{background:linear-gradient(90deg,#9a5a1f,var(--acc))}
- #bufbar{background:linear-gradient(90deg,#2f5a9a,#4a86d8)}
- .queue{font:13px ui-monospace,monospace;color:var(--mut)}
- .banner{margin:0 0 14px;padding:.7rem .9rem;border-radius:10px;background:#3a2f12;
-  border:1px solid #5a4a17;color:#ffd98a;font-size:13px}
- .logs-h{display:flex;align-items:center;justify-content:space-between}
- .grow{flex:1;display:flex;flex-direction:column;min-height:0}
- .grow .logs{flex:1;min-height:0}
- .copybtn{font:600 11px system-ui;letter-spacing:.04em;text-transform:uppercase;
-  background:#262c34;color:var(--fg);border:1px solid var(--bd);border-radius:7px;
-  padding:5px 11px;cursor:pointer}
- .copybtn:hover{border-color:var(--acc)}
- .logs{margin:0;background:#0b0e12;border:1px solid var(--bd);border-radius:10px;padding:12px;
-  overflow:auto;font:12px/1.55 ui-monospace,SFMono-Regular,Menlo,monospace;
-  color:#b9c4d0;white-space:pre-wrap;word-break:break-word}
+ body{font:14px system-ui;margin:2rem;max-width:680px;background:#14171a;color:#e8e8e8}
+ h1{font-size:18px} button{font:14px system-ui;padding:.5rem 1rem;border:0;border-radius:6px;
+ margin-right:.5rem;cursor:pointer;color:#fff} .start{background:#2e7d32}.stop{background:#b3261e}
+ .step{background:#33373b;padding:.4rem .7rem;font-weight:700}
+ .stat{display:inline-block;margin-right:1.5rem;vertical-align:top}
+ .n{font-size:22px;font-weight:700} code{background:#222;padding:2px 6px;border-radius:4px}
+ .q,.gpu{margin-top:1rem;color:#9aa} .bar{height:8px;border-radius:4px;background:#222;overflow:hidden;
+ max-width:320px;margin-top:4px} .bar>i{display:block;height:100%;background:#3f7d3f}
+ .row{margin:.8rem 0}
 </style></head><body>
-<div class=wrap>
- <header>
-  <div class=brand><span class=logo>◆</span> FabledCurator <span class=sub>GPU agent</span></div>
-  <div class=conn><span class="dot" id=dot></span><span id=connlbl>—</span></div>
- </header>
- <p class=meta>Server <code id=fc>—</code> · token <code id=cfg>—</code> · build <code id=build>__BUILD__</code></p>
-
- <div id=verbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3">
-  a newer agent version is running — reload this page (Ctrl+Shift+R) to update the controls
- </div>
- <div id=banner class=banner style=display:none>
-  curator unreachable — holding work + retrying, resumes on its own (no restart needed)
- </div>
-
- <section class=card>
-  <div class=card-h>Control</div>
-  <div class=controls>
-   <button class="btn start" id=startbtn onclick=act('start')>▶ Start</button>
-   <button class="btn stop" id=stopbtn onclick=act('stop')>■ Stop</button>
-   <div class=spacer></div>
-   <label class=switch><input type=checkbox id=autochk onchange="setauto(this.checked)"><span class=track></span>Auto</label>
-   <div class=stepper>
-    <button class=step onclick=setc(-1)>−</button>
-    <input id=conc type=number min=1 value=1 onchange="setv(this.value)">
-    <button class=step onclick=setc(1)>+</button>
-   </div>
-   <div class=stepper title="aggregate download cap, downloads + video streams combined — 0 = unlimited">
-    <input id=bw type=number min=0 step=1 value=8 onchange="setbw(this.value)">
-    <span class=unit>MB/s</span>
-   </div>
-  </div>
-  <div class=hint id=conchint>auto-tuning downloaders to keep the GPU fed · max 8</div>
- </section>
-
- <section class=card>
-  <div class=card-h>Status</div>
-  <div class=tiles>
-   <div class=tile><div class=n id=state>—</div><div class=l>state</div></div>
-   <div class=tile><div class=n id=jpm>—</div><div class=l>jobs / min</div></div>
-   <div class=tile><div class=n id=dpm>—</div><div class=l>downloads / min</div></div>
-   <div class=tile><div class="n ok" id=done>0</div><div class=l>processed</div></div>
-   <div class=tile><div class=n id=err>0</div><div class=l>errors</div></div>
-   <div class=tile><div class=n id=waited>0</div><div class=l>waited out</div></div>
-  </div>
-  <div class=meters>
-   <div class=meter><div class=meter-h><span>GPU util</span><b id=utillbl>—</b></div>
-    <div class=bar><i id=utilbar></i></div></div>
-   <div class=meter><div class=meter-h><span>VRAM</span><b id=vramlbl>—</b></div>
-    <div class=bar><i id=gpubar></i></div></div>
-   <div class=meter><div class=meter-h><span>buffer occupancy</span><b id=buflbl>—</b></div>
-    <div class=bar><i id=bufbar></i></div></div>
-  </div>
-  <div class=queue id=pipe>downloaders — · consumers — · on GPU 0</div>
-  <div class=queue id=queue>queue —</div>
- </section>
-
- <section class="card grow">
-  <div class="card-h logs-h">Logs
-   <button class=copybtn id=copybtn onclick=copyLogs()>Copy</button>
-  </div>
-  <pre class=logs id=logs>waiting for activity…</pre>
- </section>
+<h1>FabledCurator GPU agent</h1>
+<p>FC: <code id=fc>—</code> · token <code id=cfg>—</code></p>
+<div class=row>
+ <button class=start onclick=act('start')>Start</button>
+ <button class=stop onclick=act('stop')>Stop</button>
 </div>
+<div class=row>
+ workers
+ <button class=step onclick=setc(-1)>−</button>
+ <b id=conc style=margin:0+.5rem>1</b>
+ <button class=step onclick=setc(1)>+</button>
+ <span class=cap style=color:#9aa>(more = faster + more GPU)</span>
+</div>
+<div class=row>
+ <span class=stat><span class=n id=state>stopped</span><br>state</span>
+ <span class=stat><span class=n id=active>0</span><br>active now</span>
+ <span class=stat><span class=n id=done>0</span><br>processed</span>
+ <span class=stat><span class=n id=err>0</span><br>errors</span>
+</div>
+<div class=gpu id=gpu>GPU — …</div>
+<div class=bar><i id=gpubar style=width:0%></i></div>
+<div class=q id=queue></div>
 <script>
- const PAGE_BUILD="__BUILD__"
- let CAP=8
- // Optimistic transitional state on click, then apply the POST's own status
- // response (it returns worker.status()) for instant feedback — don't wait on the
- // separate /status poll, which can lag behind the curator queue call.
- async function act(p){
-  pending(p==='start'?'starting':'stopping')
-  // Abort a slow POST after 8s so the buttons never stay stuck — the periodic
-  // /status refresh (now always fast) recovers the true state either way.
-  const ac=new AbortController(); const to=setTimeout(()=>ac.abort(),8000)
-  try{ applyStatus(await (await fetch('/'+p,{method:'POST',signal:ac.signal})).json()) }
-  catch{ refresh() /* on abort/error, repaint the real state from /status */ }
-  finally{ clearTimeout(to) }
- }
- function pending(label){
-  // Instant optimistic feedback on click; applyStatus (POST response, then the
-  // periodic poll) then owns the real state + which buttons are enabled.
-  state.textContent=label; state.className='n busy'
-  dot.className='dot amber'
-  startbtn.disabled=true; stopbtn.disabled=true
- }
- function setc(d){ if(conc.disabled)return; setv((parseInt(conc.value||'1'))+d) }
- async function setv(v){
-  v=Math.max(1,Math.min(CAP,parseInt(v)||1)); conc.value=v
+ async function act(p){await fetch('/'+p,{method:'POST'});refresh()}
+ async function setc(d){
+  const v=Math.max(1,Math.min(8,parseInt(conc.textContent||'1')+d))
   await fetch('/concurrency',{method:'POST',headers:{'Content-Type':'application/json'},
    body:JSON.stringify({value:v})});refresh()
  }
- async function setauto(on){
-  await fetch('/auto',{method:'POST',headers:{'Content-Type':'application/json'},
-   body:JSON.stringify({value:on})});refresh()
- }
- async function setbw(v){
-  v=Math.max(0,parseFloat(v)||0); bw.value=v
-  await fetch('/bandwidth',{method:'POST',headers:{'Content-Type':'application/json'},
-   body:JSON.stringify({value:v})});refresh()
- }
  async function refresh(){
-  let s; try{ s=await (await fetch('/status')).json() }catch{ return }
-  applyStatus(s)
+  const s=await (await fetch('/status')).json()
+  state.textContent=s.state; active.textContent=s.active; done.textContent=s.processed
+  err.textContent=s.errors; conc.textContent=s.concurrency; fc.textContent=s.fc_url
+  cfg.textContent=s.configured?'set':'MISSING'
+  if(s.gpu){
+   gpu.textContent=`GPU — ${s.gpu.util_pct}% util · VRAM ${s.gpu.mem_used_mb}/${s.gpu.mem_total_mb} MB · ${s.gpu.temp_c}°C`
+   gpubar.style.width=Math.round(100*s.gpu.mem_used_mb/s.gpu.mem_total_mb)+'%'
+  } else { gpu.textContent='GPU — n/a (CPU fallback?)'; gpubar.style.width='0%' }
+  queue.textContent=s.queue?`queue — pending ${s.queue.pending} · in flight ${s.queue.leased} · done ${s.queue.done} · errored ${s.queue.error}`:'queue — unreachable'
  }
- function applyStatus(s){
-  // NB: don't write a separate `capn` element here — conchint.textContent below
-  // rewrites the whole hint (incl. the max), and any child element nested in it
-  // would be destroyed by that write, breaking the NEXT applyStatus call.
-  CAP=s.max_concurrency||8
-  // The backend owns the state now (stopped|starting|running|stopping) and drives
-  // every transition, so the pill is always truthful — no client-side guessing
-  // from active>0, which used to wedge on "stopping" forever.
-  const st=s.state||'stopped'
-  const running=st==='running'
-  const busy=(st==='starting'||st==='stopping')
-  // Stale-page guard: if the server is a newer build than this page, the cached
-  // controls may misbehave — tell the operator to reload.
-  if(s.build && s.build!==PAGE_BUILD) verbanner.style.display='block'
-  state.textContent=st
-  state.className='n'+(busy?' busy':'')
-  // Buttons follow the real state so you can't fight a transition: Start only
-  // from stopped; Stop only while up; both disabled through "stopping" until the
-  // backend truthfully lands on "stopped".
-  startbtn.disabled=(st!=='stopped')
-  stopbtn.disabled=!(running||st==='starting')
-  // Throughput rates arrive READY from the backend (jobs/min ≈ GPU throughput,
-  // dl/min ≈ fetch throughput), computed there on a fixed cadence — so they show
-  // a real number no matter how often this tab polls (a backgrounded tab throttles
-  // its timers, which used to leave a client-side delta-rate blank forever).
-  jpm.textContent=(s.jobs_per_min!=null)?Math.round(s.jobs_per_min):'—'
-  dpm.textContent=(s.downloads_per_min!=null)?Math.round(s.downloads_per_min):'—'
-  done.textContent=s.processed
-  err.textContent=s.errors; err.className='n'+(s.errors>0?' warn':'')
-  waited.textContent=s.transient||0
-  // Instantaneous pool state → demoted to the sub-line, where its jumpiness reads
-  // as live churn rather than a "broken" headline metric.
-  pipe.textContent='downloaders '+(s.downloaders!=null?s.downloaders:'—')+' · consumers '+(s.consumers!=null?s.consumers:'—')+' · on GPU '+(s.active||0)
-   +' · net '+(s.net_mb_s!=null?s.net_mb_s.toFixed(1):'—')+' MB/s'
-   +(s.bandwidth_limit_mb_s>0?(' / cap '+s.bandwidth_limit_mb_s):'')
-  if(document.activeElement!==bw && s.bandwidth_limit_mb_s!=null) bw.value=s.bandwidth_limit_mb_s
-  // Buffer occupancy bar (also driven here so it tracks the /status cadence).
-  if(s.buffer!=null && s.buffer_max){ const p=Math.round(100*s.buffer/s.buffer_max)
-   buflbl.textContent=s.buffer+' / '+s.buffer_max; bufbar.style.width=p+'%' }
-  // Auto on → dial reflects the auto-chosen count (read-only); off → manual.
-  if(document.activeElement!==autochk) autochk.checked=!!s.auto
-  conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.55:1
-  conchint.textContent=(s.auto?('auto-tuning downloaders to keep the GPU fed · max '+CAP):('manual downloaders · max '+CAP))
-   +(s.idle?' · idle — queue empty, lease poll backed off (new work noticed within ~15 min)'
-     :(s.bw_capped?' · holding at the bandwidth cap (more downloaders would not go faster)':''))
-  if(document.activeElement!==conc) conc.value=s.concurrency
-  conc.max=CAP
-  // Connection pill + queue come only from the /status poll (the Start/Stop POST
-  // responses skip the slow curator call to stay snappy) — guard so an action
-  // response doesn't blank them.
-  if('configured' in s){
-   const ok=s.configured
-   fc.textContent=s.fc_url; cfg.textContent=ok?'set':'MISSING'
-   // Pill colour + label track the real state: green only when running AND
-   // curator is answering; amber for the transient states + a running-but-
-   // unreachable curator; grey when stopped; red with no token.
-   let dc='dot', lbl='stopped'
-   if(!ok){ dc='dot red'; lbl='no token' }
-   else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable' }
-   else if(st==='starting'){ dc='dot amber'; lbl='starting…' }
-   else if(st==='stopping'){ dc='dot amber'; lbl='stopping…' }
-   dot.className=dc; connlbl.textContent=lbl
-   banner.style.display=(st==='running' && !s.queue)?'block':'none'
-   queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
-  }
- }
- // GPU meters poll their OWN endpoint on a fast cadence — kept off /status so a
- // slow curator queue call can't freeze the bars (they only stale on refresh).
- let UAVG=null   // smoothed util for the bar (raw util swings 0↔99; show the trend)
- async function refreshGpu(){
-  let g; try{ g=await (await fetch('/gpu')).json() }catch{ return }
-  if(g && g.util_pct!=null){
-   // Prefer the agent's own EWMA (util_smooth) when running; otherwise smooth
-   // the raw reading here so a stopped agent's bar still glides, not jumps.
-   const raw=g.util_pct
-   UAVG = (g.util_smooth!=null) ? g.util_smooth
-         : (UAVG==null ? raw : 0.25*raw + 0.75*UAVG)
-   const used=g.mem_used_mb, tot=g.mem_total_mb||1
-   utillbl.textContent=Math.round(UAVG)+'% · '+g.temp_c+'°C'; utilbar.style.width=Math.round(UAVG)+'%'
-   vramlbl.textContent=used+' / '+tot+' MB'; gpubar.style.width=Math.round(100*used/tot)+'%'
-  } else { UAVG=null; utillbl.textContent='n/a'; vramlbl.textContent='n/a (CPU?)'; utilbar.style.width='0%'; gpubar.style.width='0%' }
- }
- async function refreshLogs(){
-  try{
-   const r=await (await fetch('/logs')).json()
-   const el=logs, atBottom=el.scrollHeight-el.scrollTop-el.clientHeight<40
-   el.textContent=(r.lines&&r.lines.length)?r.lines.join('\\n'):'waiting for activity…'
-   if(atBottom) el.scrollTop=el.scrollHeight
-  }catch{}
- }
- async function copyLogs(){
-  const txt=logs.textContent||''
-  try{ await navigator.clipboard.writeText(txt) }
-  catch{ const t=document.createElement('textarea'); t.value=txt; document.body.appendChild(t);
-   t.select(); try{document.execCommand('copy')}catch{}; t.remove() }
-  copybtn.textContent='Copied'; setTimeout(()=>{copybtn.textContent='Copy'},1200)
- }
- refresh(); refreshGpu(); refreshLogs()
- setInterval(refresh,3000); setInterval(refreshGpu,1500); setInterval(refreshLogs,2500)
+ refresh(); setInterval(refresh,3000)
 </script></body></html>"""

agent/fc_agent/client.py

@@ -4,70 +4,14 @@ The agent's ONLY contact with FC — lease/submit/heartbeat/fail + fetch image
 bytes, all over HTTP with the bearer token. No DB/Redis.
 """
 import requests
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 
 
 class FcClient:
     def __init__(self, base_url: str, token: str, agent_id: str):
         self.base = base_url.rstrip("/")
         self.agent_id = agent_id
-        # Main session: NO in-request retry — lease/fetch are cheap to redo and
-        # the worker loop already backs off + re-leases on failure. (Auto-retrying
-        # a lease could double-claim a batch if a response is lost.)
-        self.s = self._session(token)
-        # Submit session: retry in-place, because by submit time the GPU work is
-        # already DONE — a momentary blip (dropped connection, gateway 5xx during
-        # a curator redeploy) must not throw that work away and force a full
-        # re-download + recompute on another agent. A duplicate submit after a
-        # lost response is harmless: the job is already closed, so it just returns
-        # 409 lease_invalid (a no-op). Idempotent enough to retry POST safely.
-        retry = Retry(
-            total=3, connect=3, read=3, status=3,
-            backoff_factor=0.5,                       # ~0.5s, 1s, 2s between tries
-            status_forcelist=(500, 502, 503, 504),    # transient server/gateway
-            allowed_methods=frozenset({"POST"}),
-            raise_on_status=False,                     # let raise_for_status decide
-        )
-        self._submit_s = self._session(token, retry)
-
-    @staticmethod
-    def _session(token: str, retry: Retry | None = None) -> requests.Session:
-        s = requests.Session()
-        s.headers["Authorization"] = f"Bearer {token}"
-        # Many worker threads share a Session; the default pool (10) would
-        # throttle them + spam "connection pool is full". Size it for the cap.
-        adapter = HTTPAdapter(
-            pool_connections=64, pool_maxsize=64, max_retries=retry or 0
-        )
-        s.mount("http://", adapter)
-        s.mount("https://", adapter)
-        return s
-
-    def _submit(self, path: str, payload: dict) -> dict:
-        """POST to a submit endpoint on the RETRYING session (by submit time the
-        GPU work is done — a blip must not throw it away), raise on a hard error,
-        and return the parsed JSON. `agent_id` is added to every body."""
-        r = self._submit_s.post(
-            f"{self.base}{path}",
-            json={"agent_id": self.agent_id, **payload},
-            timeout=120,
-        )
-        r.raise_for_status()
-        return r.json()
-
-    def _post_quiet(self, path: str, payload: dict) -> None:
-        """Fire-and-forget POST on the main session — heartbeat/fail/release are
-        best-effort, so a transport error is swallowed (the worker's own retry and
-        the server's orphan-recovery cover a lost call). `agent_id` is added."""
-        try:
-            self.s.post(
-                f"{self.base}{path}",
-                json={"agent_id": self.agent_id, **payload},
-                timeout=30,
-            )
-        except requests.RequestException:
-            pass
+        self.s = requests.Session()
+        self.s.headers["Authorization"] = f"Bearer {token}"
 
     def lease(self, batch_size: int) -> list[dict]:
         r = self.s.post(
@@ -79,62 +23,57 @@ class FcClient:
         return r.json().get("jobs", [])
 
     def submit(self, job_id: int, regions: list[dict], replace_kinds: list[str]) -> dict:
-        return self._submit("/api/gpu/jobs/submit", {
-            "job_id": job_id, "regions": regions, "replace_kinds": replace_kinds,
-        })
-
-    def submit_embedding(self, job_id: int, embedding: list, version: str) -> dict:
-        """Post a whole-image SigLIP embedding (the 'embed' task) → image_record."""
-        return self._submit("/api/gpu/jobs/submit_embedding", {
-            "job_id": job_id, "embedding": embedding, "embedding_version": version,
-        })
+        r = self.s.post(
+            f"{self.base}/api/gpu/jobs/submit",
+            json={
+                "agent_id": self.agent_id, "job_id": job_id,
+                "regions": regions, "replace_kinds": replace_kinds,
+            },
+            timeout=120,
+        )
+        r.raise_for_status()
+        return r.json()
 
     def heartbeat(self, job_ids: list[int]) -> None:
-        self._post_quiet("/api/gpu/jobs/heartbeat", {"job_ids": job_ids})
+        try:
+            self.s.post(
+                f"{self.base}/api/gpu/jobs/heartbeat",
+                json={"agent_id": self.agent_id, "job_ids": job_ids},
+                timeout=30,
+            )
+        except requests.RequestException:
+            pass
 
     def fail(self, job_id: int, error: str) -> None:
-        self._post_quiet("/api/gpu/jobs/fail", {"job_id": job_id, "error": error})
+        try:
+            self.s.post(
+                f"{self.base}/api/gpu/jobs/fail",
+                json={"agent_id": self.agent_id, "job_id": job_id, "error": error},
+                timeout=30,
+            )
+        except requests.RequestException:
+            pass
 
     def release(self, job_ids: list[int]) -> None:
         # Graceful hand-back on stop so orphaned work is re-leased at once.
         if not job_ids:
             return
-        self._post_quiet("/api/gpu/jobs/release", {"job_ids": job_ids})
-
-    def fetch_image(self, image_url: str, throttle=None) -> bytes:
-        # image_url is a server-relative path ("/images/...").
-        # timeout=(connect, read): the read timeout is BETWEEN-BYTES, not total,
-        # so a large-but-flowing download still completes — but a stuck/dead
-        # connection (curator overloaded) fails in 60s instead of hanging a
-        # downloader for 180s and piling up concurrent stuck requests on curator.
-        # With a throttle (the worker's shared TokenBucket), the body is streamed
-        # in chunks and each chunk is charged to the global bandwidth budget —
-        # pausing between reads lets TCP flow control pace curator's send side.
-        with self.s.get(
-            f"{self.base}{image_url}", timeout=(10, 60), stream=throttle is not None
-        ) as r:
-            r.raise_for_status()
-            if throttle is None:
-                return r.content
-            buf = bytearray()
-            for chunk in r.iter_content(chunk_size=262_144):
-                throttle.take(len(chunk))
-                buf.extend(chunk)
-            return bytes(buf)
-
-    def is_reachable(self) -> bool:
-        """Cheap 'is curator responding at all right now?' check. Used to decide,
-        when a video can't be sampled, between a transient outage (keep retrying —
-        survives a redeploy) and an unprocessable file (fail it, don't loop)."""
         try:
-            r = self.s.get(f"{self.base}/api/gpu/status", timeout=5)
-            return r.status_code < 500
+            self.s.post(
+                f"{self.base}/api/gpu/jobs/release",
+                json={"agent_id": self.agent_id, "job_ids": job_ids},
+                timeout=30,
+            )
         except requests.RequestException:
-            return False
+            pass
+
+    def fetch_image(self, image_url: str) -> bytes:
+        # image_url is a server-relative path ("/images/...").
+        r = self.s.get(f"{self.base}{image_url}", timeout=180)
+        r.raise_for_status()
+        return r.content
 
     def queue_status(self) -> dict:
-        # Short timeout: this backs the UI /status poll, so a busy curator must
-        # not hang the page for long (the GPU meters poll /gpu separately).
-        r = self.s.get(f"{self.base}/api/gpu/status", timeout=5)
+        r = self.s.get(f"{self.base}/api/gpu/status", timeout=15)
         r.raise_for_status()
         return r.json()

agent/fc_agent/config.py

@@ -1,18 +1,8 @@
 """Agent config, all from env (the control container is configured at run)."""
-# Lazy annotations so the `from_env(cls) -> Config` self-reference is a string,
-# not evaluated at class-definition time — otherwise it NameErrors on the agent's
-# Python 3.10 (CI lints on 3.14, where PEP 649 hides this).
-from __future__ import annotations
-
 import os
 from dataclasses import dataclass
 
 
-def _bool_env(name: str, default: str = "") -> bool:
-    """A boolean env var — present + truthy ('1'/'true'/'yes') → True."""
-    return os.environ.get(name, default).lower() in ("1", "true", "yes")
-
-
 @dataclass
 class Config:
     fc_url: str          # base URL of the FabledCurator web service
@@ -23,37 +13,9 @@ class Config:
     ccip_model: str      # imgutils CCIP model name ("" → imgutils default)
     detector_level: str  # imgutils person-detector level: n|s|m|x
     poll_idle_seconds: float   # wait between empty leases
-    embed_dtype: str     # torch dtype for the crop embedder: float16|float32
-    embed_model_override: str  # force a SigLIP-family model ("" → use the one
-    #                            the server announces in the lease)
-    auto_start: bool     # start the worker pool on boot (so a container restart
-    #                      resumes processing without anyone clicking Start)
-    auto_scale: bool     # autoscale the worker count (throughput hill-climb)
-    # Crop PROPOSERS (extra YOLO detectors that say where to crop). Each weight
-    # spec is an ultralytics name | http(s) URL | "hf_repo::file" ("" = off).
-    person_weights: str  # general COCO person detector (Western/realistic figs)
-    person_conf: float
-    anatomy_weights: str  # booru_yolo anime/furry/NSFW components
-    anatomy_conf: float
-    panel_weights: str    # comic-panel detector
-    panel_conf: float
-    max_components: int   # cap anatomy component crops per frame
-    max_panels: int       # cap panel crops per frame
-    max_figures: int      # cap figure boxes per frame (each = a CCIP call + crop)
-    max_regions: int      # hard cap on total regions per JOB (submit-size backstop)
-    dedupe_iou: float     # crops overlapping >= this (same kind) are near-dupes,
-    #                       dropped before the embed; >=1.0 disables it
-    frame_dedupe_distance: int  # video frames whose dHash differs by < this many
-    #                             bits are near-duplicates, dropped before detect;
-    #                             higher keeps more frames, 0 disables
-    ffmpeg_timeout: float  # hard ceiling (s) for ffmpeg-from-URL video sampling;
-    #                        generous so a SLOW media link still completes
-    bandwidth_limit_mb_s: float  # aggregate download cap in MEGABYTES/s across
-    #                              all downloaders + video streams (0 = unlimited);
-    #                              tunable live from the agent UI
 
     @classmethod
-    def from_env(cls) -> Config:
+    def from_env(cls) -> "Config":
         return cls(
             fc_url=os.environ.get("FC_URL", "http://localhost:8000").rstrip("/"),
             token=os.environ.get("FC_TOKEN", ""),
@@ -63,28 +25,4 @@ class Config:
             ccip_model=os.environ.get("CCIP_MODEL", ""),
             detector_level=os.environ.get("DETECTOR_LEVEL", "m"),
             poll_idle_seconds=float(os.environ.get("POLL_IDLE_SECONDS", "10")),
-            embed_dtype=os.environ.get("SIGLIP_DTYPE", "float16"),
-            embed_model_override=os.environ.get("EMBED_MODEL_NAME", ""),
-            auto_start=_bool_env("AUTO_START"),
-            auto_scale=_bool_env("AUTO_SCALE", "true"),
-            person_weights=os.environ.get("PERSON_WEIGHTS", "yolo11n.pt"),
-            person_conf=float(os.environ.get("PERSON_CONF", "0.35")),
-            anatomy_weights=os.environ.get("ANATOMY_WEIGHTS", ""),
-            anatomy_conf=float(os.environ.get("ANATOMY_CONF", "0.30")),
-            panel_weights=os.environ.get("PANEL_WEIGHTS", ""),
-            panel_conf=float(os.environ.get("PANEL_CONF", "0.30")),
-            max_components=int(os.environ.get("MAX_COMPONENTS", "8")),
-            max_panels=int(os.environ.get("MAX_PANELS", "8")),
-            max_figures=int(os.environ.get("MAX_FIGURES", "8")),
-            max_regions=int(os.environ.get("MAX_REGIONS", "128")),
-            dedupe_iou=float(os.environ.get("DEDUPE_IOU", "0.85")),
-            frame_dedupe_distance=int(os.environ.get("FRAME_DEDUPE_DISTANCE", "8")),
-            ffmpeg_timeout=float(os.environ.get("FFMPEG_TIMEOUT", "1200")),
-            # Default 8 MB/s (~64 Mbit/s): ~20% of the measured ~300 Mbit/s home
-            # WiFi, so browsing stays snappy while the agent works — yet MORE
-            # sweep throughput than the self-inflicted congestion collapse this
-            # replaces (2026-07-02: 8 unthrottled downloaders bufferbloated the
-            # link to ~1-1.5 MB/s per stream, browser included). Raise it (or 0)
-            # from the agent UI on wired/faster networks.
-            bandwidth_limit_mb_s=float(os.environ.get("BANDWIDTH_LIMIT_MB_S", "8")),
         )

agent/fc_agent/detectors.py

@@ -1,218 +0,0 @@
-"""Region PROPOSERS — small YOLO detectors that decide WHERE to crop. They run
-on the agent GPU and their boxes feed the crop → SigLIP → max-over-bag pipeline:
-
-  - person (general COCO yolo11n): full-figure boxes for realistic / Western art
-    the anime person-detector misses; NMS-merged with imgutils detect_person and
-    fed to CCIP (identity) + a concept crop.
-  - anatomy (booru_yolo): anime / furry / NSFW torso components (head, cat-head,
-    boob, hip, …) — concept crops aligned to the operator's tag vocabulary.
-  - panel (mosesb): a comic page → panel regions → concept crops.
-
-Each proposer is INDEPENDENTLY optional + guarded: a bad weight path or an
-inference error disables just that proposer (logged) and never breaks the
-worker, which still falls back to imgutils detection. Weights resolve from an
-ultralytics builtin name ("yolo11n.pt"), an http(s) URL, or "hf_repo::file" —
-cached under HF_HOME so the download happens once.
-"""
-import logging
-import os
-import threading
-import types
-from pathlib import Path
-
-log = logging.getLogger("fc_agent.detectors")
-_CACHE = Path(os.environ.get("HF_HOME", "/models")) / "yolo"
-
-
-def _resolve(spec: str) -> str | None:
-    """A local weights path (downloading if needed) or an ultralytics builtin
-    name. None if the spec is empty/unresolvable."""
-    if not spec:
-        return None
-    if "::" in spec:                                    # hf_repo::filename
-        repo, _, fname = spec.partition("::")
-        from huggingface_hub import hf_hub_download
-        return hf_hub_download(
-            repo_id=repo, filename=fname, cache_dir=str(_CACHE)
-        )
-    if spec.startswith(("http://", "https://")):
-        _CACHE.mkdir(parents=True, exist_ok=True)
-        dest = _CACHE / spec.rsplit("/", 1)[-1]
-        if not dest.is_file():
-            import requests
-            r = requests.get(spec, timeout=300)
-            r.raise_for_status()
-            dest.write_bytes(r.content)
-        return str(dest)
-    return spec                                         # ultralytics builtin name
-
-
-def _iou(a, b) -> float:
-    ax, ay, aw, ah = a
-    bx, by, bw, bh = b
-    ix = max(0.0, min(ax + aw, bx + bw) - max(ax, bx))
-    iy = max(0.0, min(ay + ah, by + bh) - max(ay, by))
-    inter = ix * iy
-    union = aw * ah + bw * bh - inter
-    return inter / union if union > 0 else 0.0
-
-
-def nms_merge(boxes, iou_thresh: float = 0.6):
-    """Greedy NMS over (bbox_norm, score, label) from possibly several detectors,
-    so the same figure found by two of them collapses to one (higher-score) box."""
-    kept = []
-    for bb, sc, lb in sorted(boxes, key=lambda b: b[1], reverse=True):
-        if all(_iou(bb, k[0]) < iou_thresh for k in kept):
-            kept.append((bb, sc, lb))
-    return kept
-
-
-def dedupe_crops(pending, iou_thresh: float = 0.85):
-    """Greedy high-IoU dedupe over a list of (crop, region_template) pairs, run
-    just before the batched SigLIP embed so we never embed the same region twice.
-
-    Figure boxes are already NMS-merged and each YOLO self-NMSes, but the combined
-    per-frame pile (figure→concept ∪ anatomy component→concept ∪ panel) can still
-    carry genuine near-duplicates across proposers — e.g. a figure box that nearly
-    coincides with an anatomy component on a solo bust, or overlapping booru head
-    classes on one head. Those embed the same region twice, wasting GPU and a slot
-    against max_regions.
-
-    Boxes are compared ONLY within the same output kind and dropped when they
-    overlap at >= iou_thresh, keeping the highest-scoring one. The HIGH default
-    threshold is deliberate: it collapses only true near-identical boxes while
-    preserving intentional nested crops across scopes (a whole figure vs a small
-    head component sit well below it) and distinct kinds (concept vs panel). A
-    value >= 1.0 effectively disables it (nothing but an exact box matches)."""
-    kept = []
-    kept_boxes: dict = {}   # kind -> [bbox, ...] already kept
-    for crop, tmpl in sorted(
-        pending, key=lambda p: p[1].get("score") or 0.0, reverse=True
-    ):
-        bb = tmpl.get("bbox")
-        prior = kept_boxes.setdefault(tmpl.get("kind"), [])
-        if bb is not None and any(_iou(bb, kb) >= iou_thresh for kb in prior):
-            continue
-        prior.append(bb)
-        kept.append((crop, tmpl))
-    return kept
-
-
-class YoloProposer:
-    """One lazily-loaded ultralytics YOLO. detect(image) → [(bbox_norm, score,
-    label)] with bbox normalized (x, y, w, h) in [0,1]. Self-disables on any
-    load/inference failure."""
-
-    def __init__(self, name, weights, conf=0.25, keep_labels=None):
-        self.name = name
-        self._spec = weights
-        self._conf = conf
-        self._keep = [k.lower() for k in keep_labels] if keep_labels else None
-        self._model = None
-        self._ok = True
-        self._lock = threading.Lock()
-
-    def _load(self):
-        if self._model is not None or not self._ok:
-            return
-        with self._lock:
-            if self._model is not None or not self._ok:
-                return
-            try:
-                from ultralytics import YOLO
-                path = _resolve(self._spec)
-                if path is None:
-                    self._ok = False
-                    return
-                self._model = YOLO(path)
-                # Disable ultralytics' load-time Conv+BN fusion. AutoBackend fuses
-                # the graph on the first predict; some checkpoints (yolo11n, the
-                # comic-panel model) crash that step with "'Conv' object has no
-                # attribute 'bn'" (a partially-fused / version-mismatched graph),
-                # which silently disabled those proposers (operator-flagged
-                # 2026-07-01). Unfused inference is correct — only marginally
-                # slower — and this is robust across ultralytics versions; if a
-                # future version ignores the override, the detect() guard below
-                # still self-disables the proposer instead of spamming per image.
-                inner = getattr(self._model, "model", None)
-                if inner is not None:
-                    inner.fuse = types.MethodType(lambda self, *a, **k: self, inner)
-                log.info("detector %s loaded (%s)", self.name, path)
-            except Exception as exc:   # noqa: BLE001
-                log.warning("detector %s disabled (load failed): %s", self.name, exc)
-                self._ok = False
-
-    def detect(self, image):
-        self._load()
-        if self._model is None:
-            return []
-        try:
-            res = self._model.predict(image, conf=self._conf, verbose=False)[0]
-        except Exception as exc:   # noqa: BLE001
-            # Permanently self-disable on the FIRST inference failure rather than
-            # re-throwing (and re-logging) on every image forever — an unfixable
-            # model fault degrades to "this proposer is off", logged once.
-            log.warning("detector %s disabled (inference failed): %s", self.name, exc)
-            self._ok = False
-            self._model = None
-            return []
-        iw, ih = image.size
-        names = getattr(res, "names", None) or {}
-        out = []
-        for b in res.boxes:
-            label = str(names.get(int(b.cls), int(b.cls))).lower()
-            if self._keep is not None and not any(k in label for k in self._keep):
-                continue
-            x0, y0, x1, y1 = (float(v) for v in b.xyxy[0].tolist())
-            out.append((
-                (x0 / iw, y0 / ih, (x1 - x0) / iw, (y1 - y0) / ih),
-                float(b.conf), label,
-            ))
-        return out
-
-
-class Proposers:
-    """The agent's proposer set, built from config. Each detector is optional —
-    an empty weight spec leaves that proposer off."""
-
-    def __init__(self, cfg):
-        self.cfg = cfg
-        self._person = (
-            YoloProposer("person-coco", cfg.person_weights,
-                         conf=cfg.person_conf, keep_labels=["person"])
-            if cfg.person_weights else None
-        )
-        self._anatomy = (
-            YoloProposer("anatomy", cfg.anatomy_weights, conf=cfg.anatomy_conf)
-            if cfg.anatomy_weights else None
-        )
-        self._panel = (
-            YoloProposer("panel", cfg.panel_weights, conf=cfg.panel_conf)
-            if cfg.panel_weights else None
-        )
-
-    def figures(self, image, base_boxes):
-        """Merge imgutils person boxes (base_boxes: [(bbox, score)]) with the
-        general COCO person detector → NMS'd figure boxes [(bbox, score, label)],
-        capped to the highest-scoring max_figures. Uncapped, a busy/huge image
-        (many characters) yields hundreds of boxes → hundreds of per-figure CCIP
-        calls + crops → a 30s+ job and an oversized submit (operator-flagged)."""
-        boxes = [(bb, sc if sc is not None else 1.0, "person") for bb, sc in base_boxes]
-        if self._person is not None:
-            boxes += self._person.detect(image)
-        return nms_merge(boxes)[: self.cfg.max_figures]   # nms_merge is score-desc
-
-    @staticmethod
-    def _top(detector, image, cap: int):
-        """Top-`cap` detections by score from an optional proposer (None → the
-        proposer is off → []). Shared by the anatomy + panel proposers, which
-        differ only in which detector and which cap."""
-        if detector is None:
-            return []
-        return sorted(detector.detect(image), key=lambda b: b[1], reverse=True)[:cap]
-
-    def components(self, image):
-        return self._top(self._anatomy, image, self.cfg.max_components)
-
-    def panels(self, image):
-        return self._top(self._panel, image, self.cfg.max_panels)

agent/fc_agent/embedder.py

@@ -1,77 +0,0 @@
-"""Crop EMBEDDER for the concept bag — model-agnostic (CLIP/SigLIP-family).
-
-The server trains its per-concept heads in the embedding space of whatever model
-its `embedder_model_version` names; a crop must be embedded with the SAME model
-or its vector lands in a different coordinate system and every head misfires. So
-the model identity (HF name + version) is ANNOUNCED BY THE SERVER in the lease —
-nothing here is hardcoded to SigLIP. Whatever name the server sends is loaded via
-transformers `get_image_features` (the CLIP/SigLIP-family image-tower call); a
-non-CLIP backbone (e.g. a DINO encoder) would need its own pooling adapter.
-
-torch on CUDA, fp16 by default to keep VRAM low on a shared desktop GPU — the
-tiny fp16-vs-fp32 difference is negligible for the linear heads (cosine ~0.999).
-A single inference lock serializes the forward pass: the pipeline is I/O-bound,
-so the GPU isn't the bottleneck, and one model shared across worker threads is
-safest behind a lock.
-"""
-import threading
-
-import numpy as np
-from PIL import Image
-
-
-class CropEmbedder:
-    def __init__(self, model_name: str, dtype: str = "float16"):
-        self._name = model_name
-        self._dtype_name = dtype
-        self._model = None
-        self._processor = None
-        self._torch = None
-        self._device = None
-        self._dt = None
-        self._load_lock = threading.Lock()
-        self._infer_lock = threading.Lock()
-
-    @property
-    def model_name(self) -> str:
-        return self._name
-
-    def load(self) -> None:
-        if self._model is not None:
-            return
-        with self._load_lock:
-            if self._model is not None:
-                return
-            import torch
-            from transformers import AutoImageProcessor, AutoModel
-
-            self._torch = torch
-            self._device = "cuda" if torch.cuda.is_available() else "cpu"
-            dt = getattr(torch, self._dtype_name, torch.float16)
-            if self._device == "cpu":
-                dt = torch.float32   # fp16 matmul is unsupported/slow on CPU
-            self._dt = dt
-            self._processor = AutoImageProcessor.from_pretrained(self._name)
-            model = AutoModel.from_pretrained(self._name, torch_dtype=dt)
-            model.eval().to(self._device)
-            self._model = model
-
-    def embed(self, image: Image.Image) -> list[float]:
-        """A crop → its embedding as a plain float list, ready to POST."""
-        return self.embed_batch([image])[0]
-
-    def embed_batch(self, images: list) -> list[list[float]]:
-        """Embed many crops in ONE forward pass — far better GPU utilisation +
-        only one lock acquisition than embedding each crop separately (which
-        starved the GPU and serialised the whole pool)."""
-        if not images:
-            return []
-        self.load()
-        torch = self._torch
-        enc = self._processor(images=images, return_tensors="pt")
-        pixel_values = enc["pixel_values"].to(self._device, self._dt)
-        with self._infer_lock, torch.no_grad():
-            out = self._model.get_image_features(pixel_values=pixel_values)
-        pooled = out.pooler_output if hasattr(out, "pooler_output") else out
-        arr = pooled.float().cpu().numpy().astype(np.float32)
-        return [row.reshape(-1).tolist() for row in arr]

agent/fc_agent/gpu.py

@@ -1,24 +1,10 @@
 """GPU load readout via nvidia-smi (present in the container thanks to the
 NVIDIA Container Toolkit's `utility` capability). Returns None if unavailable —
-the UI just shows n/a (e.g. CPU-fallback run).
-
-Reads are CACHED and de-duplicated: the UI meter polls fast, /status reads it,
-and the autoscaler samples it — if each spawned its own `nvidia-smi` (slow on a
-busy GPU) those blocking subprocesses would pile up in the server's thread pool
-and make the Start/Stop buttons feel dead. So a short TTL serves recent callers
-from cache, and only ONE probe runs at a time (others get the last value)."""
+the UI just shows n/a (e.g. CPU-fallback run)."""
 import subprocess
-import threading
-import time
-
-_TTL = 1.0                       # seconds a sample is reused before re-probing
-_lock = threading.Lock()
-_cache: dict | None = None
-_cache_t = 0.0
-_probing = False
 
 
-def _probe() -> dict | None:
+def read_gpu() -> dict | None:
     try:
         out = subprocess.run(
             [
@@ -42,24 +28,3 @@ def _probe() -> dict | None:
         }
     except (ValueError, IndexError):
         return None
-
-
-def read_gpu(max_age: float = _TTL) -> dict | None:
-    """Latest GPU reading, cached. Serves from cache when fresh; when stale,
-    exactly one caller re-probes while the rest get the last value — so request
-    threads never block behind more than one `nvidia-smi`."""
-    global _cache, _cache_t, _probing
-    now = time.monotonic()
-    with _lock:
-        fresh = _cache is not None and (now - _cache_t) < max_age
-        if fresh or _probing:        # fresh, or a probe is already running
-            return _cache
-        _probing = True
-    try:
-        val = _probe()
-    finally:
-        with _lock:
-            _cache = val
-            _cache_t = time.monotonic()
-            _probing = False
-    return val

agent/fc_agent/logbuf.py

@@ -1,44 +0,0 @@
-"""In-memory log ring buffer so the control UI can show recent agent logs
-(detector loads, job errors, autoscaler decisions, outage back-offs) without
-needing `docker logs`. A bounded deque holds the last N formatted lines; a
-logging.Handler appends to it; the UI polls /logs."""
-import logging
-from collections import deque
-
-LINES: deque[str] = deque(maxlen=400)
-
-
-class RingHandler(logging.Handler):
-    def emit(self, record: logging.LogRecord) -> None:
-        try:
-            LINES.append(self.format(record))
-        except Exception:
-            pass
-
-
-_installed = False
-
-
-def install(level: int = logging.INFO) -> None:
-    """Attach the ring handler to the root logger once. fc_agent module loggers
-    propagate to root, so their records land here."""
-    global _installed
-    if _installed:
-        return
-    _installed = True
-    h = RingHandler()
-    h.setFormatter(
-        logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s", "%H:%M:%S")
-    )
-    root = logging.getLogger()
-    root.addHandler(h)
-    if root.level == logging.NOTSET or root.level > level:
-        root.setLevel(level)
-    # Keep the buffer signal-rich: silence the chatty HTTP/download libs (every
-    # HF model fetch logs per-request) so the console shows agent activity —
-    # detector loads, job errors, autoscale moves — not request spam.
-    for noisy in (
-        "uvicorn.access", "ultralytics", "httpx", "httpcore",
-        "huggingface_hub", "urllib3", "filelock",
-    ):
-        logging.getLogger(noisy).setLevel(logging.WARNING)

agent/fc_agent/media.py

@@ -2,77 +2,17 @@
 (ffmpeg) at the cadence FC sends — so a video becomes a bag of per-frame
 instances, each with a timestamp."""
 import io
-import logging
 import os
-import signal
 import subprocess
 import tempfile
-import time
 
-from PIL import Image, ImageFile
-
-from .throttle import PidReadMeter
-
-log = logging.getLogger("fc_agent.media")
-
-# Load slightly-truncated images (a few missing trailing bytes) instead of
-# raising — matches the server embedder. These are common in scraped libraries
-# and would otherwise fail the job 3× then error (operator-flagged 2026-06-30).
-ImageFile.LOAD_TRUNCATED_IMAGES = True
-
-# Disable PIL's decompression-bomb guard: this is a TRUSTED local library, not an
-# untrusted upload surface, so a legitimately huge image (high-res scans/prints,
-# 90M+ pixels) must load. The default 89M-pixel limit only WARNS, but PIL raises
-# DecompressionBombError at 2× (~179M px) — which would fail those jobs outright
-# (operator-flagged 2026-06-30, images of 90–95M px).
-Image.MAX_IMAGE_PIXELS = None
+from PIL import Image
 
 
 def is_video(mime: str) -> bool:
     return bool(mime) and (mime.startswith("video/") or mime in {"image/gif"})
 
 
-def _dhash(img: Image.Image, size: int = 8) -> int:
-    """Difference hash: compare adjacent pixels of a (size+1 × size) grayscale
-    thumbnail → a `size*size`-bit fingerprint. Cheap (64 comparisons on a 72-px
-    thumbnail) and robust to scaling/compression noise — near-identical frames
-    hash within a few bits, a real scene change moves many."""
-    small = img.convert("L").resize((size + 1, size))
-    px = list(small.getdata())
-    bits = 0
-    for row in range(size):
-        base = row * (size + 1)
-        for col in range(size):
-            bits = (bits << 1) | int(px[base + col] > px[base + col + 1])
-    return bits
-
-
-def dedupe_frames(
-    frames: list[tuple[float, Image.Image]], min_distance: int
-) -> list[tuple[float, Image.Image]]:
-    """Drop visually near-duplicate frames. A near-static video sampled into many
-    frames re-runs the WHOLE detect→CCIP→SigLIP chain on ~identical frames — the
-    dominant video load. Greedy perceptual-hash dedup: keep a frame only if its
-    dHash differs from every already-kept frame by >= min_distance bits (Hamming),
-    so a static run collapses to one frame while genuinely distinct scenes all
-    survive. Order + timestamps preserved. CPU-only (64-bit int XORs), so it runs
-    in the decode stage and spares the GPU the skipped frames entirely.
-
-    min_distance is the coarseness dial: higher keeps more frames (safer for brief
-    localized changes an 8×8 hash can miss), 0 disables. The first frame is always
-    kept (nothing to compare against)."""
-    if min_distance <= 0 or len(frames) <= 1:
-        return frames
-    kept: list[tuple[float, Image.Image]] = []
-    hashes: list[int] = []
-    for t, frame in frames:
-        h = _dhash(frame)
-        if all(bin(h ^ k).count("1") >= min_distance for k in hashes):
-            hashes.append(h)
-            kept.append((t, frame))
-    return kept
-
-
 def to_rgb(img: Image.Image) -> Image.Image:
     """RGB, flattening any transparency onto white first. A naive convert('RGB')
     on a palette-with-transparency image (common for character PNGs on a clear
@@ -92,162 +32,32 @@ def load_image(data: bytes) -> Image.Image:
     return to_rgb(Image.open(io.BytesIO(data)))
 
 
-# ffmpeg reconnect flags — resume a dropped HTTP transfer (a slow/contended media
-# store can cut a long stream) instead of failing the whole job. Relies only on
-# HTTP + Range, which every FC deployment serves → environment-agnostic.
-_RECONNECT = [
-    "-reconnect", "1", "-reconnect_streamed", "1",
-    "-reconnect_on_network_error", "1", "-reconnect_delay_max", "5",
-]
-
-
-def _collect_frames(
-    tmp: str, interval: float, cap: int
+def sample_frames(
+    data: bytes, interval_seconds: float, max_frames: int
 ) -> list[tuple[float, Image.Image]]:
-    out: list[tuple[float, Image.Image]] = []
-    names = sorted(n for n in os.listdir(tmp) if n.startswith("f_"))
-    for i, name in enumerate(names[:cap]):
-        with Image.open(os.path.join(tmp, name)) as im:
-            out.append((round(i * interval, 2), to_rgb(im)))
-    return out
-
-
-def _terminate(proc: subprocess.Popen) -> None:
-    """Stop an ffmpeg cleanly, then hard-kill if it ignores SIGTERM."""
-    try:
-        # A bandwidth-paused (SIGSTOPped) process can't receive SIGTERM until it
-        # resumes — always CONT first so termination is prompt, not queued.
-        proc.send_signal(signal.SIGCONT)
-    except OSError:
-        pass
-    proc.terminate()
-    try:
-        proc.wait(timeout=2)
-    except subprocess.TimeoutExpired:
-        proc.kill()
-        try:
-            proc.wait(timeout=2)
-        except subprocess.TimeoutExpired:
-            pass
-
-
-def _pause(proc: subprocess.Popen, seconds: float, should_stop) -> bool:
-    """SIGSTOP ffmpeg for ~`seconds` of bandwidth debt, staying responsive to
-    Stop. While paused, the kernel socket buffer fills and TCP flow control
-    stalls curator's send side — that's the throttle. SIGCONT is ALWAYS sent
-    before returning. False = a Stop arrived mid-pause."""
-    try:
-        proc.send_signal(signal.SIGSTOP)
-    except OSError:
-        return True                     # already exited — nothing to pause
-    try:
-        end = time.monotonic() + seconds
-        while (left := end - time.monotonic()) > 0:
-            if should_stop and should_stop():
-                return False
-            time.sleep(min(0.5, left))
-        return True
-    finally:
-        try:
-            proc.send_signal(signal.SIGCONT)
-        except OSError:
-            pass
-
-
-def sample_frames_from_url(
-    url: str, interval_seconds: float, max_frames: int,
-    *, headers: str = "", timeout: float = 1200.0, should_stop=None,
-    governor=None,
-) -> tuple[list[tuple[float, Image.Image]], str | None]:
-    """Sample frames by pointing ffmpeg STRAIGHT at the media URL — it Range-reads
-    only the video index + up to max_frames worth of content, so the agent never
-    downloads the whole file (VR/4K originals run 800MB+ and would buffer ~1GB in
-    RAM and get cut off mid-download). Reconnect flags resume a dropped transfer;
-    the timeout is the per-video ceiling (a slow/reconnecting stream can otherwise
-    run for minutes). `should_stop` is polled while ffmpeg runs so a Stop KILLS the
-    subprocess at once — otherwise a downloader stuck in a long decode keeps the
-    agent "working" long after Stop. `governor` (the worker's shared TokenBucket)
-    meters ffmpeg's network reads from outside via /proc/<pid>/io and SIGSTOPs
-    the process into budget, so video streaming honors the same aggregate
-    bandwidth cap as still downloads.
-
-    Returns (frames, reason): frames is empty on failure/stop/timeout, and
-    `reason` then carries the SPECIFIC cause (ffmpeg's stderr tail / timeout) so
-    the caller can put it in the job's error — a bare "no frames" hid a filter
-    bug as "unprocessable" for weeks. None reason on success."""
+    """Extract up to max_frames frames at one-every-interval_seconds via ffmpeg.
+    Returns [(timestamp_seconds, frame)]. Empty on failure (caller falls back)."""
     interval = max(0.5, float(interval_seconds or 4.0))
     cap = max(1, int(max_frames or 64))
-    hdr = ["-headers", headers] if headers else []
-    # select (NOT the fps filter): always keep the FIRST frame, then one per
-    # `interval` seconds of timestamp. fps=1/N emits round(duration/N) frames,
-    # which is ZERO for any clip shorter than ~N/2 seconds — a whole class of
-    # short animation loops failed as "unprocessable" that way (operator-flagged
-    # 2026-07-02: 0.5s/1.75s clips). scale=out_range=full converts limited-range
-    # yuv420p to full range so the mjpeg (jpg) encoder accepts it at default
-    # strictness instead of erroring on "non full-range YUV".
-    vf = (
-        f"select='isnan(prev_selected_t)+gte(t-prev_selected_t\\,{interval})',"
-        "scale=out_range=full"
-    )
     with tempfile.TemporaryDirectory() as tmp:
+        src = os.path.join(tmp, "in")
+        with open(src, "wb") as fh:
+            fh.write(data)
         pattern = os.path.join(tmp, "f_%05d.jpg")
-        cmd = ["ffmpeg", "-nostdin", "-loglevel", "error", *_RECONNECT, *hdr,
-               "-i", url, "-vf", vf, "-fps_mode", "vfr",
-               "-frames:v", str(cap), "-q:v", "3", pattern]
-        # ffmpeg's stderr goes to a file (not a PIPE, which could fill and
-        # deadlock; not DEVNULL, which is how a filter bug hid as "unprocessable"
-        # for weeks) — on failure its tail is logged so the operator can see WHY.
-        errpath = os.path.join(tmp, "stderr.txt")
         try:
-            with open(errpath, "wb") as errf:
-                proc = subprocess.Popen(
-                    cmd, stdin=subprocess.DEVNULL,
-                    stdout=subprocess.DEVNULL, stderr=errf,
-                )
-                meter = PidReadMeter(proc.pid) if governor is not None else None
-                # Poll rather than block, so a Stop (or the per-video timeout) can
-                # kill a slow/wedged ffmpeg promptly instead of waiting it out.
-                start = time.monotonic()
-                while True:
-                    try:
-                        proc.wait(timeout=0.5)
-                        break
-                    except subprocess.TimeoutExpired:
-                        stopped = should_stop and should_stop()
-                        if stopped or (time.monotonic() - start > timeout):
-                            _terminate(proc)
-                            if stopped:
-                                return [], "stopped"
-                            log.warning("ffmpeg timed out after %.0fs: %s",
-                                        timeout, url)
-                            return [], f"ffmpeg timed out after {timeout:.0f}s"
-                        if meter is not None:
-                            read = meter.delta()
-                            if read is None:      # /proc gone → stop governing
-                                meter = None
-                            elif (debt := governor.charge(read)) > 0:
-                                # Over budget: pause ffmpeg until the bucket
-                                # recovers. Pause time counts toward `timeout`
-                                # (it stays the wedge backstop either way).
-                                if not _pause(proc, debt, should_stop):
-                                    _terminate(proc)
-                                    return [], "stopped"
-        except (OSError, ValueError) as exc:
-            return [], f"ffmpeg not runnable: {exc}"
-        frames = _collect_frames(tmp, interval, cap)
-        if not frames:
-            reason = f"ffmpeg exit {proc.returncode}: {_tail(errpath)}"
-            log.warning("ffmpeg produced no frames for %s — %s", url, reason)
-            return [], reason
-        return frames, None
-
-
-def _tail(path: str, limit: int = 300) -> str:
-    """Last `limit` chars of a (stderr) file, flattened — for failure logs."""
-    try:
-        with open(path, "rb") as f:
-            f.seek(0, os.SEEK_END)
-            f.seek(max(0, f.tell() - limit))
-            return f.read().decode("utf-8", "replace").replace("\n", " ").strip()
-    except OSError:
-        return "?"
+            subprocess.run(
+                [
+                    "ffmpeg", "-nostdin", "-loglevel", "error", "-i", src,
+                    "-vf", f"fps=1/{interval}", "-frames:v", str(cap),
+                    "-q:v", "3", pattern,
+                ],
+                check=True, timeout=600,
+            )
+        except (subprocess.SubprocessError, FileNotFoundError):
+            return []
+        out: list[tuple[float, Image.Image]] = []
+        names = sorted(n for n in os.listdir(tmp) if n.startswith("f_"))
+        for i, name in enumerate(names[:cap]):
+            with Image.open(os.path.join(tmp, name)) as im:
+                out.append((round(i * interval, 2), to_rgb(im)))
+        return out

agent/fc_agent/throttle.py

@@ -1,111 +0,0 @@
-"""Global download-bandwidth governor (one token bucket for the whole agent).
-
-The agent lives on someone's desktop and shares that desktop's network —
-typically WiFi, where saturating the link doesn't just slow other apps: it
-bufferbloats the airtime (RTT 21→45ms) and collapses EVERY connection,
-the operator's browser included. Measured 2026-07-02: the idle link moved
-~38 MB/s single-stream, but under the 8-downloader sweep every stream on the
-machine crawled at ~1-1.5 MB/s. So the cap is on the AGGREGATE, not per
-stream: still downloads pump their chunks through take(), and ffmpeg video
-streams — whose sockets live in a subprocess we can't wrap — are metered from
-outside via /proc/<pid>/io and paused (SIGSTOP) into budget using charge()'s
-debt signal; TCP flow control then stalls the sender while ffmpeg sleeps.
-
-Accounting is post-paid (charge the bytes first, then wait out any debt): the
-bytes have already crossed the network by the time we count them, and it means
-a chunk larger than one second of budget can never deadlock the bucket.
-Stdlib-only on purpose — unit-tested in CI, where the agent's ML deps
-don't exist.
-"""
-import threading
-import time
-
-
-class TokenBucket:
-    """Thread-safe token bucket in bytes/second. rate 0 = unlimited.
-
-    `consumed` is the monotonic total of bytes charged (throttled or not) —
-    the worker's rate loop derives the UI's "net MB/s" readout from it.
-    """
-
-    def __init__(self, rate_bytes_per_s: float = 0.0):
-        self._cond = threading.Condition()
-        self._rate = max(0.0, float(rate_bytes_per_s))
-        # Burst = one second of budget: enough that chunked reads stay smooth,
-        # small enough that a burst can't meaningfully lift the average.
-        self._level = self._rate
-        self._stamp = time.monotonic()
-        self.consumed = 0
-
-    @property
-    def rate(self) -> float:
-        return self._rate
-
-    def set_rate(self, rate_bytes_per_s: float) -> None:
-        """Retune live (the UI dial). Waiters re-check immediately, so raising
-        the cap (or lifting it with 0) unblocks a mid-download wait at once."""
-        with self._cond:
-            self._refill_locked()          # settle elapsed time at the OLD rate
-            self._rate = max(0.0, float(rate_bytes_per_s))
-            self._level = min(self._level, self._rate)
-            self._cond.notify_all()
-
-    def _refill_locked(self) -> None:
-        now = time.monotonic()
-        self._level = min(self._rate, self._level + (now - self._stamp) * self._rate)
-        self._stamp = now
-
-    def take(self, n: int) -> None:
-        """Charge n bytes and block until the budget recovers (stills path)."""
-        with self._cond:
-            self.consumed += n
-            if self._rate <= 0:
-                return
-            self._refill_locked()
-            self._level -= n
-            while self._level < 0:
-                # Wake early on set_rate; cap the wait so a big debt is paid in
-                # re-checked slices rather than one long uninterruptible sleep.
-                self._cond.wait(min(-self._level / self._rate, 0.5))
-                if self._rate <= 0:
-                    return
-                self._refill_locked()
-
-    def charge(self, n: int) -> float:
-        """Charge n bytes WITHOUT blocking; return seconds of debt (0 = within
-        budget). The ffmpeg governor can't block the subprocess's own reads, so
-        it SIGSTOPs the process for (about) the returned debt instead."""
-        with self._cond:
-            self.consumed += n
-            if self._rate <= 0:
-                return 0.0
-            self._refill_locked()
-            self._level -= n
-            return max(0.0, -self._level / self._rate)
-
-
-class PidReadMeter:
-    """Cumulative read-bytes meter for a subprocess, via /proc/<pid>/io.
-
-    `rchar` counts every read() syscall's bytes — for a streaming ffmpeg the
-    network reads dominate, so the delta is a good-enough aggregate-bandwidth
-    signal (it's a governor, not a billing meter). Returns None when /proc is
-    unavailable (process exited, or a non-Linux host): the caller then simply
-    doesn't govern — degrade to unthrottled rather than break video sampling.
-    """
-
-    def __init__(self, pid: int):
-        self._path = f"/proc/{pid}/io"
-        self._last = 0
-
-    def delta(self) -> int | None:
-        try:
-            with open(self._path, "rb") as f:
-                for line in f:
-                    if line.startswith(b"rchar:"):
-                        total = int(line.split()[1])
-                        d, self._last = total - self._last, total
-                        return max(0, d)
-        except (OSError, ValueError):
-            return None
-        return None

agent/requirements.txt

@@ -3,13 +3,6 @@ dghs-imgutils>=0.4
 # GPU inference for the ONNX models. Swap to onnxruntime (CPU) for a slow
 # server-side fallback run.
 onnxruntime-gpu
-# The crop EMBEDDER (concept bag). torch is installed separately in the
-# Dockerfile from the CUDA-12.4 wheel index so the GPU build is deterministic;
-# transformers loads whatever SigLIP-family model the server announces.
-transformers>=4.45
-# Crop PROPOSERS — small YOLO detectors (booru_yolo anatomy, COCO person, comic
-# panel) that decide where to crop. Uses the torch already installed above.
-ultralytics>=8.3
 # Control surface + HTTP.
 fastapi
 uvicorn[standard]

agent/ruff.toml

@@ -1,7 +0,0 @@
-# The agent runs on the CUDA base image's Python 3.12 (Ubuntu 24.04) — NOT the
-# 3.14 that CI's ci-python image and the repo-root ruff.toml target. Pin the
-# agent to py312 so ruff enforces 3.12 compatibility and never auto-applies a
-# 3.14-only fix (e.g. unquoting a self-referential annotation, which PEP 649
-# makes safe on 3.14 but NameErrors on 3.12). Inherit the root lint rules.
-extend = "../ruff.toml"
-target-version = "py312"

alembic/versions/0063_ccip_match_threshold.py

@@ -1,33 +0,0 @@
-"""ml_settings.ccip_match_threshold — tunable CCIP character-match cut (#114)
-
-The v1 matcher used a flat 0.75 cosine; live data showed that over-fires (a
-high-reference character matched a scatter of images). 0.85 keeps the confident
-single-character matches and drops the noise. Tunable from the GPU agent card.
-
-Revision ID: 0063
-Revises: 0062
-Create Date: 2026-06-29
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0063"
-down_revision: Union[str, None] = "0062"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "ccip_match_threshold", sa.Float(), nullable=False,
-            server_default="0.85",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "ccip_match_threshold")

alembic/versions/0064_ccip_auto_apply.py

@@ -1,42 +0,0 @@
-"""ml_settings: CCIP auto-apply switch + threshold (#114)
-
-Confident CCIP character matches auto-tag (source='ccip_auto') on a daily sweep,
-so identity tags keep flowing without pressing a button. ON by default (opt-out,
-like head auto-apply); the high threshold (0.92, above the 0.85 suggest cut) +
-single-character references keep it safe, and every auto-tag is reversible.
-
-Revision ID: 0064
-Revises: 0063
-Create Date: 2026-06-30
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0064"
-down_revision: Union[str, None] = "0063"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "ccip_auto_apply_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.true(),
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "ccip_auto_apply_threshold", sa.Float(), nullable=False,
-            server_default="0.92",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "ccip_auto_apply_threshold")
-    op.drop_column("ml_settings", "ccip_auto_apply_enabled")

alembic/versions/0065_embedder_model_name.py

@@ -1,35 +0,0 @@
-"""ml_settings: embedder_model_name (#1190 operator model swap)
-
-The embedder MODEL VERSION was already a setting (and stamps image_record.
-siglip_model_version); the HF model NAME was env-only, so an operator couldn't
-actually point the pipeline at a different embedder. Storing the name as a
-setting makes the model an operator choice: set name + version → re-embed (the
-GPU agent) → retrain heads. Default = the current SigLIP so400m.
-
-Revision ID: 0065
-Revises: 0064
-Create Date: 2026-06-30
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0065"
-down_revision: Union[str, None] = "0064"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "embedder_model_name", sa.String(length=128), nullable=False,
-            server_default="google/siglip-so400m-patch14-384",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "embedder_model_name")

alembic/versions/0066_drop_centroids.py

@@ -1,57 +0,0 @@
-"""drop the dead per-tag centroid subsystem (#1189 cleanup)
-
-The v2 pivot replaced per-tag SigLIP centroids with learned heads + CCIP.
-Nothing read the centroids anymore — they were recomputed (on merge + a daily
-beat) but never consumed for suggestions or auto-apply. Remove the storage +
-its two now-unused settings columns. (The recompute tasks, beat, endpoint,
-service, and UI card are removed in the same change.)
-
-Revision ID: 0066
-Revises: 0065
-Create Date: 2026-06-30
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0066"
-down_revision: Union[str, None] = "0065"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.drop_table("tag_reference_embedding")
-    op.drop_column("ml_settings", "centroid_similarity_threshold")
-    op.drop_column("ml_settings", "min_reference_images")
-
-
-def downgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "min_reference_images", sa.Integer(), nullable=False,
-            server_default="5",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "centroid_similarity_threshold", sa.Float(), nullable=False,
-            server_default="0.55",
-        ),
-    )
-    op.create_table(
-        "tag_reference_embedding",
-        sa.Column("tag_id", sa.Integer(), nullable=False),
-        sa.Column("embedding", sa.LargeBinary(), nullable=False),
-        sa.Column("reference_count", sa.Integer(), nullable=False),
-        sa.Column("model_version", sa.String(length=128), nullable=False),
-        sa.Column(
-            "updated_at", sa.DateTime(timezone=True),
-            server_default=sa.func.now(), nullable=False,
-        ),
-        sa.ForeignKeyConstraint(["tag_id"], ["tag.id"], ondelete="CASCADE"),
-        sa.PrimaryKeyConstraint("tag_id"),
-    )

alembic/versions/0067_retire_camie_allowlist.py

@@ -1,66 +0,0 @@
-"""retire the Camie tagger + allowlist bulk-apply (#1189)
-
-The v2 pivot made heads + CCIP the tag source and head auto-apply the earned
-propagation. The Camie tagger ran only to feed the allowlist bulk-apply (its
-predictions had no other consumer), and the allowlist was a second, un-earned
-auto-apply path parallel to heads. Both are retired — drop their storage.
-
-(image_prediction = Camie's per-image predictions; tag_allowlist = the bulk-
-apply allowlist. Nothing references INTO these tables, so the drop is clean.)
-
-Revision ID: 0067
-Revises: 0066
-Create Date: 2026-06-30
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0067"
-down_revision: Union[str, None] = "0066"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.drop_table("image_prediction")
-    op.drop_table("tag_allowlist")
-
-
-def downgrade() -> None:
-    op.create_table(
-        "tag_allowlist",
-        sa.Column("tag_id", sa.Integer(), nullable=False),
-        sa.Column(
-            "min_confidence", sa.Float(), nullable=False, server_default="0.9"
-        ),
-        sa.Column(
-            "created_at", sa.DateTime(timezone=True),
-            server_default=sa.func.now(), nullable=False,
-        ),
-        sa.ForeignKeyConstraint(["tag_id"], ["tag.id"], ondelete="CASCADE"),
-        sa.PrimaryKeyConstraint("tag_id"),
-        sa.CheckConstraint(
-            "min_confidence >= 0 AND min_confidence <= 1",
-            name="ck_tag_allowlist_confidence_range",
-        ),
-    )
-    op.create_table(
-        "image_prediction",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column("image_record_id", sa.Integer(), nullable=False),
-        sa.Column("raw_name", sa.String(length=255), nullable=False),
-        sa.Column("category", sa.String(length=32), nullable=False),
-        sa.Column("score", sa.Float(), nullable=False),
-        sa.ForeignKeyConstraint(
-            ["image_record_id"], ["image_record.id"], ondelete="CASCADE"
-        ),
-    )
-    op.create_index(
-        "ix_image_prediction_image", "image_prediction", ["image_record_id"]
-    )
-    op.create_index(
-        "ix_image_prediction_name_score", "image_prediction",
-        ["raw_name", "score"],
-    )

alembic/versions/0068_drop_dead_tagger_settings.py

@@ -1,80 +0,0 @@
-"""drop dead tagger/suggestion settings + columns left after Camie retirement (#1199)
-
-Hygiene follow-up to #1189. These were left inert to bound that change; nothing
-reads them now:
-- ml_settings: tagger_store_floor + tagger_model_version (only the deleted Camie
-  tagger used them), suggestion_threshold_character/general (already dead pre-
-  retirement — scoring uses per-head thresholds), video_min_tag_frames (only the
-  deleted video-prediction aggregator used it).
-- image_record: tagger_model_version (no writer now), centroid_scores (long-dead
-  JSON cache, no reader).
-
-Revision ID: 0068
-Revises: 0067
-Create Date: 2026-06-30
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0068"
-down_revision: Union[str, None] = "0067"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.drop_column("ml_settings", "suggestion_threshold_character")
-    op.drop_column("ml_settings", "suggestion_threshold_general")
-    op.drop_column("ml_settings", "tagger_store_floor")
-    op.drop_column("ml_settings", "video_min_tag_frames")
-    op.drop_column("ml_settings", "tagger_model_version")
-    op.drop_column("image_record", "tagger_model_version")
-    op.drop_column("image_record", "centroid_scores")
-
-
-def downgrade() -> None:
-    op.add_column(
-        "image_record",
-        sa.Column("centroid_scores", sa.JSON(), nullable=True),
-    )
-    op.add_column(
-        "image_record",
-        sa.Column("tagger_model_version", sa.String(length=128), nullable=True),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "tagger_model_version", sa.String(length=128), nullable=False,
-            server_default="camie-tagger-v2",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "video_min_tag_frames", sa.Integer(), nullable=False,
-            server_default="3",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "tagger_store_floor", sa.Float(), nullable=False,
-            server_default="0.7",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "suggestion_threshold_general", sa.Float(), nullable=False,
-            server_default="0.7",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "suggestion_threshold_character", sa.Float(), nullable=False,
-            server_default="0.7",
-        ),
-    )

alembic/versions/0069_default_siglip2.py

@@ -1,51 +0,0 @@
-"""default the embedder to SigLIP 2 — for FRESH installs only (#1203)
-
-Make SigLIP 2 (so400m, 512px; a 1152-d drop-in) the default embedder. New
-installs start on it. An EXISTING library is NOT touched: flipping its stored
-embedder version would mark every embedding stale (the scorer is version-gated)
-and kill suggestions until a full re-embed+retrain — so an existing instance
-switches deliberately via Settings → GPU agent → Embedding model → Re-embed →
-Retrain. We detect "fresh" by the absence of any embedded image.
-
-Revision ID: 0069
-Revises: 0068
-Create Date: 2026-06-30
-"""
-from typing import Sequence, Union
-
-from alembic import op
-
-revision: str = "0069"
-down_revision: Union[str, None] = "0068"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-_NEW_NAME = "google/siglip2-so400m-patch16-512"
-_NEW_VERSION = "siglip2-so400m-patch16-512"
-_OLD_NAME = "google/siglip-so400m-patch14-384"
-_OLD_VERSION = "siglip-so400m-patch14-384"
-
-
-def upgrade() -> None:
-    # Fresh install (nothing embedded yet) → adopt SigLIP 2.
-    op.execute(
-        f"""
-        UPDATE ml_settings SET
-            embedder_model_name = '{_NEW_NAME}',
-            embedder_model_version = '{_NEW_VERSION}'
-        WHERE NOT EXISTS (
-            SELECT 1 FROM image_record WHERE siglip_embedding IS NOT NULL
-        )
-        """
-    )
-    op.alter_column("ml_settings", "embedder_model_name", server_default=_NEW_NAME)
-    op.alter_column(
-        "ml_settings", "embedder_model_version", server_default=_NEW_VERSION
-    )
-
-
-def downgrade() -> None:
-    op.alter_column("ml_settings", "embedder_model_name", server_default=_OLD_NAME)
-    op.alter_column(
-        "ml_settings", "embedder_model_version", server_default=_OLD_VERSION
-    )

alembic/versions/0070_gpu_job_lease_indexes.py

@@ -1,44 +0,0 @@
-"""partial indexes so GPU-job leasing stays O(batch), not O(completed)
-
-The lease claims the lowest-id pending (or expired-leased) jobs. With only a
-plain `status` index, `... ORDER BY id LIMIT n` walked the primary-key index from
-the start, skipping the entire prefix of already-done/error rows before reaching
-pending ones — so leasing slowed to a crawl as `done` piled up (the whole reason
-throughput fell off a cliff mid-run and /status stalled). Two partial indexes fix
-it: the pending one is id-ordered so the hot path reads just the first n entries,
-and the leased-expiry one keeps the crash-recovery reclaim + the orphan sweep
-cheap. They cover only the small live slice of the table, so they stay tiny even
-as the done/error history grows to millions.
-
-Revision ID: 0070
-Revises: 0069
-Create Date: 2026-06-30
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0070"
-down_revision: Union[str, None] = "0069"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # Hot path: lowest-id pending jobs. Index on id, restricted to pending, so
-    # `WHERE status='pending' ORDER BY id LIMIT n` is a short index-order scan.
-    op.create_index(
-        "ix_gpu_job_pending", "gpu_job", ["id"],
-        postgresql_where=sa.text("status = 'pending'"),
-    )
-    # Crash-recovery: expired leases, for the lease backstop + recover_orphaned.
-    op.create_index(
-        "ix_gpu_job_leased_expires", "gpu_job", ["lease_expires_at"],
-        postgresql_where=sa.text("status = 'leased'"),
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_gpu_job_leased_expires", table_name="gpu_job")
-    op.drop_index("ix_gpu_job_pending", table_name="gpu_job")

alembic/versions/0071_image_record_earliest_post_date.py

@@ -1,80 +0,0 @@
-"""image_record.earliest_post_date: original-publish gallery sort key + index
-
-Revision ID: 0071
-Revises: 0070
-Create Date: 2026-07-01
-
-effective_date (0035) keys off the PRIMARY post — which is often the repost /
-download the file actually came from — and falls back to created_at, so the
-gallery's default order surfaces download dates rather than when content was
-first posted (operator-flagged 2026-07-01). Materialize a second sort key,
-earliest_post_date = MIN(post_date) across ALL of an image's provenance posts
-(every post it appears in), falling back to created_at only when no linked post
-carries a date. Indexed (DESC, id DESC) so the "post date" gallery sort is an
-index range scan just like effective_date.
-
-Backfill mirrors 0035: created_at baseline, then override with the MIN over
-image_provenance ⋈ post. New rows get the created_at-equivalent server default;
-services/importer.py recomputes it whenever a dated post is linked.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0071"
-down_revision: Union[str, None] = "0070"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # Add nullable first so the backfill can populate before NOT NULL.
-    op.add_column(
-        "image_record",
-        sa.Column("earliest_post_date", sa.DateTime(timezone=True), nullable=True),
-    )
-    # Baseline: download date. Set-based (no per-row binds) → immune to the
-    # 65535 bind-parameter ceiling regardless of library size.
-    op.execute(
-        """
-        UPDATE image_record
-        SET earliest_post_date = created_at
-        """
-    )
-    # Override with the earliest post_date across EVERY post the image appears
-    # in (image_provenance is the many-to-many edge; ignore posts with no date).
-    op.execute(
-        """
-        UPDATE image_record AS ir
-        SET earliest_post_date = sub.min_date
-        FROM (
-            SELECT ip.image_record_id AS iid, MIN(p.post_date) AS min_date
-            FROM image_provenance AS ip
-            JOIN post AS p ON p.id = ip.post_id
-            WHERE p.post_date IS NOT NULL
-            GROUP BY ip.image_record_id
-        ) AS sub
-        WHERE ir.id = sub.iid
-        """
-    )
-    op.alter_column(
-        "image_record",
-        "earliest_post_date",
-        nullable=False,
-        server_default=sa.text("now()"),
-    )
-    # DESC/DESC matches the gallery's ORDER BY earliest_post_date DESC, id DESC
-    # so the "post date" scroll is a forward index scan; raw SQL because
-    # alembic's column list doesn't express per-column DESC cleanly.
-    op.execute(
-        "CREATE INDEX ix_image_record_earliest_post_date "
-        "ON image_record (earliest_post_date DESC, id DESC)"
-    )
-
-
-def downgrade() -> None:
-    op.drop_index(
-        "ix_image_record_earliest_post_date", table_name="image_record"
-    )
-    op.drop_column("image_record", "earliest_post_date")

alembic/versions/0072_gpu_job_triage_status.py

@@ -1,32 +0,0 @@
-"""gpu_job.triage_status — the probe's verdict on an errored job's FILE
-
-Failure triage (#125): a periodic sweep probes each errored image's file
-(sha256 + decode, verify_integrity's machinery) exactly once and stores the
-verdict here — 'defect' (the file is bad: recovery material, excluded from
-/retry_errors) or 'file_ok' (failure was operational, safe to retry). NULL
-means not yet probed; selecting on NULL is what makes the sweep resumable.
-No index: the errored slice the sweep scans is tiny by design (tombstones).
-
-Revision ID: 0072
-Revises: 0071
-Create Date: 2026-07-02
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0072"
-down_revision: Union[str, None] = "0071"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "gpu_job", sa.Column("triage_status", sa.String(16), nullable=True)
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("gpu_job", "triage_status")

alembic/versions/0073_drop_tag_eval_run.py

@@ -1,46 +0,0 @@
-"""drop tag_eval_run — the head-vs-centroid eval harness is retired
-
-The eval (#1130) existed to prove the heads tagging spine on the operator's own
-data. It did; the operator accepted the system and retired the harness
-(2026-07-02) — card, API, task, model and this table all go. The eval's data
-loaders + metric helpers live on in services/ml/training_data.py, where the
-production heads trainer uses them nightly.
-
-Revision ID: 0073
-Revises: 0072
-Create Date: 2026-07-02
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-from sqlalchemy.dialects import postgresql
-
-revision: str = "0073"
-down_revision: Union[str, None] = "0072"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.drop_index("ix_tag_eval_run_status", table_name="tag_eval_run")
-    op.drop_table("tag_eval_run")
-
-
-def downgrade() -> None:
-    # Recreates the shape from 0056 (data is not restorable).
-    op.create_table(
-        "tag_eval_run",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column("params", postgresql.JSONB(), nullable=False),
-        sa.Column("status", sa.String(length=16), nullable=False,
-                  server_default="running"),
-        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("report", postgresql.JSONB(), nullable=True),
-        sa.Column("error", sa.Text(), nullable=True),
-        sa.Column("last_progress_at", sa.DateTime(timezone=True),
-                  nullable=True),
-    )
-    op.create_index("ix_tag_eval_run_status", "tag_eval_run", ["status"])

alembic/versions/0074_ml_settings_cpu_embed_enabled.py

@@ -1,35 +0,0 @@
-"""ml_settings.cpu_embed_enabled — the CPU embed fallback becomes a switch
-
-B3 (operator 2026-07-02): the ml-worker's only processing role is the CPU
-whole-image embed for stacks without a GPU agent. ON by default (a fresh
-install works agent-less); agent-equipped stacks that drop the ml-worker
-container turn it off so import hooks stop queueing embed work into a queue
-nothing consumes — the daily GPU 'embed' backfill covers those images.
-
-Revision ID: 0074
-Revises: 0073
-Create Date: 2026-07-02
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0074"
-down_revision: Union[str, None] = "0073"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "cpu_embed_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.true(),
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "cpu_embed_enabled")

alembic/versions/0075_tag_is_system.py

@@ -1,60 +0,0 @@
-"""tag.is_system + seed the three hygiene system tags
-
-Training hygiene (operator 2026-07-03, milestone #128): rough WIPs tagged as a
-character poison that character's head and CCIP references; banners/editor
-screenshots pollute whole-image similarity. The fix keys on SYSTEM tags the
-product ships — not operator configuration — so the seed lives here.
-
-Seeding ADOPTS an existing same-(name, kind=general) tag (case-insensitive,
-matching TagService.rename's collision stance) instead of inserting a
-duplicate, so an operator who already tagged `wip` keeps their applications.
-
-Revision ID: 0075
-Revises: 0074
-Create Date: 2026-07-03
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0075"
-down_revision: Union[str, None] = "0074"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot")
-
-
-def upgrade() -> None:
-    op.add_column(
-        "tag",
-        sa.Column(
-            "is_system", sa.Boolean(), nullable=False,
-            server_default=sa.false(),
-        ),
-    )
-    conn = op.get_bind()
-    for name in SYSTEM_TAG_NAMES:
-        adopted = conn.execute(
-            sa.text(
-                "UPDATE tag SET is_system = true "
-                "WHERE lower(name) = lower(:name) AND kind = 'general'"
-            ),
-            {"name": name},
-        )
-        if adopted.rowcount == 0:
-            conn.execute(
-                sa.text(
-                    "INSERT INTO tag (name, kind, is_system) "
-                    "VALUES (:name, 'general', true)"
-                ),
-                {"name": name},
-            )
-
-
-def downgrade() -> None:
-    # The seeded rows survive as ordinary general tags — dropping the flag is
-    # enough to disarm the mechanism, and deleting rows would orphan any
-    # operator applications made while the flag existed.
-    op.drop_column("tag", "is_system")

alembic/versions/0076_pixiv_ledgers.py

@@ -1,82 +0,0 @@
-"""pixiv_seen_media + pixiv_failed_media: per-source ledgers
-
-Revision ID: 0076
-Revises: 0075
-Create Date: 2026-07-03
-
-Pixiv native ingester (milestone #129, gallery-dl → native-core migration).
-Mirrors the Patreon (0037/0038) and SubscribeStar (0054) ledger tables: a
-seen-ledger so routine walks skip already-ingested media (recovery bypasses
-it) and a dead-letter ledger so persistently-failing media stops re-burning
-backfill chunks. Pixiv URLs carry no content hash, so `filehash` is always the
-synthesized ``<illust_id>:p<num>`` / ``<illust_id>:ugoira`` key — String(128)
-matches the siblings. UNIQUE (source_id, filehash) is the upsert key on each.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0076"
-down_revision: Union[str, None] = "0075"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "pixiv_seen_media",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "source_id",
-            sa.Integer,
-            sa.ForeignKey("source.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column("filehash", sa.String(128), nullable=False),
-        sa.Column("post_id", sa.String(64), nullable=True),
-        sa.Column(
-            "seen_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-        sa.UniqueConstraint(
-            "source_id", "filehash", name="uq_pixiv_seen_media_source_id"
-        ),
-    )
-    op.create_table(
-        "pixiv_failed_media",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "source_id",
-            sa.Integer,
-            sa.ForeignKey("source.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column("filehash", sa.String(128), nullable=False),
-        sa.Column("attempts", sa.Integer, nullable=False, server_default="1"),
-        sa.Column("last_error", sa.Text, nullable=True),
-        sa.Column(
-            "first_failed_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-        sa.Column(
-            "last_failed_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-        sa.UniqueConstraint(
-            "source_id", "filehash", name="uq_pixiv_failed_media_source_id"
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_table("pixiv_failed_media")
-    op.drop_table("pixiv_seen_media")

alembic/versions/0077_artist_name_not_unique.py

@@ -1,32 +0,0 @@
-"""drop uq_artist_name — decouple display name from identity/storage
-
-Revision ID: 0077
-Revises: 0076
-Create Date: 2026-07-04
-
-Artist model fragility fix (milestone #130). One `slug` column was doing
-identity + storage-path + display, and BOTH `name` and `slug` were UNIQUE, so
-the display name couldn't be edited freely and two genuinely different creators
-collided. Decouple: `slug` stays the immutable, unique storage/identity key (the
-on-disk path component — untouched here); `name` becomes freely editable, NON-
-unique display text. This migration only drops the `uq_artist_name` constraint;
-no data moves and no path changes.
-"""
-from typing import Sequence, Union
-
-from alembic import op
-
-revision: str = "0077"
-down_revision: Union[str, None] = "0076"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.drop_constraint("uq_artist_name", "artist", type_="unique")
-
-
-def downgrade() -> None:
-    # Re-adding the UNIQUE would fail if duplicate names now exist; callers that
-    # need to reverse this must dedupe names first.
-    op.create_unique_constraint("uq_artist_name", "artist", ["name"])

alembic/versions/0078_ml_settings_detectors.py

@@ -1,83 +0,0 @@
-"""ml_settings crop-proposer / detector config (#134)
-
-Move the WHERE-to-crop detector config (per-proposer enable + weights + conf,
-plus caps + dedupe IoU) into the DB so it's UI-tunable and announced to the GPU
-agent in the lease (like the embedder model) — no restart, agent env is now
-bootstrap-only. All server_defaults are the working values so existing rows +
-fresh installs crop out-of-the-box with all three proposers ON.
-
-Revision ID: 0078
-Revises: 0077
-Create Date: 2026-07-05
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0078"
-down_revision: Union[str, None] = "0077"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-_ANATOMY_DEFAULT = (
-    "https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt"
-)
-_PANEL_DEFAULT = "mosesb/best-comic-panel-detection::best.pt"
-
-
-def upgrade() -> None:
-    op.add_column("ml_settings", sa.Column(
-        "detector_person_enabled", sa.Boolean(), nullable=False,
-        server_default=sa.true()))
-    op.add_column("ml_settings", sa.Column(
-        "detector_person_weights", sa.String(512), nullable=False,
-        server_default="yolo11n.pt"))
-    op.add_column("ml_settings", sa.Column(
-        "detector_person_conf", sa.Float(), nullable=False,
-        server_default=sa.text("0.35")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_anatomy_enabled", sa.Boolean(), nullable=False,
-        server_default=sa.true()))
-    op.add_column("ml_settings", sa.Column(
-        "detector_anatomy_weights", sa.String(512), nullable=False,
-        server_default=_ANATOMY_DEFAULT))
-    op.add_column("ml_settings", sa.Column(
-        "detector_anatomy_conf", sa.Float(), nullable=False,
-        server_default=sa.text("0.30")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_panel_enabled", sa.Boolean(), nullable=False,
-        server_default=sa.true()))
-    op.add_column("ml_settings", sa.Column(
-        "detector_panel_weights", sa.String(512), nullable=False,
-        server_default=_PANEL_DEFAULT))
-    op.add_column("ml_settings", sa.Column(
-        "detector_panel_conf", sa.Float(), nullable=False,
-        server_default=sa.text("0.30")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_max_figures", sa.Integer(), nullable=False,
-        server_default=sa.text("8")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_max_components", sa.Integer(), nullable=False,
-        server_default=sa.text("8")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_max_panels", sa.Integer(), nullable=False,
-        server_default=sa.text("8")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_max_regions", sa.Integer(), nullable=False,
-        server_default=sa.text("128")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_dedupe_iou", sa.Float(), nullable=False,
-        server_default=sa.text("0.85")))
-
-
-def downgrade() -> None:
-    for col in (
-        "detector_person_enabled", "detector_person_weights", "detector_person_conf",
-        "detector_anatomy_enabled", "detector_anatomy_weights", "detector_anatomy_conf",
-        "detector_panel_enabled", "detector_panel_weights", "detector_panel_conf",
-        "detector_max_figures", "detector_max_components", "detector_max_panels",
-        "detector_max_regions", "detector_dedupe_iou",
-    ):
-        op.drop_column("ml_settings", col)

alembic/versions/0079_character_prototypes.py

@@ -1,77 +0,0 @@
-"""character prototype store (#1317) — precomputed, incremental CCIP references
-
-New tables character_prototype + ccip_prototype_state, plus MLSettings columns
-ccip_ref_signature (cheap global change gate) + ccip_prototype_cap (per-character
-reference cap). The reference set the CCIP matcher uses becomes a precomputed
-artifact refreshed incrementally off the request path. See milestone 138 /
-backend.app.services.ml.character_prototypes.
-
-Revision ID: 0079
-Revises: 0078
-Create Date: 2026-07-06
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-from pgvector.sqlalchemy import Vector
-
-revision: str = "0079"
-down_revision: Union[str, None] = "0078"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-# Matches models.image_region.CCIP_DIM (the CCIP figure-embedding width).
-_CCIP_DIM = 768
-
-
-def upgrade() -> None:
-    op.create_table(
-        "character_prototype",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column(
-            "tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="CASCADE"), nullable=False,
-        ),
-        sa.Column("ccip_embedding", Vector(_CCIP_DIM), nullable=False),
-        sa.Column(
-            "region_id", sa.Integer(),
-            sa.ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True,
-        ),
-    )
-    op.create_index(
-        "ix_character_prototype_tag_id", "character_prototype", ["tag_id"]
-    )
-    op.create_table(
-        "ccip_prototype_state",
-        sa.Column(
-            "tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True,
-        ),
-        sa.Column("fingerprint", sa.String(64), nullable=False),
-        sa.Column(
-            "updated_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column("ccip_ref_signature", sa.String(128), nullable=True),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "ccip_prototype_cap", sa.Integer(), nullable=False,
-            server_default=sa.text("64"),
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "ccip_prototype_cap")
-    op.drop_column("ml_settings", "ccip_ref_signature")
-    op.drop_table("ccip_prototype_state")
-    op.drop_index(
-        "ix_character_prototype_tag_id", table_name="character_prototype"
-    )
-    op.drop_table("character_prototype")

alembic/versions/0080_tag_head_train_fingerprint.py

@@ -1,31 +0,0 @@
-"""tag_head.train_fingerprint (#1317 phase 2) — incremental head retraining
-
-A per-head training-data fingerprint (positive + rejection count/latest-timestamp)
-so a manual Retrain refits only the tags whose data changed; the nightly run
-ignores it (full reconcile). Nullable — a NULL fingerprint (existing heads) forces
-a refit on the first incremental run, then it's stamped.
-
-Revision ID: 0080
-Revises: 0079
-Create Date: 2026-07-06
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0080"
-down_revision: Union[str, None] = "0079"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "tag_head",
-        sa.Column("train_fingerprint", sa.String(128), nullable=True),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("tag_head", "train_fingerprint")

alembic/versions/0081_stricter_auto_apply_defaults.py

@@ -1,43 +0,0 @@
-"""stricter auto-apply defaults (milestone 139) — cut auto-apply misfires
-
-head_auto_apply_min_positives 30→50 and ccip_auto_apply_threshold 0.92→0.95
-(operator-asked 2026-07-06). The head graduation precision bar stays 0.97 — the
-operator confirmed the general-tag confidence was already well tuned; only the
-support floor + the CCIP match confidence are raised. The model defaults change
-for fresh installs; here we bump the existing singleton row IFF it is still at
-the previous default, so a deliberate operator change is NOT clobbered.
-
-Revision ID: 0081
-Revises: 0080
-Create Date: 2026-07-06
-"""
-from typing import Sequence, Union
-
-from alembic import op
-
-revision: str = "0081"
-down_revision: Union[str, None] = "0080"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.execute(
-        "UPDATE ml_settings SET head_auto_apply_min_positives = 50 "
-        "WHERE head_auto_apply_min_positives = 30"
-    )
-    op.execute(
-        "UPDATE ml_settings SET ccip_auto_apply_threshold = 0.95 "
-        "WHERE ccip_auto_apply_threshold = 0.92"
-    )
-
-
-def downgrade() -> None:
-    op.execute(
-        "UPDATE ml_settings SET head_auto_apply_min_positives = 30 "
-        "WHERE head_auto_apply_min_positives = 50"
-    )
-    op.execute(
-        "UPDATE ml_settings SET ccip_auto_apply_threshold = 0.92 "
-        "WHERE ccip_auto_apply_threshold = 0.95"
-    )

alembic/versions/0082_presentation_auto_hide.py

@@ -1,85 +0,0 @@
-"""presentation-chrome auto-hide (#141) — settings knobs + review table
-
-MLSettings gains presentation_auto_apply_enabled / _threshold and
-presentation_conflict_threshold: banner + editor-screenshot auto-hide on the
-sweep with a FLAT threshold (decoupled from content-head graduation), and a
-conflict threshold that flags an auto-hide that "also looks like content".
-
-New table presentation_review records an auto-hidden chrome image that also
-scored high on a content head, surfaced in the Hidden view for a keep-hidden /
-un-hide decision. Resolved rows are pruned by retention.
-
-Revision ID: 0082
-Revises: 0081
-Create Date: 2026-07-07
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0082"
-down_revision: Union[str, None] = "0081"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "presentation_auto_apply_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.text("true"),
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "presentation_auto_apply_threshold", sa.Float(), nullable=False,
-            server_default=sa.text("0.90"),
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "presentation_conflict_threshold", sa.Float(), nullable=False,
-            server_default=sa.text("0.50"),
-        ),
-    )
-    op.create_table(
-        "presentation_review",
-        sa.Column(
-            "image_record_id", sa.Integer(),
-            sa.ForeignKey("image_record.id", ondelete="CASCADE"),
-            primary_key=True,
-        ),
-        sa.Column(
-            "tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True,
-        ),
-        sa.Column(
-            "conflict_tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="SET NULL"), nullable=True,
-        ),
-        sa.Column("conflict_score", sa.Float(), nullable=False),
-        sa.Column(
-            "created_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
-    )
-    # The review list queries the unresolved flags (resolved_at IS NULL).
-    op.create_index(
-        "ix_presentation_review_resolved_at", "presentation_review",
-        ["resolved_at"],
-    )
-
-
-def downgrade() -> None:
-    op.drop_index(
-        "ix_presentation_review_resolved_at", table_name="presentation_review"
-    )
-    op.drop_table("presentation_review")
-    op.drop_column("ml_settings", "presentation_conflict_threshold")
-    op.drop_column("ml_settings", "presentation_auto_apply_threshold")
-    op.drop_column("ml_settings", "presentation_auto_apply_enabled")

alembic/versions/0083_post_translation.py

@@ -1,73 +0,0 @@
-"""post-text translation via Interpreter (milestone 143) — Post columns + settings
-
-Post gains the translated title/description + the detected source language,
-Interpreter engine_version (cache key), and translated_at — filled by the
-translate sweep. ImportSettings gains translation_enabled (OFF by default),
-interpreter_base_url (EMPTY — the operator sets their own, behind a reverse
-proxy), and translation_target_lang (en).
-
-Revision ID: 0083
-Revises: 0082
-Create Date: 2026-07-07
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0083"
-down_revision: Union[str, None] = "0082"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "post", sa.Column("post_title_translated", sa.Text(), nullable=True)
-    )
-    op.add_column(
-        "post", sa.Column("description_translated", sa.Text(), nullable=True)
-    )
-    op.add_column(
-        "post",
-        sa.Column("translated_source_lang", sa.String(8), nullable=True),
-    )
-    op.add_column(
-        "post",
-        sa.Column("translation_engine_version", sa.String(128), nullable=True),
-    )
-    op.add_column(
-        "post",
-        sa.Column("translated_at", sa.DateTime(timezone=True), nullable=True),
-    )
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "translation_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.text("false"),
-        ),
-    )
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "interpreter_base_url", sa.Text(), nullable=False, server_default="",
-        ),
-    )
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "translation_target_lang", sa.Text(), nullable=False,
-            server_default="en",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("import_settings", "translation_target_lang")
-    op.drop_column("import_settings", "interpreter_base_url")
-    op.drop_column("import_settings", "translation_enabled")
-    op.drop_column("post", "translated_at")
-    op.drop_column("post", "translation_engine_version")
-    op.drop_column("post", "translated_source_lang")
-    op.drop_column("post", "description_translated")
-    op.drop_column("post", "post_title_translated")

alembic/versions/0084_translation_strictness_override.py

@@ -1,51 +0,0 @@
-"""translation strictness setting + per-post translation override (milestone 155)
-
-ImportSettings gains ``translation_min_confidence`` (the latin-script acceptance
-floor, now operator-tunable in the UI; default 0.9 — stricter than the old
-hardcoded 0.8, since Interpreter confidently mis-detects short ASCII English at
-~0.86). Post gains ``translation_override`` — a sticky per-post choice of
-auto / force / original so the operator can force a skipped translation on, or
-knock a wrongly-translated one back to the original, and have it survive a
-Re-translate-all.
-
-Revision ID: 0084
-Revises: 0083
-Create Date: 2026-07-10
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0084"
-down_revision: Union[str, None] = "0083"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "translation_min_confidence", sa.Float(), nullable=False,
-            server_default=sa.text("0.9"),
-        ),
-    )
-    op.add_column(
-        "post",
-        sa.Column(
-            "translation_override", sa.String(16), nullable=False,
-            server_default="auto",
-        ),
-    )
-    op.create_check_constraint(
-        "ck_post_translation_override",
-        "post",
-        "translation_override IN ('auto', 'force', 'original')",
-    )
-
-
-def downgrade() -> None:
-    op.drop_constraint("ck_post_translation_override", "post", type_="check")
-    op.drop_column("post", "translation_override")
-    op.drop_column("import_settings", "translation_min_confidence")

alembic/versions/0085_wip_title_tagging.py

@@ -1,35 +0,0 @@
-"""title-based WIP auto-tagging (task #1458) — ImportSettings toggle
-
-ImportSettings gains wip_title_tagging_enabled (ON by default): when a freshly
-imported post's title explicitly declares work-in-progress ("WIP" / "work in
-progress"), the importer applies the `wip` system tag to its images. No new
-table — the tag itself is the seeded `wip` system tag (migration 0075) and the
-application reuses image_tag with source='wip_title'.
-
-Revision ID: 0085
-Revises: 0084
-Create Date: 2026-07-12
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0085"
-down_revision: Union[str, None] = "0084"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "wip_title_tagging_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.text("true"),
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("import_settings", "wip_title_tagging_enabled")

alembic/versions/0086_process_auto_apply_settings.py

@@ -1,61 +0,0 @@
-"""process auto-apply settings + review mode (#1464) — system-tag refactor
-
-The system-tag behavior refactor gives `wip` / `editor screenshot` (the PROCESS
-group) their own provisional auto-apply, parallel to the presentation (chrome)
-sweep. MLSettings gains three knobs: enabled (OFF by default — a new whole-library
-auto-tagger is opt-in), the flat apply threshold, and the ring-loud conflict
-threshold. presentation_review gains a `mode` column so one review surface serves
-both chrome and process flags (existing rows backfill 'chrome'). server_defaults
-so the existing rows fill cleanly.
-
-Revision ID: 0086
-Revises: 0085
-Create Date: 2026-07-13
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0086"
-down_revision: Union[str, None] = "0085"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "process_auto_apply_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.text("false"),
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "process_auto_apply_threshold", sa.Float(), nullable=False,
-            server_default="0.90",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "process_conflict_threshold", sa.Float(), nullable=False,
-            server_default="0.50",
-        ),
-    )
-    op.add_column(
-        "presentation_review",
-        sa.Column(
-            "mode", sa.String(16), nullable=False,
-            server_default="chrome",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("presentation_review", "mode")
-    op.drop_column("ml_settings", "process_conflict_threshold")
-    op.drop_column("ml_settings", "process_auto_apply_threshold")
-    op.drop_column("ml_settings", "process_auto_apply_enabled")

alembic/versions/0087_wip_soft_title_tagging.py

@@ -1,33 +0,0 @@
-"""soft WIP title tier toggle (#1474) — ImportSettings.wip_soft_title_tagging_enabled
-
-The soft tier also tags sketch/doodle/scribble titles, but with a provisional source
-that never trains the head. OFF by default (a lower-precision tier is opt-in).
-server_default so the existing singleton row (id=1) fills cleanly.
-
-Revision ID: 0087
-Revises: 0086
-Create Date: 2026-07-13
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0087"
-down_revision: Union[str, None] = "0086"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "wip_soft_title_tagging_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.text("false"),
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("import_settings", "wip_soft_title_tagging_enabled")

backend/app/__init__.py

@@ -34,23 +34,6 @@ def create_app() -> Quart:
     app = Quart(__name__)
     app.secret_key = cfg.secret_key
 
-    # Stream files in 4 MiB chunks instead of Quart's 8 KiB default. The image
-    # library lives on a CIFS/SMB share (mounted rsize=4 MiB), so 8 KiB reads
-    # meant ~19k network round-trips for one large original — 30–58s downloads
-    # that starved both the GPU agent and the browser (operator-flagged
-    # 2026-07-01). 4 MiB matches the mount's read size → one round-trip per read,
-    # ~500× fewer. buffer_size is the MAX read, so small thumbnails still read in
-    # a single gulp, and Range/mime/ETag/conditional handling lives on Response,
-    # so this keeps all of it. Guarded so a future Quart-internal change can't
-    # break boot — worst case we fall back to the slow default.
-    try:
-        from quart.wrappers.response import FileBody
-        FileBody.buffer_size = 4 * 1024 * 1024
-    except Exception:
-        logging.getLogger(__name__).warning(
-            "could not raise FileBody.buffer_size — file serving stays on 8 KiB chunks"
-        )
-
     for bp in all_blueprints():
         app.register_blueprint(bp)
     # Registered last so /api/* routes win over the SPA catch-all.

backend/app/api/__init__.py

@@ -16,6 +16,7 @@ api_bp.add_url_rule("/health", view_func=health.get_health, methods=["GET"])
 def all_blueprints() -> list[Blueprint]:
     from .admin import admin_bp
     from .aliases import aliases_bp
+    from .allowlist import allowlist_bp
     from .artist import artist_bp
     from .artists import artists_bp
     from .attachments import attachments_bp
@@ -38,6 +39,7 @@ def all_blueprints() -> list[Blueprint]:
     from .suggestions import suggestions_bp
     from .system_activity import system_activity_bp
     from .system_backup import system_backup_bp
+    from .tag_eval import tag_eval_bp
     from .tags import tags_bp
     from .thumbnails import thumbnails_bp
     return [
@@ -56,7 +58,9 @@ def all_blueprints() -> list[Blueprint]:
         cleanup_bp,
         import_admin_bp,
         suggestions_bp,
+        allowlist_bp,
         aliases_bp,
+        tag_eval_bp,
         heads_bp,
         gpu_bp,
         ccip_bp,

backend/app/api/admin.py

@@ -1,13 +1,13 @@
 """FC-3k: /api/admin — destructive admin actions.
 
-Action surfaces:
+Five action surfaces:
   POST   /api/admin/artists/<slug>/cascade-delete  (Tier C)
   POST   /api/admin/images/bulk-delete             (Tier C)
   DELETE /api/admin/tags/<int:tag_id>              (Tier B)
   POST   /api/admin/tags/<int:dest_id>/merge       (Tier B)
   POST   /api/admin/tags/prune-unused              (Tier A)
   POST   /api/admin/posts/prune-bare               (Tier A)
-  POST   /api/admin/posts/refetch-external         (Tier A)
+  POST   /api/admin/tags/purge-legacy              (Tier A)
   GET    /api/admin/tags/<int:tag_id>/usage-count  (helper)
 
 Tier-C ops take a dry_run body flag (returns projection inline,
@@ -23,7 +23,7 @@ from quart import Blueprint, jsonify, request
 from sqlalchemy import select, text
 
 from ..extensions import get_session
-from ..models import Artist, Post
+from ..models import Artist
 from ..services.cleanup_service import project_artist_cascade, project_bulk_image_delete
 from ._responses import error_response as _bad
 
@@ -156,10 +156,6 @@ async def tag_delete(tag_id: int):
             )
         except LookupError:
             return _bad("not_found", status=404)
-        except ValueError as exc:
-            # System tags (#128) — the training-hygiene machinery keys on
-            # these rows.
-            return _bad("system_tag", detail=str(exc))
     return jsonify(result)
 
 
@@ -281,86 +277,31 @@ async def posts_reconcile_duplicates():
     return await _run_dry_run_op(reconcile_duplicate_posts, source_id=source_id)
 
 
-@admin_bp.route("/posts/refetch-external", methods=["POST"])
-async def posts_refetch_external():
-    """Surgical re-fetch of a post's external file-host links (operator
-    2026-07-03): the normal cadence never re-walks deep back-catalogue posts,
-    so a deleted external file only comes back by resetting its ExternalLink
-    row(s) — this endpoint does that per post and dispatches the fetches.
-    Sha-dedupe discards payload files that still exist, so only what's
-    missing lands. Body: {external_post_id: str, source_id?: int (to
-    disambiguate the same external id across sources)}."""
-    from ..services.external_links import refetch_links_for_post
+@admin_bp.route("/tags/purge-legacy", methods=["POST"])
+async def tags_purge_legacy():
+    """Tier-A: delete legacy IR-migration tags — archive/post/artist
+    kinds (e.g. `BlenderKnight:Hannah_BJ_Loops`) PLUS general tags with
+    a legacy name prefix (`source:*`, from IR's source kind that fell
+    back to general). dry-run preview returns per-kind + per-prefix
+    counts + a sample so the UI shows exactly what'll go before the
+    operator confirms with dry_run=false."""
+    from ..services.cleanup_service import purge_legacy_tags
 
-    body = await request.get_json(silent=True) or {}
-    ext_id = str(body.get("external_post_id") or "").strip()
-    if not ext_id:
-        return _bad("missing_external_post_id",
-                    detail="external_post_id is required")
-    raw_source = body.get("source_id")
-    try:
-        source_id = int(raw_source) if raw_source is not None else None
-    except (TypeError, ValueError):
-        return _bad("invalid_source_id", detail="source_id must be an integer")
-
-    async with get_session() as session:
-        stmt = select(Post.id).where(Post.external_post_id == ext_id)
-        if source_id is not None:
-            stmt = stmt.where(Post.source_id == source_id)
-        post_ids = (await session.execute(stmt)).scalars().all()
-        if not post_ids:
-            return _bad("post_not_found", status=404,
-                        detail=f"no post with external_post_id {ext_id!r}")
-        results = {}
-        for pid in post_ids:
-            results[str(pid)] = await session.run_sync(
-                lambda s, p=pid: refetch_links_for_post(s, p)
-            )
-    return jsonify({"posts": results})
-
-
-def _reset_content_confirm_token(projection: dict) -> str:
-    """Stable 8-hex token derived from the live counts (mirrors the Tier-C
-    bulk-delete token): it changes whenever the data changes, so the apply can
-    only ever run against numbers the operator just previewed."""
-    canon = f"reset-content:{projection.get('count')}:{projection.get('applications')}"
-    return hashlib.sha256(canon.encode("utf-8")).hexdigest()[:8]
+    return await _run_dry_run_op(purge_legacy_tags)
 
 
 @admin_bp.route("/tags/reset-content", methods=["POST"])
 async def tags_reset_content():
-    """Full-instance reset of the CONTENT vocabulary: deletes ALL general +
-    character tags and their image applications — INCLUDING the examples the
-    tagging heads learned from. Suggestions do NOT repopulate on their own
-    (the Camie predictions that once did are long retired): the operator
-    re-tags from scratch and the heads retrain from the new signal. fandom +
-    series tags + series_page ordering are preserved.
-
-    Deliberately Tier-C-gated despite the Tier-A shape (operator 2026-07-02:
-    the full reset stays, but behind extra steps): dry_run returns the
-    projection + a `confirm` token derived from the live counts; the apply
-    must echo that token back or it is rejected."""
+    """Tier-A: delete ALL general + character tags (the Camie-suggestable
+    content vocabulary) so the operator can re-tag from scratch via
+    auto-suggest. fandom + series tags + series_page ordering are preserved,
+    and image_prediction rows are untouched so suggestions repopulate.
+    dry-run preview returns per-kind counts + applications + a sample so the
+    UI shows exactly what'll go before the operator confirms (dry_run=false).
+    Irreversible except via DB backup restore."""
     from ..services.cleanup_service import reset_content_tagging
 
-    body = await request.get_json(silent=True) or {}
-    dry_run = bool(body.get("dry_run", False))
-    async with get_session() as session:
-        projection = await session.run_sync(
-            lambda s: reset_content_tagging(s, dry_run=True)
-        )
-        token = _reset_content_confirm_token(projection)
-        if dry_run:
-            projection["confirm"] = token
-            return jsonify(projection)
-        if str(body.get("confirm", "")) != token:
-            return _bad(
-                "confirm_mismatch",
-                detail="run a fresh preview and echo its confirm token",
-            )
-        result = await session.run_sync(
-            lambda s: reset_content_tagging(s, dry_run=False)
-        )
-    return jsonify(result)
+    return await _run_dry_run_op(reset_content_tagging)
 
 
 @admin_bp.route("/tags/normalize", methods=["POST"])

backend/app/api/allowlist.py

@@ -0,0 +1,84 @@
+"""Allowlist API: list, adjust threshold, remove."""
+
+from quart import Blueprint, jsonify, request
+
+from ..extensions import get_session
+from ..models import TagAllowlist
+from ..services.ml.allowlist import AllowlistService
+
+allowlist_bp = Blueprint("allowlist", __name__, url_prefix="/api")
+
+
+@allowlist_bp.route("/allowlist", methods=["GET"])
+async def list_allowlist():
+    async with get_session() as session:
+        rows = await AllowlistService(session).list_all()
+    return jsonify(
+        [
+            {
+                "tag_id": r.tag_id,
+                "tag_name": r.tag_name,
+                "tag_kind": r.tag_kind,
+                "min_confidence": r.min_confidence,
+                "applied_count": r.applied_count,
+                "coverage_count": r.coverage_count,
+            }
+            for r in rows
+        ]
+    )
+
+
+@allowlist_bp.route("/tags/<int:tag_id>/allowlist/coverage", methods=["GET"])
+async def coverage(tag_id: int):
+    """Live "at threshold T, a sweep would cover ~N images" projection for the
+    allowlist tuning dashboard. Defaults to the tag's stored threshold."""
+    raw = request.args.get("threshold")
+    async with get_session() as session:
+        svc = AllowlistService(session)
+        if raw is not None:
+            try:
+                threshold = float(raw)
+            except ValueError:
+                return jsonify({"error": "threshold must be a float"}), 400
+            if not (0 < threshold <= 1):
+                return jsonify({"error": "threshold must be in (0, 1]"}), 400
+        else:
+            row = await session.get(TagAllowlist, tag_id)
+            if row is None:
+                return jsonify({"error": "not on allowlist"}), 404
+            threshold = row.min_confidence
+        count = await svc.coverage(tag_id, threshold)
+    return jsonify({"count": count, "threshold": threshold})
+
+
+@allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["GET"])
+async def get_one(tag_id: int):
+    async with get_session() as session:
+        row = await session.get(TagAllowlist, tag_id)
+    if row is None:
+        return jsonify({"error": "not on allowlist"}), 404
+    return jsonify(
+        {"min_confidence": row.min_confidence, "added_at": row.added_at.isoformat()}
+    )
+
+
+@allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["PATCH"])
+async def patch_threshold(tag_id: int):
+    body = await request.get_json()
+    if not body or "min_confidence" not in body:
+        return jsonify({"error": "min_confidence required"}), 400
+    mc = float(body["min_confidence"])
+    if not (0 < mc <= 1):
+        return jsonify({"error": "min_confidence must be in (0, 1]"}), 400
+    async with get_session() as session:
+        await AllowlistService(session).update_threshold(tag_id, mc)
+        await session.commit()
+    return "", 204
+
+
+@allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["DELETE"])
+async def remove(tag_id: int):
+    async with get_session() as session:
+        await AllowlistService(session).remove(tag_id)
+        await session.commit()
+    return "", 204

backend/app/api/artists.py

@@ -31,24 +31,6 @@ async def create_or_get():
     }), 201
 
 
-@artists_bp.route("/<int:artist_id>", methods=["PATCH"])
-async def rename(artist_id: int):
-    """Rename an artist's DISPLAY NAME (#130). Name only — the slug and every
-    on-disk path stay put, so this is instant and safe. Name is non-unique."""
-    body = await request.get_json()
-    if not isinstance(body, dict) or not isinstance(body.get("name"), str):
-        return jsonify({"error": "invalid_body"}), 400
-    async with get_session() as session:
-        svc = ArtistService(session)
-        try:
-            artist = await svc.rename(artist_id, body["name"])
-        except ValueError as exc:
-            return jsonify({"error": "empty_name", "detail": str(exc)}), 400
-        if artist is None:
-            return jsonify({"error": "not_found"}), 404
-        return jsonify({"id": artist.id, "name": artist.name, "slug": artist.slug})
-
-
 @artists_bp.route("/autocomplete", methods=["GET"])
 async def autocomplete():
     q = request.args.get("q") or ""

backend/app/api/ccip.py

@@ -37,15 +37,6 @@ async def overview():
                 .where(ImageRegion.ccip_embedding.is_not(None))
             )
         ).scalar_one()
-        # Concept-crop (SigLIP bag) coverage — how far the back-catalogue embed
-        # has progressed, so the max-over-bag scorer's reach is checkable.
-        images_with_concept_siglip = (
-            await session.execute(
-                select(func.count(distinct(ImageRegion.image_record_id)))
-                .where(ImageRegion.kind == "concept")
-                .where(ImageRegion.siglip_embedding.is_not(None))
-            )
-        ).scalar_one()
         # Per-character reference counts (no vectors loaded) — which characters
         # have enough examples to match on.
         ref_rows = (
@@ -71,23 +62,14 @@ async def overview():
                 )
             ).all() if v
         ]
-        auto_applied = (
-            await session.execute(
-                select(func.count()).select_from(image_tag).where(
-                    image_tag.c.source == "ccip_auto"
-                )
-            )
-        ).scalar_one()
     return jsonify({
         "regions_by_kind": by_kind,
         "images_with_figure_ccip": images_with_figure_ccip,
-        "images_with_concept_siglip": images_with_concept_siglip,
         "characters_with_references": len(ref_rows),
         "character_references": [
             {"tag_id": t, "name": n, "n_refs": c} for (t, n, c) in ref_rows
         ],
         "embedding_versions": versions,
-        "auto_applied": auto_applied,
     })
 
 

backend/app/api/extension.py

@@ -84,15 +84,11 @@ async def quick_add_source():
     if not isinstance(url, str) or not url.strip():
         return _bad("invalid_body", detail="url is required")
 
-    from .credentials import _get_crypto
-
     async with get_session() as session:
         if not await _ext_key_required(session):
             return _bad("unauthorized", status=401)
         try:
-            # crypto lets a pixiv add resolve the artist's display name via the
-            # stored OAuth token (else it falls back to the numeric id). #130.
-            result = await ExtensionService(session, _get_crypto()).quick_add_source(url)
+            result = await ExtensionService(session).quick_add_source(url)
         except UnknownPlatformError as exc:
             return _bad(
                 "unknown_platform",

backend/app/api/gallery.py

@@ -3,18 +3,9 @@
 from datetime import UTC, datetime, timedelta
 
 from quart import Blueprint, jsonify, request
-from sqlalchemy import delete, select, update
-from sqlalchemy.orm import aliased
 
 from ..extensions import get_session
-from ..models import (
-    ImageRecord,
-    PresentationReview,
-    Tag,
-    TagSuggestionRejection,
-)
-from ..models.tag import image_tag
-from ..services.gallery_service import GalleryService, image_url, thumbnail_url
+from ..services.gallery_service import GalleryService
 
 gallery_bp = Blueprint("gallery", __name__, url_prefix="/api/gallery")
 
@@ -53,8 +44,7 @@ def _parse_filters():
       the image must match at least one tag from EACH group (groups ANDed).
     - `tag_not` is a comma-separated exclude list (image must carry none).
 
-    `media` is image|video; `sort` is newest|oldest|posted_new|posted_old
-    (default posted_new); `platform` selects one
+    `media` is image|video; `sort` is newest|oldest; `platform` selects one
     platform (or the UNSOURCED_PLATFORM sentinel); `untagged`/`no_artist` are
     boolean flags; `date_from`/`date_to` are inclusive calendar-day bounds
     (date_to is widened by a day so the whole day is covered by the service's
@@ -77,18 +67,11 @@ def _parse_filters():
     artist_id = int(artist_id_raw) if artist_id_raw else None
     media = request.args.get("media")
     media_type = media if media in ("image", "video") else None
-    # newest/oldest key off effective_date (primary post / download); posted_new/
-    # posted_old off earliest_post_date (original publish across all posts). The
-    # default is posted_new so the grid leads with original publish date, not the
-    # download/repost the primary post points at (operator-flagged 2026-07-01).
     sort = request.args.get("sort")
-    sort = sort if sort in ("newest", "oldest", "posted_new", "posted_old") else "posted_new"
+    sort = sort if sort in ("newest", "oldest") else "newest"
     platform = request.args.get("platform") or None
     untagged = request.args.get("untagged") in ("1", "true", "yes")
     no_artist = request.args.get("no_artist") in ("1", "true", "yes")
-    # Show the presentation chrome (banner / editor screenshot) that the default
-    # gallery hides — the Hidden view sets this (milestone 141).
-    include_hidden = request.args.get("include_hidden") in ("1", "true", "yes")
     date_from = _parse_date(request.args.get("date_from"))
     date_to = _parse_date(request.args.get("date_to"))
     if date_to is not None:
@@ -100,7 +83,6 @@ def _parse_filters():
         "platform": platform,
         "untagged": untagged, "no_artist": no_artist,
         "date_from": date_from, "date_to": date_to,
-        "include_hidden": include_hidden,
     }
     return filters, sort
 
@@ -145,32 +127,12 @@ async def similar():
         filters, _sort = _parse_filters()
     except (KeyError, ValueError):
         return jsonify({"error": "similar_to query param required"}), 400
-    # Explore passes exclude_wip=1 to also drop work-in-progress from the
-    # rabbit-hole; the gallery's own "similar" button omits it (keeps wip, #1274).
-    exclude_wip = request.args.get("exclude_wip") in ("1", "true", "True")
-    # Explore reach (#1476): 0 = nearest (gallery default), →1 reaches into farther
-    # distance bands so the walk can escape a dense cluster. exclude_ids = the
-    # breadcrumb, so already-walked images aren't re-served as neighbours.
-    try:
-        reach = max(0.0, min(1.0, float(request.args.get("reach", "0"))))
-    except ValueError:
-        reach = 0.0
-    exclude_ids = [
-        int(x) for x in request.args.get("exclude_ids", "").split(",")
-        if x.strip().isdigit()
-    ] or None
     # post_id is the exclusive post-detail view — not a similarity scope.
-    # include_hidden is a gallery-browse flag; similar() has its OWN presentation
-    # exclusion (a similarity-quality concern, #1274), so drop it here (#141).
-    scope = {
-        k: v for k, v in filters.items() if k not in ("post_id", "include_hidden")
-    }
+    scope = {k: v for k, v in filters.items() if k != "post_id"}
     async with get_session() as session:
         svc = GalleryService(session)
         try:
-            images = await svc.similar(
-                image_id=similar_to, limit=limit, exclude_wip=exclude_wip,
-                reach=reach, exclude_ids=exclude_ids, **scope)
+            images = await svc.similar(image_id=similar_to, limit=limit, **scope)
         except ValueError as exc:
             return jsonify({"error": str(exc)}), 400
     if images is None:
@@ -244,101 +206,6 @@ async def jump():
     return jsonify({"cursor": cursor})
 
 
-# -- Hidden-view review (#141): auto-hidden chrome flagged "also looks like
-#    content", surfaced in the gallery's Show-hidden review strip. -----------
-@gallery_bp.route("/hidden-review", methods=["GET"])
-async def hidden_review():
-    """Unresolved system-tag auto-apply review flags (chrome + process, #1464),
-    most-concerning first (highest content score) — for the review strip. `mode`
-    tells the client whether the flagged tag hid the image ('chrome') or left it
-    visible ('process'), which decides the resolve labels (un-hide vs remove-tag)."""
-    ptag = aliased(Tag)
-    ctag = aliased(Tag)
-    async with get_session() as session:
-        rows = (await session.execute(
-            select(
-                PresentationReview.image_record_id,
-                PresentationReview.tag_id,
-                PresentationReview.conflict_tag_id,
-                PresentationReview.conflict_score,
-                PresentationReview.mode,
-                ImageRecord.path, ImageRecord.thumbnail_path,
-                ImageRecord.sha256, ImageRecord.mime,
-                ptag.name.label("tag_name"),
-                ctag.name.label("conflict_name"),
-            )
-            .join(ImageRecord, ImageRecord.id == PresentationReview.image_record_id)
-            .join(ptag, ptag.id == PresentationReview.tag_id)
-            .outerjoin(ctag, ctag.id == PresentationReview.conflict_tag_id)
-            .where(PresentationReview.resolved_at.is_(None))
-            .order_by(PresentationReview.conflict_score.desc())
-        )).all()
-    return jsonify({"items": [
-        {
-            "image_id": r.image_record_id,
-            "tag_id": r.tag_id,
-            "tag_name": r.tag_name,
-            "conflict_tag_id": r.conflict_tag_id,
-            "conflict_name": r.conflict_name,
-            "conflict_score": r.conflict_score,
-            "mode": r.mode,
-            "thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime),
-            "image_url": image_url(r.path),
-        }
-        for r in rows
-    ]})
-
-
-@gallery_bp.route(
-    "/hidden-review/<int:image_id>/<int:tag_id>/keep", methods=["POST"]
-)
-async def hidden_review_keep(image_id, tag_id):
-    """Keep the auto-hide: resolve the flag; the tag stays applied (#141)."""
-    async with get_session() as session:
-        await session.execute(
-            update(PresentationReview)
-            .where(
-                PresentationReview.image_record_id == image_id,
-                PresentationReview.tag_id == tag_id,
-            )
-            .values(resolved_at=datetime.now(UTC))
-        )
-        await session.commit()
-    return "", 204
-
-
-@gallery_bp.route(
-    "/hidden-review/<int:image_id>/<int:tag_id>/unhide", methods=["POST"]
-)
-async def hidden_review_unhide(image_id, tag_id):
-    """Un-hide: remove the presentation tag (image returns to the gallery), record
-    a rejection so the head LEARNS it misfired, and resolve the flag (#141)."""
-    from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-    async with get_session() as session:
-        await session.execute(
-            delete(image_tag).where(
-                image_tag.c.image_record_id == image_id,
-                image_tag.c.tag_id == tag_id,
-            )
-        )
-        await session.execute(
-            pg_insert(TagSuggestionRejection)
-            .values(image_record_id=image_id, tag_id=tag_id)
-            .on_conflict_do_nothing()
-        )
-        await session.execute(
-            update(PresentationReview)
-            .where(
-                PresentationReview.image_record_id == image_id,
-                PresentationReview.tag_id == tag_id,
-            )
-            .values(resolved_at=datetime.now(UTC))
-        )
-        await session.commit()
-    return "", 204
-
-
 @gallery_bp.route("/image/<int:image_id>", methods=["GET"])
 async def image_detail(image_id: int):
     async with get_session() as session:

backend/app/api/gpu.py

@@ -9,25 +9,19 @@ homelab admin.
 """
 
 import secrets
-from pathlib import Path
 
 from quart import Blueprint, jsonify, request
-from sqlalchemy import func, or_, select, update
+from sqlalchemy import func, select
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 
 from ..extensions import get_session
 from ..models import AppSetting, GpuJob, ImageRecord, MLSettings
 from ..services.gallery_service import image_url
-from ..services.ml.gpu_jobs import GpuJobService, error_dedupe_statements
-from ..services.ml.gpu_triage import classify_reason, recover_defective_image
+from ..services.ml.gpu_jobs import GpuJobService
 from ..services.ml.regions import RegionService
 
 gpu_bp = Blueprint("gpu", __name__, url_prefix="/api/gpu")
 
-# Same container mount the maintenance tasks use (tasks/admin.py) — recovery
-# deletes the defective original + thumbnail under it.
-_IMAGES_ROOT = Path("/images")
-
 _TOKEN_KEY = "gpu_agent_token"
 
 
@@ -96,152 +90,12 @@ async def backfill():
     """Enqueue a job for every image that doesn't already have one for `task`."""
     body = await request.get_json(silent=True) or {}
     task = str(body.get("task") or "ccip")
-    from ..tasks.gpu_queue import enqueue_gpu_backfill
+    from ..tasks.ml import enqueue_gpu_backfill
 
     r = enqueue_gpu_backfill.delay(task)
     return jsonify({"celery_task_id": r.id, "task": task}), 202
 
 
-@gpu_bp.route("/reprocess", methods=["POST"])
-async def reprocess():
-    """Reset every done/error job of `task` back to pending so the agent re-runs
-    the WHOLE library under the current pipeline (e.g. after adding crop
-    detectors). Heavy — the back-catalogue is otherwise skipped by the backfills."""
-    body = await request.get_json(silent=True) or {}
-    task = str(body.get("task") or "ccip")
-    from ..tasks.gpu_queue import reprocess_gpu_jobs
-
-    r = reprocess_gpu_jobs.delay(task)
-    return jsonify({"celery_task_id": r.id, "task": task}), 202
-
-
-@gpu_bp.route("/retry_errors", methods=["POST"])
-async def retry_errors():
-    """Requeue every ERRORED job (all task types) back to pending — the scoped
-    recovery after an agent-side fix (e.g. the short-video sampler), where
-    /reprocess would needlessly re-run the whole done library too. Attempts and
-    the stored error reset so each job gets its full retry budget under the
-    fixed pipeline. Stale tombstones are pruned FIRST (loop-era duplicates and
-    rows a later success made moot — the same statements the backfills run), so
-    one failing file requeues as ONE job, never a fan-out of duplicates. Small
-    row count (errors only) → inline statements; the response carries the
-    counts for the UI toast. Triage-confirmed defects are NOT requeued (see
-    the WHERE below) — they stay on the recovery surface."""
-    async with get_session() as session:
-        pruned = 0
-        for stmt in error_dedupe_statements():
-            pruned += (await session.execute(stmt)).rowcount or 0
-        res = await session.execute(
-            update(GpuJob)
-            .where(
-                GpuJob.status == "error",
-                # Triage-confirmed DEFECTS stay errored: the integrity probe
-                # already proved the FILE is bad, so re-running the job just
-                # burns agent time re-minting the same tombstone — those go
-                # through /errors/<id>/recover instead.
-                or_(GpuJob.triage_status.is_(None),
-                    GpuJob.triage_status != "defect"),
-            )
-            .values(
-                status="pending", attempts=0, error=None, lease_token=None,
-                leased_at=None, lease_expires_at=None, triage_status=None,
-                updated_at=func.now(),
-            )
-        )
-        kept = (
-            await session.execute(
-                select(func.count()).select_from(GpuJob)
-                .where(GpuJob.status == "error")
-            )
-        ).scalar_one()
-        await session.commit()
-    return jsonify({
-        "requeued": res.rowcount or 0, "pruned": pruned, "defects_kept": kept,
-    })
-
-
-# --- Failure triage + recovery (#125) ------------------------------------
-
-@gpu_bp.route("/errors", methods=["GET"])
-async def errors():
-    """The triage view of the error tombstones: every errored job joined with
-    its image's integrity verdict, bucketed by reason for the overview. The
-    probe sweep (triage_gpu_errors, 15-min beat) fills triage_status; 'defect'
-    rows are the recovery surface's list."""
-    async with get_session() as session:
-        rows = (
-            await session.execute(
-                select(
-                    GpuJob.id, GpuJob.image_record_id, GpuJob.task,
-                    GpuJob.error, GpuJob.triage_status, GpuJob.updated_at,
-                    ImageRecord.integrity_status, ImageRecord.mime,
-                    ImageRecord.path, ImageRecord.thumbnail_path,
-                )
-                .join(ImageRecord, ImageRecord.id == GpuJob.image_record_id)
-                .where(GpuJob.status == "error")
-                .order_by(GpuJob.updated_at.desc())
-                .limit(500)
-            )
-        ).all()
-        total = (
-            await session.execute(
-                select(func.count()).select_from(GpuJob)
-                .where(GpuJob.status == "error")
-            )
-        ).scalar_one()
-    by_class: dict[str, int] = {}
-    triage = {"defect": 0, "file_ok": 0, "unclassified": 0}
-    items = []
-    for r in rows:
-        cls = classify_reason(r.error)
-        by_class[cls] = by_class.get(cls, 0) + 1
-        bucket = r.triage_status or "unclassified"
-        triage[bucket] = triage.get(bucket, 0) + 1
-        items.append({
-            "job_id": r.id,
-            "image_id": r.image_record_id,
-            "task": r.task,
-            "error": r.error,
-            "reason_class": cls,
-            "triage_status": r.triage_status,
-            "integrity_status": r.integrity_status,
-            "mime": r.mime,
-            "image_url": image_url(r.path),
-            "thumbnail_url": (
-                image_url(r.thumbnail_path) if r.thumbnail_path else None
-            ),
-            "updated_at": r.updated_at.isoformat() if r.updated_at else None,
-        })
-    return jsonify({
-        "total": total, "by_class": by_class, "triage": triage, "items": items,
-    })
-
-
-@gpu_bp.route("/errors/triage", methods=["POST"])
-async def errors_triage():
-    """Run the probe sweep NOW (the card's button) instead of waiting out the
-    15-minute beat cadence."""
-    from ..tasks.maintenance import triage_gpu_errors
-
-    r = triage_gpu_errors.delay()
-    return jsonify({"celery_task_id": r.id}), 202
-
-
-@gpu_bp.route("/errors/<int:image_id>/recover", methods=["POST"])
-async def errors_recover(image_id: int):
-    """Recover a defect-triaged original: delete the bad copy + record and
-    re-poll its subscription Source (a fresh fetch re-imports the file, which
-    re-enters the GPU pipeline). Returns status 'no_source' when nothing
-    pollable resolves — the file needs manual replacement there."""
-    async with get_session() as session:
-        result = await session.run_sync(
-            lambda s: recover_defective_image(
-                s, image_id, images_root=_IMAGES_ROOT,
-            )
-        )
-    return jsonify(result)
-
-
 # --- Agent (bearer token): lease / submit / heartbeat / fail ------------
 
 @gpu_bp.route("/jobs/lease", methods=["POST"])
@@ -256,7 +110,9 @@ async def lease():
         if not await _agent_authed(session):
             return jsonify({"error": "unauthorized"}), 401
         jobs = await GpuJobService(session).lease(agent_id, batch_size=batch)
-        ml = await MLSettings.load(session)
+        ml = (
+            await session.execute(select(MLSettings).where(MLSettings.id == 1))
+        ).scalar_one()
         # image rows for url/mime in one shot
         ids = [j.image_record_id for j in jobs]
         imgs = {
@@ -267,33 +123,6 @@ async def lease():
             ).scalars()
         } if ids else {}
         await session.commit()
-        # Crop-proposer config, announced FROM THE SETTING like embed_model_name
-        # (#134): the agent builds its detectors from this, rebuilding live when
-        # it changes — so tuning is a DB/UI edit, never an agent restart. Same
-        # block for every job in the batch (it's global), built once. An enabled
-        # toggle off is carried through so the agent skips that proposer.
-        detectors = {
-            "person": {
-                "enabled": ml.detector_person_enabled,
-                "weights": ml.detector_person_weights,
-                "conf": ml.detector_person_conf,
-            },
-            "anatomy": {
-                "enabled": ml.detector_anatomy_enabled,
-                "weights": ml.detector_anatomy_weights,
-                "conf": ml.detector_anatomy_conf,
-            },
-            "panel": {
-                "enabled": ml.detector_panel_enabled,
-                "weights": ml.detector_panel_weights,
-                "conf": ml.detector_panel_conf,
-            },
-            "max_figures": ml.detector_max_figures,
-            "max_components": ml.detector_max_components,
-            "max_panels": ml.detector_max_panels,
-            "max_regions": ml.detector_max_regions,
-            "dedupe_iou": ml.detector_dedupe_iou,
-        }
         out = []
         for j in jobs:
             img = imgs.get(j.image_record_id)
@@ -308,14 +137,6 @@ async def lease():
                 # For video/animated: the agent samples at this cadence.
                 "frame_interval_seconds": ml.video_frame_interval_seconds,
                 "max_frames": ml.video_max_frames,
-                # The embedding model the agent must use for concept crops + the
-                # whole-image 'embed' task, so its vectors land in the SAME space
-                # the heads trained in. Server-announced FROM THE SETTING → the
-                # agent stays model-agnostic; an operator swap is a setting + a
-                # re-embed, never an agent change.
-                "embed_model_name": ml.embedder_model_name,
-                "embed_version": ml.embedder_model_version,
-                "detectors": detectors,
             })
     return jsonify({"jobs": out})
 
@@ -360,34 +181,6 @@ async def submit():
     return jsonify({"ok": True, "stored": len(regions)})
 
 
-@gpu_bp.route("/jobs/submit_embedding", methods=["POST"])
-async def submit_embedding():
-    """Store a whole-image SigLIP embedding (the 'embed' task) on image_record +
-    close the job. Body: {agent_id, job_id, embedding:[...], embedding_version}.
-    This is how the GPU agent re-embeds the library under a new model (#1190) —
-    much faster than the CPU ml-worker at higher resolutions."""
-    body = await request.get_json(silent=True) or {}
-    agent_id = str(body.get("agent_id") or "agent")
-    job_id = body.get("job_id")
-    embedding = body.get("embedding")
-    version = body.get("embedding_version")
-    if job_id is None or not embedding or not version:
-        return jsonify({"error": "job_id, embedding, embedding_version required"}), 400
-    async with get_session() as session:
-        if not await _agent_authed(session):
-            return jsonify({"error": "unauthorized"}), 401
-        job = await session.get(GpuJob, int(job_id))
-        if job is None or job.status != "leased" or job.lease_token != agent_id:
-            return jsonify({"error": "lease_invalid"}), 409
-        img = await session.get(ImageRecord, job.image_record_id)
-        if img is not None:
-            img.siglip_embedding = embedding
-            img.siglip_model_version = version
-        await GpuJobService(session).complete(agent_id, int(job_id))
-        await session.commit()
-    return jsonify({"ok": True})
-
-
 @gpu_bp.route("/jobs/fail", methods=["POST"])
 async def fail():
     body = await request.get_json(silent=True) or {}

backend/app/api/ml_admin.py

@@ -1,104 +1,68 @@
-"""ML admin API: settings + backfill trigger."""
+"""ML admin API: settings, backfill trigger, centroid recompute trigger."""
 
 from quart import Blueprint, jsonify, request
 
 from ..extensions import get_session
 from ..models import MLSettings
-from ..services.ml.heads import AUTO_APPLY_THRESHOLD_MAX, AUTO_APPLY_THRESHOLD_MIN
 
 ml_admin_bp = Blueprint("ml_admin", __name__, url_prefix="/api/ml")
 
 
-# Crop-proposer / detector config (#134). Announced to the GPU agent in the lease
-# → tunable here with no restart. weights = ultralytics name | URL | hf_repo::file
-# (empty, or enabled off, skips that proposer).
-_DETECTOR_FIELDS = (
-    "detector_person_enabled",
-    "detector_person_weights",
-    "detector_person_conf",
-    "detector_anatomy_enabled",
-    "detector_anatomy_weights",
-    "detector_anatomy_conf",
-    "detector_panel_enabled",
-    "detector_panel_weights",
-    "detector_panel_conf",
-    "detector_max_figures",
-    "detector_max_components",
-    "detector_max_panels",
-    "detector_max_regions",
-    "detector_dedupe_iou",
-)
-
 _EDITABLE = (
-    "cpu_embed_enabled",
+    "suggestion_threshold_character",
+    "suggestion_threshold_general",
+    "centroid_similarity_threshold",
+    "min_reference_images",
+    "tagger_store_floor",
     "video_frame_interval_seconds",
     "video_max_frames",
+    "video_min_tag_frames",
     "head_min_positives",
     "head_auto_apply_precision",
     "head_auto_apply_enabled",
     "head_auto_apply_min_positives",
-    "ccip_match_threshold",
-    "ccip_auto_apply_enabled",
-    "ccip_auto_apply_threshold",
-    "presentation_auto_apply_enabled",
-    "presentation_auto_apply_threshold",
-    "presentation_conflict_threshold",
-    "process_auto_apply_enabled",
-    "process_auto_apply_threshold",
-    "process_conflict_threshold",
-    "embedder_model_name",
-    "embedder_model_version",
-    *_DETECTOR_FIELDS,
 )
 
 
-# Supported embedders for the Settings dropdown — all 1152-d so a swap is a
-# drop-in (re-embed + retrain, no schema change). Server-authoritative so the UI
-# never free-types a model name.
-SUPPORTED_EMBEDDERS = (
-    {
-        "name": "google/siglip2-so400m-patch16-512",
-        "version": "siglip2-so400m-patch16-512",
-        "label": "SigLIP 2 · so400m · 512px (recommended)",
-        "dim": 1152,
-    },
-    {
-        "name": "google/siglip2-so400m-patch16-384",
-        "version": "siglip2-so400m-patch16-384",
-        "label": "SigLIP 2 · so400m · 384px (faster)",
-        "dim": 1152,
-    },
-    {
-        "name": "google/siglip-so400m-patch14-384",
-        "version": "siglip-so400m-patch14-384",
-        "label": "SigLIP 1 · so400m · 384px (original)",
-        "dim": 1152,
-    },
-)
-
-
-@ml_admin_bp.route("/embedder-models", methods=["GET"])
-async def embedder_models():
-    return jsonify({"models": list(SUPPORTED_EMBEDDERS)})
-
-
 @ml_admin_bp.route("/settings", methods=["GET"])
 async def get_settings():
+    from sqlalchemy import select
+
     async with get_session() as session:
-        s = await MLSettings.load(session)
-    # Table-driven off _EDITABLE (which PATCH also writes) so a new settings field
-    # can never be silently absent from GET — the split that historically dropped
-    # fields. _EDITABLE already includes *_DETECTOR_FIELDS.
-    return jsonify({f: getattr(s, f) for f in _EDITABLE})
+        s = (
+            await session.execute(select(MLSettings).where(MLSettings.id == 1))
+        ).scalar_one()
+    return jsonify(
+        {
+            "suggestion_threshold_character": s.suggestion_threshold_character,
+            "suggestion_threshold_general": s.suggestion_threshold_general,
+            "centroid_similarity_threshold": s.centroid_similarity_threshold,
+            "min_reference_images": s.min_reference_images,
+            "tagger_store_floor": s.tagger_store_floor,
+            "video_frame_interval_seconds": s.video_frame_interval_seconds,
+            "video_max_frames": s.video_max_frames,
+            "video_min_tag_frames": s.video_min_tag_frames,
+            "tagger_model_version": s.tagger_model_version,
+            "embedder_model_version": s.embedder_model_version,
+            "head_min_positives": s.head_min_positives,
+            "head_auto_apply_precision": s.head_auto_apply_precision,
+            "head_auto_apply_enabled": s.head_auto_apply_enabled,
+            "head_auto_apply_min_positives": s.head_auto_apply_min_positives,
+        }
+    )
 
 
 @ml_admin_bp.route("/settings", methods=["PATCH"])
 async def patch_settings():
+    from sqlalchemy import select
+
     body = await request.get_json()
     if not isinstance(body, dict):
         return jsonify({"error": "body must be an object"}), 400
     async with get_session() as session:
-        s = await MLSettings.load(session)
+        s = (
+            await session.execute(select(MLSettings).where(MLSettings.id == 1))
+        ).scalar_one()
 
         # Merge the patch over current values, then validate the result as a
         # whole — the store-floor invariant couples three fields, so they
@@ -119,53 +83,38 @@ async def patch_settings():
 
 
 def _validate(p: dict) -> str | None:
-    """Returns an error string if the proposed settings are invalid, else None."""
-    # Video embedding (#747).
+    """Returns an error string if the proposed settings are invalid, else None.
+
+    Invariant (plan-task #764): the per-category suggestion thresholds can't
+    drop below tagger_store_floor — nothing below the floor is stored, so a
+    lower threshold would silently surface nothing in that gap. The UI clamps
+    the sliders to the floor; this is the server-side backstop.
+    """
+    floor = p["tagger_store_floor"]
+    if not (0.0 <= floor <= 1.0):
+        return "tagger_store_floor must be between 0 and 1"
+    for cat in ("character", "general"):
+        if p[f"suggestion_threshold_{cat}"] < floor:
+            return (
+                f"suggestion_threshold_{cat} cannot be below tagger_store_floor "
+                f"({floor}) — predictions below the floor are not stored"
+            )
+    # Video tagging (#747).
     if p["video_frame_interval_seconds"] <= 0:
         return "video_frame_interval_seconds must be > 0"
     if p["video_max_frames"] < 1:
         return "video_max_frames must be >= 1"
+    if p["video_min_tag_frames"] < 1:
+        return "video_min_tag_frames must be >= 1"
+    if p["video_min_tag_frames"] > p["video_max_frames"]:
+        return "video_min_tag_frames cannot exceed video_max_frames"
     # Head training (#114).
     if int(p["head_min_positives"]) < 1:
         return "head_min_positives must be >= 1"
-    if not (AUTO_APPLY_THRESHOLD_MIN <= float(p["head_auto_apply_precision"]) <= AUTO_APPLY_THRESHOLD_MAX):
-        return f"head_auto_apply_precision must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
+    if not (0.5 <= float(p["head_auto_apply_precision"]) <= 0.999):
+        return "head_auto_apply_precision must be between 0.5 and 0.999"
     if int(p["head_auto_apply_min_positives"]) < 1:
         return "head_auto_apply_min_positives must be >= 1"
-    if not (AUTO_APPLY_THRESHOLD_MIN <= float(p["ccip_match_threshold"]) <= AUTO_APPLY_THRESHOLD_MAX):
-        return f"ccip_match_threshold must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
-    if not (AUTO_APPLY_THRESHOLD_MIN <= float(p["ccip_auto_apply_threshold"]) <= AUTO_APPLY_THRESHOLD_MAX):
-        return f"ccip_auto_apply_threshold must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
-    # Presentation chrome auto-hide (#141). Auto-apply runs high (hiding is
-    # consequential); the conflict cut is a plain probability [0,1].
-    if not (AUTO_APPLY_THRESHOLD_MIN <= float(p["presentation_auto_apply_threshold"]) <= AUTO_APPLY_THRESHOLD_MAX):
-        return f"presentation_auto_apply_threshold must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
-    if not (0.0 <= float(p["presentation_conflict_threshold"]) <= 1.0):
-        return "presentation_conflict_threshold must be between 0 and 1"
-    # Process auto-apply (#1464). wip/editor stay VISIBLE so a false apply is
-    # low-harm (excludes-from-training + a review flag), but keep the same bar.
-    if not (AUTO_APPLY_THRESHOLD_MIN <= float(p["process_auto_apply_threshold"]) <= AUTO_APPLY_THRESHOLD_MAX):
-        return f"process_auto_apply_threshold must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
-    if not (0.0 <= float(p["process_conflict_threshold"]) <= 1.0):
-        return "process_conflict_threshold must be between 0 and 1"
-    # Embedder model swap (#1190): both must be non-empty. Changing them means a
-    # different embedding space — the operator must re-embed + retrain after.
-    for key in ("embedder_model_name", "embedder_model_version"):
-        if not str(p[key]).strip():
-            return f"{key} must not be empty"
-    # Crop proposers (#134). Weights may be empty (that proposer is just off);
-    # confidences are probabilities; caps are positive counts; IoU is [0,1].
-    for key in ("detector_person_conf", "detector_anatomy_conf", "detector_panel_conf"):
-        if not (0.0 <= float(p[key]) <= 1.0):
-            return f"{key} must be between 0 and 1"
-    for key in (
-        "detector_max_figures", "detector_max_components",
-        "detector_max_panels", "detector_max_regions",
-    ):
-        if int(p[key]) < 1:
-            return f"{key} must be >= 1"
-    if not (0.0 <= float(p["detector_dedupe_iou"]) <= 1.0):
-        return "detector_dedupe_iou must be between 0 and 1"
     return None
 
 
@@ -175,3 +124,11 @@ async def trigger_backfill():
 
     r = backfill.delay()
     return jsonify({"celery_task_id": r.id}), 202
+
+
+@ml_admin_bp.route("/recompute-centroids", methods=["POST"])
+async def trigger_recompute():
+    from ..tasks.ml import recompute_centroids
+
+    r = recompute_centroids.delay()
+    return jsonify({"celery_task_id": r.id}), 202

backend/app/api/posts.py

@@ -3,28 +3,12 @@
 from quart import Blueprint, jsonify, request
 
 from ..extensions import get_session
-from ..models import ImportSettings, Post
-from ..services import interpreter_client as ic
 from ..services.post_feed_service import PostFeedService
 from ..services.source_service import KNOWN_PLATFORMS
-from ..utils.text import html_to_plain
 from ._responses import error_response as _bad
 
 posts_bp = Blueprint("posts", __name__, url_prefix="/api/posts")
 
-_TRANSLATION_OVERRIDES = ("auto", "force", "original")
-
-
-def _queue_for_sweep(post: Post) -> None:
-    """Mark a post untranslated (all translation columns NULL) so the periodic
-    sweep re-runs it under its new override — used when Interpreter is down and we
-    can't translate inline."""
-    post.post_title_translated = None
-    post.description_translated = None
-    post.translated_source_lang = None
-    post.translation_engine_version = None
-    post.translated_at = None
-
 
 @posts_bp.route("", methods=["GET"])
 async def list_posts():
@@ -98,70 +82,3 @@ async def get_post(post_id: int):
     if item is None:
         return _bad("not_found", status=404, detail=f"post id={post_id}")
     return jsonify(item)
-
-
-@posts_bp.route("/<int:post_id>/translation-override", methods=["POST"])
-async def set_translation_override(post_id: int):
-    """Sticky per-post translation override (milestone 155). Body:
-    ``{"override": "auto" | "force" | "original"}``.
-
-    'original' keeps the original (clears any stored translation now — no
-    Interpreter needed). 'force'/'auto' translate the post immediately if the
-    service is up (force bypasses the acceptance floor; auto re-runs the gate);
-    if it's down we save the flag and mark the post untranslated so the next sweep
-    applies it. The override persists, so the sweep + Re-translate-all keep
-    honoring it. Returns the updated translation fields + an ``applied`` status."""
-    body = await request.get_json(silent=True) or {}
-    override = body.get("override")
-    if override not in _TRANSLATION_OVERRIDES:
-        return _bad(
-            "invalid_override",
-            detail=f"override must be one of {list(_TRANSLATION_OVERRIDES)}",
-        )
-
-    # Lazy import (mirrors settings.py) so the API module doesn't pull the celery
-    # task graph at import time.
-    from ..tasks.translation import _store_translation, _translate_field
-
-    async with get_session() as session:
-        post = await session.get(Post, post_id)
-        if post is None:
-            return _bad("not_found", status=404, detail=f"post id={post_id}")
-        post.translation_override = override
-        cfg = await ImportSettings.load(session)
-        target = (cfg.translation_target_lang or "en").strip() or "en"
-
-        if override == "original":
-            _store_translation(post, (None, None, None), (None, None, None), target)
-            applied = "cleared"
-        else:
-            base_url = (cfg.interpreter_base_url or "").strip()
-            if cfg.translation_enabled and base_url and ic.health(base_url):
-                force = override == "force"
-                title = (post.post_title or "").strip()
-                desc = (html_to_plain(post.description) if post.description else "") or ""
-                desc = desc.strip()
-                mc = cfg.translation_min_confidence
-                try:
-                    title_res = _translate_field(title, base_url, target, mc, force=force)
-                    desc_res = _translate_field(desc, base_url, target, mc, force=force)
-                except ic.InterpreterUnavailable:
-                    _queue_for_sweep(post)
-                    applied = "queued"
-                else:
-                    _store_translation(post, title_res, desc_res, target)
-                    applied = "translated"
-            else:
-                # Disabled / no URL / unhealthy → let the sweep apply it later.
-                _queue_for_sweep(post)
-                applied = "queued"
-
-        await session.commit()
-        return jsonify({
-            "id": post.id,
-            "translation_override": post.translation_override,
-            "post_title_translated": post.post_title_translated,
-            "description_translated": post.description_translated,
-            "translated_source_lang": post.translated_source_lang,
-            "applied": applied,
-        })

backend/app/api/settings.py

@@ -1,24 +1,12 @@
 """Settings API: import filters, system stats."""
 
-import asyncio
 import secrets
 
 from quart import Blueprint, jsonify, request
-from sqlalchemy import func, or_, select
+from sqlalchemy import func, select
 
 from ..extensions import get_session
-from ..models import (
-    AppSetting,
-    Artist,
-    ImageRecord,
-    ImportBatch,
-    ImportSettings,
-    ImportTask,
-    Post,
-    Tag,
-    TaskRun,
-)
-from ..services import interpreter_client as ic
+from ..models import AppSetting, Artist, ImageRecord, ImportBatch, ImportSettings, ImportTask, Tag
 
 settings_bp = Blueprint("settings", __name__, url_prefix="/api")
 
@@ -44,12 +32,6 @@ _EDITABLE_FIELDS = (
     "extdl_mediafire_enabled",
     "extdl_dropbox_enabled",
     "extdl_pixeldrain_enabled",
-    "translation_enabled",
-    "interpreter_base_url",
-    "translation_target_lang",
-    "translation_min_confidence",
-    "wip_title_tagging_enabled",
-    "wip_soft_title_tagging_enabled",
 )
 
 # Per-host external-download toggles — all plain booleans, validated uniformly.
@@ -66,9 +48,28 @@ _EXTDL_TOGGLE_FIELDS = (
 async def get_import_settings():
     async with get_session() as session:
         row = await ImportSettings.load(session)
-    # Table-driven off _EDITABLE_FIELDS (which PATCH also writes) so a new field
-    # can't be silently absent from GET.
-    return jsonify({f: getattr(row, f) for f in _EDITABLE_FIELDS})
+    return jsonify({
+        "min_width": row.min_width,
+        "min_height": row.min_height,
+        "skip_transparent": row.skip_transparent,
+        "transparency_threshold": row.transparency_threshold,
+        "skip_single_color": row.skip_single_color,
+        "single_color_threshold": row.single_color_threshold,
+        "single_color_tolerance": row.single_color_tolerance,
+        "phash_threshold": row.phash_threshold,
+        "download_rate_limit_seconds": row.download_rate_limit_seconds,
+        "download_validate_files": row.download_validate_files,
+        "download_schedule_default_seconds": row.download_schedule_default_seconds,
+        "download_event_retention_days": row.download_event_retention_days,
+        "download_failure_warning_threshold": row.download_failure_warning_threshold,
+        "series_suggest_enabled": row.series_suggest_enabled,
+        "series_suggest_threshold": row.series_suggest_threshold,
+        "extdl_mega_enabled": row.extdl_mega_enabled,
+        "extdl_gdrive_enabled": row.extdl_gdrive_enabled,
+        "extdl_mediafire_enabled": row.extdl_mediafire_enabled,
+        "extdl_dropbox_enabled": row.extdl_dropbox_enabled,
+        "extdl_pixeldrain_enabled": row.extdl_pixeldrain_enabled,
+    })
 
 
 @settings_bp.route("/settings/import", methods=["PATCH"])
@@ -127,41 +128,12 @@ async def update_import_settings():
     for tog in _EXTDL_TOGGLE_FIELDS:
         if tog in body and not isinstance(body[tog], bool):
             return jsonify({"error": f"{tog} must be a boolean"}), 400
-    # Translation (#143): base URL may be empty (feature off until set — no
-    # default host; the operator points it at their own Interpreter proxy).
-    if "translation_enabled" in body and not isinstance(
-        body["translation_enabled"], bool
-    ):
-        return jsonify({"error": "translation_enabled must be a boolean"}), 400
-    for key in ("interpreter_base_url", "translation_target_lang"):
-        if key in body and not isinstance(body[key], str):
-            return jsonify({"error": f"{key} must be a string"}), 400
-    # Acceptance floor (milestone 155): latin-script translations below this
-    # Interpreter confidence are kept as the original.
-    if "translation_min_confidence" in body:
-        v = body["translation_min_confidence"]
-        if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
-            return jsonify(
-                {"error": "translation_min_confidence must be a number in [0, 1]"}
-            ), 400
     if "series_suggest_threshold" in body:
         v = body["series_suggest_threshold"]
         if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
             return jsonify(
                 {"error": "series_suggest_threshold must be a number in [0, 1]"}
             ), 400
-    if "wip_title_tagging_enabled" in body and not isinstance(
-        body["wip_title_tagging_enabled"], bool
-    ):
-        return jsonify(
-            {"error": "wip_title_tagging_enabled must be a boolean"}
-        ), 400
-    if "wip_soft_title_tagging_enabled" in body and not isinstance(
-        body["wip_soft_title_tagging_enabled"], bool
-    ):
-        return jsonify(
-            {"error": "wip_soft_title_tagging_enabled must be a boolean"}
-        ), 400
 
     async with get_session() as session:
         row = await ImportSettings.load(session)
@@ -173,18 +145,6 @@ async def update_import_settings():
     return await get_import_settings()
 
 
-@settings_bp.route("/settings/wip-title/scan", methods=["POST"])
-async def wip_title_scan():
-    """Enqueue the back-catalogue WIP-title scan (task #1458 Settings button):
-    apply the `wip` system tag to EXISTING posts whose title declares
-    work-in-progress. New imports are tagged live by the importer; this catches
-    the existing library. Returns the Celery task id (202)."""
-    from ..tasks.maintenance import backfill_wip_title_tags
-
-    r = backfill_wip_title_tags.delay()
-    return jsonify({"celery_task_id": r.id}), 202
-
-
 @settings_bp.route("/system/stats", methods=["GET"])
 async def system_stats():
     async with get_session() as session:
@@ -310,159 +270,3 @@ async def rotate_extension_api_key():
             row.value = new_value
         await session.commit()
     return jsonify({"key": new_value})
-
-
-# --- Translation (#143): live status + manual "Translate now" --------------
-
-
-@settings_bp.route("/settings/translation/status", methods=["GET"])
-async def translation_status():
-    """For the Settings card: is it on, is a URL set, is the service reachable,
-    and how many posts still await translation. Health runs the sync client in a
-    thread so the event loop isn't blocked."""
-    translation_tasks = (
-        "backend.app.tasks.translation.translate_posts",
-        "backend.app.tasks.translation.retranslate_posts",
-    )
-    async with get_session() as session:
-        cfg = await ImportSettings.load(session)
-        untranslated = (await session.execute(
-            select(func.count(Post.id))
-            .where(Post.translated_source_lang.is_(None))
-            .where(or_(
-                Post.post_title.is_not(None), Post.description.is_not(None),
-            ))
-        )).scalar_one()
-        # Live progress: is a sweep running now, and what did the last one do?
-        # (run-until-done re-enqueues itself, so `active` stays true across a
-        # bulk re-translate; `last_run` surfaces a completed run's outcome.)
-        active = (await session.execute(
-            select(func.count(TaskRun.id))
-            .where(TaskRun.task_name.in_(translation_tasks))
-            .where(TaskRun.status == "running")
-        )).scalar_one()
-        last = (await session.execute(
-            select(TaskRun.task_name, TaskRun.status, TaskRun.finished_at)
-            .where(TaskRun.task_name.in_(translation_tasks))
-            .where(TaskRun.finished_at.is_not(None))
-            .order_by(TaskRun.finished_at.desc())
-            .limit(1)
-        )).first()
-    base_url = (cfg.interpreter_base_url or "").strip()
-    healthy = await asyncio.to_thread(ic.health, base_url) if base_url else False
-    return jsonify({
-        "enabled": cfg.translation_enabled,
-        "base_url_set": bool(base_url),
-        "healthy": healthy,
-        "untranslated_count": int(untranslated),
-        "active": int(active) > 0,
-        "last_run": {
-            "task": last[0].rsplit(".", 1)[-1],
-            "status": last[1],
-            "finished_at": last[2].isoformat() if last[2] else None,
-        } if last else None,
-    })
-
-
-@settings_bp.route("/settings/translation/test", methods=["POST"])
-async def translation_test():
-    """On-demand reachability check for a GIVEN Interpreter base URL (the Settings
-    'Test connection' button) — pings /v1/health without saving, so the operator
-    can verify a URL before enabling. Health runs in a thread (sync client)."""
-    body = await request.get_json()
-    base_url = ""
-    if isinstance(body, dict):
-        base_url = (body.get("base_url") or "").strip()
-    healthy = await asyncio.to_thread(ic.health, base_url) if base_url else False
-    return jsonify({"healthy": healthy})
-
-
-@settings_bp.route("/settings/translation/probe", methods=["POST"])
-async def translation_probe():
-    """Diagnostic for the Settings 'Test translation' box: translate a pasted
-    snippet WITHOUT saving anything, returning what Interpreter *detected*
-    (language + confidence) alongside the result. Lets the operator see why a
-    given string was (mis-)detected — e.g. a short English title flagged as
-    another language — so a detection guard can be tuned from real numbers.
-    Read-only: no post is touched. Uses the currently-saved base URL + target."""
-    body = await request.get_json(silent=True)
-    body = body if isinstance(body, dict) else {}
-    text = (body.get("text") or "").strip()
-    if not text:
-        return jsonify({"error": "provide text to translate"}), 400
-    async with get_session() as session:
-        cfg = await ImportSettings.load(session)
-    base_url = (cfg.interpreter_base_url or "").strip()
-    if not base_url:
-        return jsonify({"error": "no Interpreter base URL is set"}), 400
-    target = (cfg.translation_target_lang or "en").strip() or "en"
-    try:
-        res = await asyncio.to_thread(
-            ic.translate, [text], base_url=base_url, target=target,
-        )
-    except ic.InterpreterUnavailable as e:
-        return jsonify({"error": f"Interpreter unavailable: {e}"}), 503
-    except ic.InterpreterBadRequest as e:
-        return jsonify({"error": f"Interpreter rejected the request: {e}"}), 400
-    translations = res.get("translations") or []
-    return jsonify({
-        "target": target,
-        "detected_lang": res.get("detected_lang"),
-        "detected_confidence": res.get("detected_confidence"),
-        "engine": res.get("engine"),
-        "engine_version": res.get("engine_version"),
-        "translated": translations[0] if translations else None,
-    })
-
-
-@settings_bp.route("/settings/translation/run", methods=["POST"])
-async def translation_run():
-    """Enqueue the translate sweep now (the Settings 'Translate now' button).
-    Runs in drain mode — run-until-done — so one press chases the whole
-    untranslated backlog to zero rather than a single 300-post chunk."""
-    async with get_session() as session:
-        cfg = await ImportSettings.load(session)
-    if not cfg.translation_enabled or not (cfg.interpreter_base_url or "").strip():
-        return jsonify(
-            {"error": "translation is disabled or no base URL is set"}
-        ), 400
-    from ..tasks.translation import translate_posts
-
-    r = translate_posts.delay(drain=True)
-    return jsonify({"celery_task_id": r.id}), 202
-
-
-@settings_bp.route("/settings/translation/retranslate", methods=["POST"])
-async def translation_retranslate():
-    """Re-translate stored translations after a model change (m146). Body:
-    ``{"artist_id": <int>}`` aims at one artist; ``{"all": true}`` re-runs every
-    artist. ``all`` must be explicit so an empty/typo body can't wipe everything.
-    Clears the scoped translations and enqueues the run-until-done retranslate
-    sweep (the Interpreter cache re-translates on a changed model, is cache-fast
-    otherwise). Same enabled + base-URL guard as 'Translate now'."""
-    body = await request.get_json(silent=True)
-    body = body if isinstance(body, dict) else {}
-    artist_id = body.get("artist_id")
-    do_all = bool(body.get("all"))
-    if artist_id is None and not do_all:
-        return jsonify(
-            {"error": "provide artist_id, or all=true to re-translate everything"}
-        ), 400
-    if artist_id is not None:
-        try:
-            artist_id = int(artist_id)
-        except (TypeError, ValueError):
-            return jsonify({"error": "artist_id must be an integer"}), 400
-
-    async with get_session() as session:
-        cfg = await ImportSettings.load(session)
-    if not cfg.translation_enabled or not (cfg.interpreter_base_url or "").strip():
-        return jsonify(
-            {"error": "translation is disabled or no base URL is set"}
-        ), 400
-    from ..tasks.translation import retranslate_posts
-
-    # artist_id wins when both are sent; otherwise all=true → None (every artist).
-    artist_ids = [artist_id] if artist_id is not None else None
-    r = retranslate_posts.delay(artist_ids=artist_ids)
-    return jsonify({"celery_task_id": r.id}), 202

backend/app/api/sources.py

@@ -136,25 +136,6 @@ async def delete_source(source_id: int):
     return "", 204
 
 
-@sources_bp.route("/<int:source_id>/reassign", methods=["POST"])
-async def reassign_source(source_id: int):
-    """Move this source (and the content it brought in) to another artist
-    (#130). Files don't move — the slug is immutable — so this just re-attributes
-    the source, its posts, and its images. Body: {target_artist_id}."""
-    body = await request.get_json(silent=True) or {}
-    target = body.get("target_artist_id")
-    if not isinstance(target, int):
-        return _bad("invalid_body", detail="target_artist_id (int) required")
-    async with get_session() as session:
-        try:
-            record = await SourceService(session).reassign(source_id, target)
-        except LookupError:
-            return _bad("not_found", status=404)
-        except ArtistNotFoundError:
-            return _bad("artist_not_found", detail="target artist not found", status=404)
-    return jsonify(record.to_dict())
-
-
 @sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
 async def set_backfill(source_id: int):
     """Plan #693/#697 + #830: start/stop a backfill, or start a recovery /
@@ -230,6 +211,50 @@ async def set_backfill(source_id: int):
     return jsonify(record.to_dict())
 
 
+@sources_bp.route("/<int:source_id>/preview", methods=["POST"])
+async def preview_source_endpoint(source_id: int):
+    """Plan #708 B4: dry-run — count what a backfill WOULD download for a native
+    platform (Patreon today), without downloading. Walks the first few feed pages
+    and counts media not already in the seen/dead ledgers. Returns
+    {total_new, posts_scanned, pages_scanned, has_more, sample[]} or 409 + reason
+    (unresolvable campaign id / auth / drift). 400 for gallery-dl platforms (no
+    cheap dry-run — their verify is a slow --simulate)."""
+    from pathlib import Path
+
+    from ..services.credential_service import CredentialService
+    from ..services.download_backends import preview_source, uses_native_ingester
+    from ..tasks._sync_engine import sync_session_factory
+    from .credentials import _get_crypto
+
+    async with get_session() as session:
+        rec = await SourceService(session).get(source_id)
+        if rec is None:
+            return _bad("not_found", status=404)
+        if not uses_native_ingester(rec.platform):
+            return _bad(
+                "unsupported",
+                detail="Preview is only available for native-ingester platforms.",
+                status=400,
+            )
+        cred = CredentialService(session, _get_crypto())
+        cookies_path = await cred.get_cookies_path(rec.platform)
+
+    # The walk + ledger reads are sync (run off the request loop); the process
+    # sync engine is the same one the download task uses.
+    result = await preview_source(
+        platform=rec.platform,
+        url=rec.url,
+        source_id=source_id,
+        config_overrides=rec.config_overrides or {},
+        cookies_path=str(cookies_path) if cookies_path else None,
+        images_root=Path("/images"),
+        sync_session_factory=sync_session_factory(),
+    )
+    if "error" in result:
+        return _bad("preview_failed", detail=result["error"], status=409)
+    return jsonify(result)
+
+
 @sources_bp.route("/<int:source_id>/check", methods=["POST"])
 async def check_source(source_id: int):
     """FC-3c: enqueue a download for this source.

backend/app/api/suggestions.py

@@ -3,19 +3,37 @@
 from quart import Blueprint, jsonify, request
 
 from ..extensions import get_session
+from ..models import Tag, TagAllowlist
 from ..services.ml.allowlist import AllowlistService
 from ..services.ml.suggestions import SuggestionService
 
 suggestions_bp = Blueprint("suggestions", __name__, url_prefix="/api")
 
 
+async def _accept_payload(session, svc, newly_added: bool, tag_id: int) -> dict:
+    """Shape the accept/alias response. When accepting newly allowlists a tag,
+    include the coverage PROJECTION (at the tag's threshold) so the UI can show
+    a non-blocking "auto-applying to ~N images" toast — the actual apply runs
+    async via apply_allowlist_tags, so this is an estimate, not a post-hoc
+    count (#7)."""
+    payload = {"allowlisted": newly_added}
+    if newly_added:
+        tag = await session.get(Tag, tag_id)
+        row = await session.get(TagAllowlist, tag_id)
+        payload["tag_id"] = tag_id
+        payload["tag_name"] = tag.name if tag is not None else None
+        payload["projected_count"] = await svc.coverage(
+            tag_id, row.min_confidence if row is not None else 0.90,
+        )
+    return payload
+
+
 @suggestions_bp.route("/images/<int:image_id>/suggestions", methods=["GET"])
 async def get_suggestions(image_id: int):
-    # ?min=<float> overrides the per-head suggest thresholds for INCLUSION. The
-    # rail sends min=0 in its single per-image fetch to get EVERY head (each row
-    # still carries above_threshold vs its natural cut), then derives the panel
-    # (above_threshold) and the typed dropdown (all, filtered by text) client-side
-    # — no second request. Omitted → only above-threshold rows.
+    # ?min=<float> overrides the configured per-category thresholds so the typed
+    # tag-input dropdown can surface EVERY stored prediction (min=0), including
+    # low-confidence actions/features, in canonical formatting. Omitted → the
+    # curated above-threshold list the Suggestions panel uses.
     override = None
     raw_min = request.args.get("min")
     if raw_min is not None:
@@ -37,19 +55,16 @@ async def get_suggestions(image_id: int):
                         "category": s.category,
                         "score": round(s.score, 4),
                         "source": s.source,
-                        # whether the score cleared the head's own suggest cut.
-                        # The single min=0 fetch returns every head; the panel
-                        # shows above_threshold, the typed dropdown shows all and
-                        # annotates each match with its score.
-                        "above_threshold": s.above_threshold,
+                        "creates_new_tag": s.creates_new_tag,
+                        # raw model key (alias is stored under this) + whether an
+                        # operator alias produced this suggestion — drive the
+                        # modal's "Treat as alias"/"Remove alias" affordances.
+                        "raw_name": s.raw_name,
+                        "via_alias": s.via_alias,
                         # operator dismissed this tag for this image — surfaced
                         # (not dropped) so the rail can show it rejected + offer
                         # one-click un-reject.
                         "rejected": s.rejected,
-                        # the crop region that produced this tag (#1206) —
-                        # {bbox,kind,detector} or null (whole-image won). Drives
-                        # the hover→overlay highlight.
-                        "grounding": s.grounding,
                     }
                     for s in items
                 ]
@@ -68,9 +83,43 @@ async def accept_suggestion(image_id: int):
         return jsonify({"error": "tag_id required"}), 400
     tag_id = body["tag_id"]
     async with get_session() as session:
-        await AllowlistService(session).accept(image_id, tag_id)
+        svc = AllowlistService(session)
+        newly_added = await svc.accept(image_id, tag_id)
+        payload = await _accept_payload(session, svc, newly_added, tag_id)
         await session.commit()
-    return jsonify({"accepted": True, "tag_id": tag_id})
+    if newly_added:
+        from ..tasks.ml import apply_allowlist_tags
+
+        apply_allowlist_tags.delay(tag_id=tag_id)
+    return jsonify(payload)
+
+
+@suggestions_bp.route(
+    "/images/<int:image_id>/suggestions/alias", methods=["POST"]
+)
+async def alias_suggestion(image_id: int):
+    body = await request.get_json()
+    required = {"alias_string", "alias_category", "canonical_tag_id"}
+    if not body or not required.issubset(body):
+        return jsonify({"error": f"required: {sorted(required)}"}), 400
+    canonical_tag_id = body["canonical_tag_id"]
+    async with get_session() as session:
+        svc = AllowlistService(session)
+        newly_added = await svc.add_alias_and_accept(
+            image_id,
+            body["alias_string"],
+            body["alias_category"],
+            canonical_tag_id,
+        )
+        payload = await _accept_payload(
+            session, svc, newly_added, canonical_tag_id,
+        )
+        await session.commit()
+    if newly_added:
+        from ..tasks.ml import apply_allowlist_tags
+
+        apply_allowlist_tags.delay(tag_id=canonical_tag_id)
+    return jsonify(payload)
 
 
 @suggestions_bp.route(

backend/app/api/tag_eval.py

@@ -0,0 +1,70 @@
+"""Tag-eval API (#1130): trigger + revisit the head-vs-centroid eval.
+
+The run + full report live in the tag_eval_run row, so the admin card rehydrates
+from GET (history / detail) on mount — the report survives navigation rather than
+living in transient frontend state.
+"""
+
+from quart import Blueprint, jsonify, request
+from sqlalchemy import select
+
+from ..extensions import get_session
+from ..models import TagEvalRun
+from ..services.ml.tag_eval import EvalAlreadyRunning, start_tag_eval_run
+
+tag_eval_bp = Blueprint("tag_eval", __name__, url_prefix="/api/tag-eval")
+
+
+def _serialize(run: TagEvalRun, *, include_report: bool) -> dict:
+    out = {
+        "id": run.id,
+        "params": run.params,
+        "status": run.status,
+        "started_at": run.started_at.isoformat() if run.started_at else None,
+        "finished_at": run.finished_at.isoformat() if run.finished_at else None,
+        "error": run.error,
+    }
+    if include_report:
+        out["report"] = run.report
+    return out
+
+
+@tag_eval_bp.route("", methods=["POST"])
+async def create():
+    body = await request.get_json(silent=True) or {}
+    params = body.get("params") or body or {}
+    async with get_session() as session:
+        try:
+            run_id = await session.run_sync(
+                lambda s: start_tag_eval_run(s, params)
+            )
+        except EvalAlreadyRunning as running:
+            return jsonify({
+                "error": "eval_already_running",
+                "running_id": int(running.args[0]),
+            }), 409
+        await session.commit()
+    return jsonify({"run_id": run_id, "status": "running"}), 202
+
+
+@tag_eval_bp.route("", methods=["GET"])
+async def history():
+    try:
+        limit = min(int(request.args.get("limit", "20")), 100)
+    except ValueError:
+        return jsonify({"error": "invalid_limit"}), 400
+    async with get_session() as session:
+        rows = (await session.execute(
+            select(TagEvalRun).order_by(TagEvalRun.id.desc()).limit(limit)
+        )).scalars().all()
+        # List is light — no full report (the detail endpoint carries it).
+        return jsonify({"runs": [_serialize(r, include_report=False) for r in rows]})
+
+
+@tag_eval_bp.route("/<int:run_id>", methods=["GET"])
+async def detail(run_id: int):
+    async with get_session() as session:
+        run = await session.get(TagEvalRun, run_id)
+        if run is None:
+            return jsonify({"error": "not_found"}), 404
+        return jsonify(_serialize(run, include_report=True))

backend/app/api/tags.py

@@ -1,17 +1,15 @@
 """Tags API: autocomplete, create, list/add/remove for an image."""
 
 from quart import Blueprint, jsonify, request
-from sqlalchemy import func, select
+from sqlalchemy import exists, select
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.exc import IntegrityError
 
 from ..extensions import get_session
-from ..models import Tag, TagHead, TagKind, TagPositiveConfirmation
-from ..models.tag import image_tag
-from ..models.tag_suggestion_rejection import TagSuggestionRejection
+from ..models import Tag, TagKind, TagPositiveConfirmation
+from ..models.tag_allowlist import TagAllowlist
 from ..services.bulk_tag_service import BulkTagService
 from ..services.ml.aliases import AliasService
-from ..services.ml.heads import ground_applied_tag
 from ..services.series_match_service import SeriesMatchService
 from ..services.series_service import SeriesError, SeriesService
 from ..services.tag_directory_service import TagDirectoryService
@@ -63,117 +61,6 @@ def _parse_bulk_ids(
     return ids, None
 
 
-# Application-source groupings (image_tag.source). HUMAN = operator signal;
-# AUTO = machine-applied (heads/CCIP, + legacy Camie ml_auto).
-_SOURCE_GROUPS = {
-    "human": ("manual", "ml_accepted"),
-    "manual": ("manual",),
-    "accepted": ("ml_accepted",),
-    "auto": ("head_auto", "ccip_auto", "ml_auto"),
-}
-
-
-@tags_bp.route("/tags/top", methods=["GET"])
-async def tags_top():
-    """Top tags by image count — a fast indexed aggregate for ANALYSIS (not the
-    paged UI directory, which is alphabetical + builds previews). Params:
-    ?kind=general|character|fandom|… ?source=all|human|manual|accepted|auto
-    ?limit=50 (cap 500) ?min_count=N. → {tags:[{tag_id,name,kind,count}]} desc."""
-    kind = _coerce_kind(request.args.get("kind"))
-    try:
-        limit = min(max(int(request.args.get("limit", "50")), 1), 500)
-    except ValueError:
-        return jsonify({"error": "limit must be an integer"}), 400
-    min_count = None
-    if "min_count" in request.args:
-        try:
-            min_count = int(request.args["min_count"])
-        except ValueError:
-            return jsonify({"error": "min_count must be an integer"}), 400
-    src_vals = _SOURCE_GROUPS.get((request.args.get("source") or "all").lower())
-
-    cnt = func.count(image_tag.c.image_record_id)
-    stmt = (
-        select(Tag.id, Tag.name, Tag.kind, cnt.label("count"))
-        .select_from(Tag)
-        .join(image_tag, image_tag.c.tag_id == Tag.id)
-        .group_by(Tag.id, Tag.name, Tag.kind)
-        .order_by(cnt.desc(), Tag.name.asc())
-        .limit(limit)
-    )
-    if kind is not None:
-        stmt = stmt.where(Tag.kind == kind)
-    if src_vals is not None:
-        stmt = stmt.where(image_tag.c.source.in_(src_vals))
-    if min_count is not None:
-        stmt = stmt.having(cnt >= min_count)
-    async with get_session() as session:
-        rows = (await session.execute(stmt)).all()
-    return jsonify({"tags": [
-        {
-            "tag_id": r.id, "name": r.name,
-            "kind": r.kind.value if hasattr(r.kind, "value") else str(r.kind),
-            "count": r.count,
-        }
-        for r in rows
-    ]})
-
-
-@tags_bp.route("/tags/<int:tag_id>/stats", methods=["GET"])
-async def tag_stats(tag_id: int):
-    """Per-tag dataset health: total + per-source application counts (human vs
-    machine), rejection count, and whether a trained head exists. Read-only,
-    analysis-shaped — backs concept-readiness + source-split decisions."""
-    async with get_session() as session:
-        tag = await session.get(Tag, tag_id)
-        if tag is None:
-            return jsonify({"error": "not found"}), 404
-        by_source = dict(
-            (
-                await session.execute(
-                    select(image_tag.c.source, func.count())
-                    .where(image_tag.c.tag_id == tag_id)
-                    .group_by(image_tag.c.source)
-                )
-            ).all()
-        )
-        rejected = (
-            await session.execute(
-                select(func.count())
-                .select_from(TagSuggestionRejection)
-                .where(TagSuggestionRejection.tag_id == tag_id)
-            )
-        ).scalar_one()
-        has_head = (
-            await session.execute(
-                select(func.count())
-                .select_from(TagHead)
-                .where(TagHead.tag_id == tag_id)
-            )
-        ).scalar_one() > 0
-    human = by_source.get("manual", 0) + by_source.get("ml_accepted", 0)
-    auto = (
-        by_source.get("head_auto", 0)
-        + by_source.get("ccip_auto", 0)
-        + by_source.get("ml_auto", 0)
-    )
-    return jsonify({
-        "tag_id": tag_id,
-        "name": tag.name,
-        "kind": tag.kind.value if hasattr(tag.kind, "value") else str(tag.kind),
-        "count_total": sum(by_source.values()),
-        "count_human": human,
-        "count_manual": by_source.get("manual", 0),
-        "count_accepted": by_source.get("ml_accepted", 0),
-        "count_auto": auto,
-        "count_head_auto": by_source.get("head_auto", 0),
-        "count_ccip_auto": by_source.get("ccip_auto", 0),
-        "count_rejected": rejected,
-        "by_source": by_source,
-        "has_head": has_head,
-    })
-
-
 @tags_bp.route("/tags/autocomplete", methods=["GET"])
 async def autocomplete():
     q = request.args.get("q", "")
@@ -311,21 +198,6 @@ async def confirm_tag_on_image(image_id: int, tag_id: int):
     return "", 204
 
 
-@tags_bp.route(
-    "/images/<int:image_id>/tags/<int:tag_id>/grounding", methods=["GET"]
-)
-async def tag_grounding(image_id: int, tag_id: int):
-    """Which crop region best explains an ALREADY-APPLIED tag on this image
-    (#1206 Step 4). Powers the hover→overlay highlight on applied tag chips,
-    mirroring the suggestion rail's live grounding. Computed on demand (applied
-    tags aren't scored live). → {grounding: {bbox,kind,detector}|null,
-    has_head: bool}; has_head False means the tag has no head to localize with,
-    so the chip shows no overlay."""
-    async with get_session() as session:
-        grounding, has_head = await ground_applied_tag(session, image_id, tag_id)
-    return jsonify({"grounding": grounding, "has_head": has_head})
-
-
 @tags_bp.route("/tags/<int:tag_id>", methods=["GET"])
 async def get_tag(tag_id: int):
     """Resolve a single tag (used by the gallery to label its active
@@ -340,7 +212,6 @@ async def get_tag(tag_id: int):
                 "name": tag.name,
                 "kind": tag.kind.value,
                 "fandom_id": tag.fandom_id,
-                "is_system": tag.is_system,
             }
         )
 
@@ -407,7 +278,6 @@ async def update_tag(tag_id: int):
                 "name": tag.name,
                 "kind": tag.kind.value,
                 "fandom_id": tag.fandom_id,
-                "is_system": tag.is_system,
             }
         )
 
@@ -427,6 +297,19 @@ async def merge_tag(source_id: int):
             status = 404 if "not found" in msg else 400
             return jsonify({"error": msg}), status
         await session.commit()
+        target_allowlisted = await session.scalar(
+            select(exists().where(TagAllowlist.tag_id == result.target_id))
+        )
+    if target_allowlisted:
+        from ..tasks.ml import apply_allowlist_tags
+
+        apply_allowlist_tags.delay(tag_id=result.target_id)
+    # Tag merge invalidates the target's centroid (the merged-in source
+    # tag's images now contribute to it). Daily list_drifted catches it
+    # within 24h, but eager recompute closes the suggestion-quality dip
+    # in the meantime. Audit 2026-06-02.
+    from ..tasks.ml import recompute_centroid
+    recompute_centroid.delay(result.target_id)
     return jsonify(
         {
             "target": {

backend/app/celery_app.py

@@ -7,7 +7,7 @@ Queues:
   download    — gallery-dl tasks (FC-3)
   scan        — periodic source checks (FC-3) — kept separate so long imports
                 don't starve the scheduler
-  maintenance — recovery sweeps, pHash backfill, GPU-queue coordination, etc.
+  maintenance — pHash recomputation, centroid rebuild, etc. (FC-2/FC-3)
   default     — anything not explicitly routed
 """
 
@@ -29,13 +29,11 @@ def make_celery() -> Celery:
             "backend.app.tasks.thumbnail",
             "backend.app.tasks.maintenance",
             "backend.app.tasks.ml",
-            "backend.app.tasks.gpu_queue",
             "backend.app.tasks.download",
             "backend.app.tasks.external",
             "backend.app.tasks.backup",
             "backend.app.tasks.admin",
             "backend.app.tasks.library_audit",
-            "backend.app.tasks.translation",
         ],
     )
     app.conf.update(
@@ -43,11 +41,6 @@ def make_celery() -> Celery:
         task_routes={
             "backend.app.tasks.import_file.*": {"queue": "import"},
             "backend.app.tasks.ml.*": {"queue": "ml"},
-            # GPU-queue coordination (backfill enqueues, orphan recovery,
-            # reprocess) is pure DB work — it rides the maintenance quick lane
-            # so the GPU agent pipeline works even on stacks that drop the
-            # (now-optional, B3) ml-worker container entirely.
-            "backend.app.tasks.gpu_queue.*": {"queue": "maintenance"},
             "backend.app.tasks.thumbnail.*": {"queue": "thumbnail"},
             "backend.app.tasks.download.*": {"queue": "download"},
             # External file-host fetches are downloads — same lane (they can run
@@ -64,20 +57,9 @@ def make_celery() -> Celery:
             "backend.app.tasks.backup.*": {"queue": "maintenance_long"},
             "backend.app.tasks.admin.*": {"queue": "maintenance_long"},
             "backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
-            # Translation backfill hits the LLM (~1–6s/item) → the long lane so it
-            # never starves the quick self-healing sweeps (#143).
-            "backend.app.tasks.translation.*": {"queue": "maintenance_long"},
         },
         # Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
         task_acks_late=True,
-        # Deploy graceful-shutdown safety: with acks_late, a task killed because
-        # it outran the container's stop-grace window (SIGKILL) is re-queued
-        # rather than silently lost. Safe because our long tasks are idempotent +
-        # chunked (translation per-post commit, downloads terminal-status, audits
-        # chunk) and the 5-min recovery sweeps re-drive anything left non-terminal
-        # — a re-run resumes cleanly and never corrupts. No redeliver-loop risk:
-        # heavy GPU work is tombstoned via gpu_queue, not run inline in a worker.
-        task_reject_on_worker_lost=True,
         worker_prefetch_multiplier=1,
         # Broker resilience (2026-06-24): a swarm overlay-network blip after a
         # redeploy left Redis healthy but transiently unreachable, and a worker
@@ -115,87 +97,30 @@ def make_celery() -> Celery:
                 "task": "backend.app.tasks.maintenance.cleanup_old_tasks",
                 "schedule": 86400.0,  # daily
             },
-            "cleanup-orphaned-temp-files": {
-                "task": "backend.app.tasks.maintenance.cleanup_orphaned_temp_files",
-                "schedule": 86400.0,  # daily — sweep .part/.partial left by a
-                # download/import killed mid-write (graceful-shutdown fallout)
+            "ml-backfill-daily": {
+                "task": "backend.app.tasks.ml.backfill",
+                "schedule": 86400.0,
+            },
+            "recompute-centroids-daily": {
+                "task": "backend.app.tasks.ml.recompute_centroids",
+                "schedule": 86400.0,
+            },
+            "apply-allowlist-sweep-daily": {
+                "task": "backend.app.tasks.ml.apply_allowlist_tags",
+                "schedule": 86400.0,
             },
             "train-heads-nightly": {
                 "task": "backend.app.tasks.ml.scheduled_train_heads",
                 "schedule": 86400.0,  # passive cadence; manual retrain stays available
             },
-            "refresh-character-prototypes": {
-                "task": "backend.app.tasks.ml.refresh_character_prototypes",
-                "schedule": 900.0,  # ~15 min; cheap global-gate no-op when idle (#1317)
-            },
-            "reconcile-character-prototypes-nightly": {
-                "task": "backend.app.tasks.ml.refresh_character_prototypes",
-                "schedule": 86400.0,  # nightly FULL reconcile (belt-and-suspenders)
-                "args": (True,),      # full=True
-            },
             "apply-head-tags-daily": {
                 "task": "backend.app.tasks.ml.scheduled_apply_head_tags",
                 "schedule": 86400.0,  # no-op unless head_auto_apply_enabled
             },
             "recover-orphaned-gpu-jobs": {
-                "task": "backend.app.tasks.gpu_queue.recover_orphaned_gpu_jobs",
+                "task": "backend.app.tasks.ml.recover_orphaned_gpu_jobs",
                 "schedule": 60.0,  # quick pickup of work a dead agent orphaned
             },
-            "triage-gpu-errors": {
-                "task": "backend.app.tasks.maintenance.triage_gpu_errors",
-                "schedule": 900.0,  # probe errored jobs' files → defect/file_ok
-            },
-            "enqueue-ccip-backfill-hourly": {
-                "task": "backend.app.tasks.gpu_queue.enqueue_gpu_backfill",
-                "schedule": 3600.0,  # auto-feed NEW images; errored are
-                "args": ("ccip",),   # tombstoned — retry is the button only
-            },
-            "enqueue-siglip-backfill-daily": {
-                "task": "backend.app.tasks.gpu_queue.enqueue_gpu_backfill",
-                "schedule": 86400.0,  # drain the concept-crop back-catalogue
-                "args": ("siglip",),  # (errored are tombstoned, not retried)
-            },
-            "enqueue-embed-backfill-daily": {
-                "task": "backend.app.tasks.gpu_queue.enqueue_gpu_backfill",
-                "schedule": 86400.0,  # whole-image re-embed under the current
-                "args": ("embed",),   # model (an operator swap) drains via agent
-            },
-            "ccip-auto-apply-daily": {
-                "task": "backend.app.tasks.ml.scheduled_ccip_auto_apply",
-                "schedule": 86400.0,  # no-op unless ccip_auto_apply_enabled
-            },
-            "retract-auto-tags-daily": {
-                "task": "backend.app.tasks.ml.scheduled_retract_auto_tags",
-                "schedule": 86400.0,  # soft auto-apply: drop auto-tags now below
-                # their threshold (m139); no-op unless the auto-apply switch is on
-            },
-            "presentation-auto-apply-daily": {
-                "task": "backend.app.tasks.ml.scheduled_presentation_auto_apply",
-                "schedule": 86400.0,  # auto-hide banner chrome (#141);
-                # no-op unless presentation_auto_apply_enabled
-            },
-            "process-auto-apply-daily": {
-                "task": "backend.app.tasks.ml.scheduled_process_auto_apply",
-                "schedule": 86400.0,  # auto-tag wip/editor process art (#1464);
-                # no-op unless process_auto_apply_enabled (opt-in)
-            },
-            "soft-wip-conflict-audit-daily": {
-                "task": "backend.app.tasks.ml.scheduled_soft_wip_conflict_audit",
-                "schedule": 86400.0,  # flag ring-loud soft-WIP (sketch/doodle) tags
-                # for review (#1474); no-op with no content heads
-            },
-            "prune-presentation-reviews-daily": {
-                "task": "backend.app.tasks.ml.prune_presentation_reviews",
-                "schedule": 86400.0,  # retention: drop resolved review flags >30d
-            },
-            "translate-posts-8h": {
-                "task": "backend.app.tasks.translation.translate_posts",
-                "schedule": 28800.0,  # every 8h: steady-state cadence for the
-                # trickle of newly-imported posts (no-op unless translation
-                # configured + healthy). One bounded 300-chunk per fire — the
-                # one-time backlog drains via the "Translate now" button
-                # (drain=True, run-until-done), not this sweep.
-            },
             "snapshot-head-metrics-daily": {
                 "task": "backend.app.tasks.maintenance.snapshot_head_metrics",
                 "schedule": 86400.0,
@@ -247,6 +172,10 @@ def make_celery() -> Celery:
                 "task": "backend.app.tasks.maintenance.recover_stalled_library_audit_runs",
                 "schedule": 300.0,
             },
+            "recover-stalled-tag-eval-runs": {
+                "task": "backend.app.tasks.maintenance.recover_stalled_tag_eval_runs",
+                "schedule": 300.0,
+            },
             "recover-stalled-head-training-runs": {
                 "task": "backend.app.tasks.maintenance.recover_stalled_head_training_runs",
                 "schedule": 300.0,

backend/app/models/__init__.py

@@ -5,7 +5,6 @@ from .artist import Artist
 from .artist_visit import ArtistVisit
 from .backup_run import BackupRun
 from .base import Base
-from .character_prototype import CcipPrototypeState, CharacterPrototype
 from .credential import Credential
 from .download_event import DownloadEvent
 from .external_link import ExternalLink
@@ -14,6 +13,7 @@ from .head_auto_apply_run import HeadAutoApplyRun
 from .head_metric import HeadMetric
 from .head_metrics_snapshot import HeadMetricsSnapshot
 from .head_training_run import HeadTrainingRun
+from .image_prediction import ImagePrediction
 from .image_provenance import ImageProvenance
 from .image_record import ImageRecord
 from .image_region import ImageRegion
@@ -24,11 +24,8 @@ from .library_audit_run import LibraryAuditRun
 from .ml_settings import MLSettings
 from .patreon_failed_media import PatreonFailedMedia
 from .patreon_seen_media import PatreonSeenMedia
-from .pixiv_failed_media import PixivFailedMedia
-from .pixiv_seen_media import PixivSeenMedia
 from .post import Post
 from .post_attachment import PostAttachment
-from .presentation_review import PresentationReview
 from .series_chapter import SeriesChapter
 from .series_page import SeriesPage
 from .series_suggestion import SeriesSuggestion
@@ -37,8 +34,11 @@ from .subscribestar_failed_media import SubscribeStarFailedMedia
 from .subscribestar_seen_media import SubscribeStarSeenMedia
 from .tag import Tag, TagKind, image_tag
 from .tag_alias import TagAlias
+from .tag_allowlist import TagAllowlist
+from .tag_eval_run import TagEvalRun
 from .tag_head import TagHead
 from .tag_positive_confirmation import TagPositiveConfirmation
+from .tag_reference_embedding import TagReferenceEmbedding
 from .tag_suggestion_rejection import TagSuggestionRejection
 from .task_run import TaskRun
 
@@ -52,17 +52,15 @@ __all__ = [
     "Credential",
     "PatreonFailedMedia",
     "PatreonSeenMedia",
-    "PixivFailedMedia",
-    "PixivSeenMedia",
     "SubscribeStarFailedMedia",
     "SubscribeStarSeenMedia",
     "Post",
     "PostAttachment",
-    "PresentationReview",
     "SeriesChapter",
     "SeriesPage",
     "SeriesSuggestion",
     "ImageRecord",
+    "ImagePrediction",
     "ImageProvenance",
     "ImageRegion",
     "Tag",
@@ -81,10 +79,11 @@ __all__ = [
     "HeadMetricsSnapshot",
     "HeadTrainingRun",
     "TagAlias",
+    "TagAllowlist",
+    "TagEvalRun",
     "TagHead",
-    "CharacterPrototype",
-    "CcipPrototypeState",
     "TagPositiveConfirmation",
+    "TagReferenceEmbedding",
     "TagSuggestionRejection",
     "TaskRun",
 ]

backend/app/models/artist.py

@@ -15,14 +15,7 @@ class Artist(Base):
     __tablename__ = "artist"
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    # Display name: freely editable, NON-unique (two real creators can share a
-    # name). Decoupled from identity/storage in migration 0077 (#130) — renaming
-    # touches ONLY this. Was unique until then.
-    name: Mapped[str] = mapped_column(String(255), nullable=False)
-    # Storage/identity key: IMMUTABLE + unique. This is the on-disk path
-    # component (download_service artist_slug = artist.slug → images_root/<slug>/
-    # <platform>/…), so it is set once at creation (collision-suffixed) and NEVER
-    # changes — a rename must not move files. Existing artists keep their slug.
+    name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
     slug: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
     notes: Mapped[str | None] = mapped_column(Text, nullable=True)
 

backend/app/models/character_prototype.py

@@ -1,62 +0,0 @@
-"""Precomputed CCIP character prototypes (#1317, milestone 138).
-
-The live matcher (ccip.match_image) needs each character's reference figure
-vectors. Building that on the request path reloaded EVERY figure CCIP vector in
-the library on any change (~4s, invalidated by every character accept). These
-tables make the references a PRECOMPUTED, INCREMENTAL artifact refreshed off the
-request path (services.ml.character_prototypes):
-
-- CharacterPrototype: a character's reference vectors, capped to
-  MLSettings.ccip_prototype_cap so MATCH cost doesn't grow with a character's
-  popularity. The async matcher only READS these.
-- CcipPrototypeState: a per-character fingerprint (reference count + max region
-  id) so a refresh rebuilds ONLY the characters whose references changed, and
-  its updated_at lets the matcher's cache reload just the advanced characters.
-"""
-
-from datetime import datetime
-
-from pgvector.sqlalchemy import Vector
-from sqlalchemy import DateTime, ForeignKey, Integer, String, func
-from sqlalchemy.orm import Mapped, mapped_column
-
-from .base import Base
-from .image_region import CCIP_DIM
-
-
-class CharacterPrototype(Base):
-    __tablename__ = "character_prototype"
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    # The character tag these vectors identify. CASCADE: deleting the tag drops
-    # its prototypes.
-    tag_id: Mapped[int] = mapped_column(
-        ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
-    )
-    # A reference figure/face CCIP vector (same space as
-    # ImageRegion.ccip_embedding).
-    ccip_embedding: Mapped[list[float]] = mapped_column(
-        Vector(CCIP_DIM), nullable=False
-    )
-    # Provenance: the region this vector was copied from. SET NULL so pruning a
-    # region doesn't delete the prototype mid-cycle (the next refresh reconciles).
-    region_id: Mapped[int | None] = mapped_column(
-        ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True
-    )
-
-
-class CcipPrototypeState(Base):
-    __tablename__ = "ccip_prototype_state"
-
-    # One row per character that currently has prototypes.
-    tag_id: Mapped[int] = mapped_column(
-        ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
-    )
-    # count(reference regions) + max(region id) at last build — the cheap
-    # per-character change detector that drives incremental rebuilds.
-    fingerprint: Mapped[str] = mapped_column(String(64), nullable=False)
-    # Bumped when this character's prototypes are rebuilt; the matcher cache
-    # reloads only characters whose updated_at advanced.
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )

backend/app/models/gpu_job.py

@@ -14,16 +14,7 @@ pending for another agent).
 
 from datetime import datetime
 
-from sqlalchemy import (
-    DateTime,
-    ForeignKey,
-    Index,
-    Integer,
-    String,
-    Text,
-    func,
-    text,
-)
+from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, func
 from sqlalchemy.orm import Mapped, mapped_column
 
 from .base import Base
@@ -32,17 +23,6 @@ from .base import Base
 class GpuJob(Base):
     __tablename__ = "gpu_job"
 
-    # Partial indexes over just the live slice (see migration 0070): the lease
-    # reads the lowest-id pending jobs on the hot path, and reclaims expired
-    # leases as a backstop — both stay O(batch) as done/error history grows.
-    __table_args__ = (
-        Index("ix_gpu_job_pending", "id", postgresql_where=text("status = 'pending'")),
-        Index(
-            "ix_gpu_job_leased_expires", "lease_expires_at",
-            postgresql_where=text("status = 'leased'"),
-        ),
-    )
-
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
     image_record_id: Mapped[int] = mapped_column(
         ForeignKey("image_record.id", ondelete="CASCADE"), index=True
@@ -62,11 +42,6 @@ class GpuJob(Base):
     )
     attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     error: Mapped[str | None] = mapped_column(Text, nullable=True)
-    # Triage verdict for an ERRORED job (#125): NULL = not yet probed;
-    # 'defect' = the integrity probe says the FILE itself is bad (surfaced for
-    # recovery, excluded from /retry_errors); 'file_ok' = the file passes —
-    # the failure was operational (timeout/transient), safe to retry.
-    triage_status: Mapped[str | None] = mapped_column(String(16), nullable=True)
     created_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )

backend/app/models/head_training_run.py

@@ -1,7 +1,7 @@
 """HeadTrainingRun — persisted lifecycle of a head-training batch (#114).
 
-A persisted run row (not transient frontend state) so the run SURVIVES
-navigation and the admin card can show live + historical status.
+Mirrors TagEvalRun so the run SURVIVES navigation and the admin card can show
+live + historical status instead of holding it in transient frontend state.
 Training is idempotent (it upserts tag_head rows), so a SIGKILL'd run is harmless
 — a maintenance recovery sweep flips a stalled `running` row to `error`, and the
 next run re-trains. State machine: running → ready / error.
@@ -37,8 +37,8 @@ class HeadTrainingRun(Base):
     n_trained: Mapped[int | None] = mapped_column(Integer, nullable=True)
     n_skipped: Mapped[int | None] = mapped_column(Integer, nullable=True)
     error: Mapped[str | None] = mapped_column(Text, nullable=True)
-    # Last time the task made progress — the recovery sweep tells a live run
-    # from a SIGKILL'd one by this.
+    # Last time the task made progress — the recovery sweep tells a live run from
+    # a SIGKILL'd one by this (mirrors TagEvalRun).
     last_progress_at: Mapped[datetime | None] = mapped_column(
         DateTime(timezone=True), nullable=True
     )

backend/app/models/image_prediction.py

@@ -0,0 +1,37 @@
+"""ImagePrediction — one row per (image, tagger vocab prediction).
+
+Replaces the image_record.tagger_predictions JSON blob (#768). Storing the
+raw Camie/booru vocab name (not a tag_id) preserves the suggestion read
+path's semantics: raw_name → canonical Tag resolution happens at read time
+via the alias map, and accepting a prediction can CREATE the Tag. The store
+floor (ml_settings.tagger_store_floor) is applied at WRITE time, so only
+predictions >= the floor land here.
+"""
+
+from sqlalchemy import Float, ForeignKey, Index, String, UniqueConstraint
+from sqlalchemy.orm import Mapped, mapped_column
+
+from .base import Base
+
+
+class ImagePrediction(Base):
+    __tablename__ = "image_prediction"
+    __table_args__ = (
+        UniqueConstraint(
+            "image_record_id", "raw_name", name="image_raw_name",
+        ),
+        # Per-image read (suggestion build) and the "images with tag X above
+        # Y" query the JSON blob never allowed.
+        Index("ix_image_prediction_image", "image_record_id"),
+        Index("ix_image_prediction_name_score", "raw_name", "score"),
+    )
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    image_record_id: Mapped[int] = mapped_column(
+        ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False,
+    )
+    # The raw tagger vocab key (booru form) — NOT a tag_id. Resolved to a
+    # canonical Tag at read time, exactly as the old JSON keys were.
+    raw_name: Mapped[str] = mapped_column(String(255), nullable=False)
+    category: Mapped[str] = mapped_column(String(64), nullable=False)
+    score: Mapped[float] = mapped_column(Float, nullable=False)

backend/app/models/image_record.py

@@ -9,6 +9,7 @@ from datetime import datetime
 
 from pgvector.sqlalchemy import Vector
 from sqlalchemy import (
+    JSON,
     BigInteger,
     DateTime,
     Enum,
@@ -76,13 +77,19 @@ class ImageRecord(Base):
         ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
     )
 
-    # ML fields (populated by the ml-worker / GPU agent). 1152 = SigLIP-so400m
-    # embedding dim; siglip_model_version stamps which model produced it (so an
-    # operator model swap, #1190, can re-embed the stale rows). A different-dim
-    # model would need a column-width migration.
+    # ML fields (populated by FC-2's ml-worker). Per-tag predictions live in the
+    # normalized image_prediction table (#768) — the tagger_predictions JSON
+    # column was dropped in migration 0046. tagger_model_version stays as the
+    # "has this been tagged / is it current?" signal the backfill sweep reads.
+    tagger_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True)
+    # 1152 = SigLIP-so400m embedding dim. Swapping models in FC-2 may require
+    # a column-width migration.
     siglip_embedding: Mapped[list[float] | None] = mapped_column(Vector(1152), nullable=True)
     siglip_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True)
 
+    # Centroid score cache (populated post-tagging)
+    centroid_scores: Mapped[dict | None] = mapped_column(JSON, nullable=True)
+
     created_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )
@@ -97,16 +104,6 @@ class ImageRecord(Base):
     effective_date: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )
-    # Denormalized ORIGINAL-publish sort key (alembic 0071) = MIN(post_date)
-    # across ALL of the image's provenance posts, else created_at. effective_date
-    # above keys off the PRIMARY post (often the repost/download the file came
-    # from); this keys off the earliest publish across EVERY post the image
-    # appears in, so the gallery can sort by when content was first posted rather
-    # than when it was downloaded (operator-flagged 2026-07-01). Maintained by
-    # services/importer.py, recomputed whenever a dated post is linked.
-    earliest_post_date: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
     updated_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True),
         nullable=False,

backend/app/models/image_region.py

@@ -31,10 +31,7 @@ class ImageRegion(Base):
         ForeignKey("image_record.id", ondelete="CASCADE"), index=True
     )
     # 'frame' (a whole video frame → SigLIP bag) | 'face' | 'figure' (→ CCIP
-    # character id) | 'concept' (→ SigLIP head bag) | 'panel' (a comic panel crop,
-    # also SigLIP → the bag). Free String, not an enum — proposers can add kinds
-    # without a migration; the bag scorer keys on a non-null siglip_embedding, not
-    # the kind, so any SigLIP-embedded region joins the bag.
+    # character id) | 'concept' (→ SigLIP head bag).
     kind: Mapped[str] = mapped_column(String(16), nullable=False)
     # For video/animated media: the source frame's timestamp in SECONDS. NULL for
     # static images. Lets a video be a BAG of per-frame instances (fixes the

backend/app/models/import_settings.py

@@ -92,48 +92,6 @@ class ImportSettings(Base):
         Boolean, nullable=False, default=True, server_default="true",
     )
 
-    # -- Post-text translation via the Interpreter LAN service (milestone 143).
-    # Off by default with NO default host — it needs a reachable Interpreter
-    # service (the operator's, behind a reverse proxy), which not every install
-    # has; the operator sets the URL and flips it on. Empty base_url OR disabled
-    # → the translate sweep no-ops.
-    translation_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=False, server_default="false",
-    )
-    interpreter_base_url: Mapped[str] = mapped_column(
-        Text, nullable=False, default="", server_default="",
-    )
-    translation_target_lang: Mapped[str] = mapped_column(
-        Text, nullable=False, default="en", server_default="en",
-    )
-    # The latin-script acceptance floor for the translation gate: a translation
-    # whose Interpreter-reported confidence is below this is kept as the original
-    # (operator-tunable, milestone 155). Default 0.9 — stricter than the old
-    # hardcoded 0.8, because Interpreter confidently mis-detects short ASCII
-    # English (e.g. "… WIP Part 1") as a European language at ~0.86. CJK stays
-    # trusted regardless (script-detected). Per-post overrides handle the misses.
-    translation_min_confidence: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.9, server_default="0.9",
-    )
-
-    # Title-based WIP auto-tagging (task #1458). When a freshly-imported post's
-    # TITLE explicitly declares work-in-progress ("WIP" / "work in progress"),
-    # the importer applies the `wip` system tag to its images — the artist's own
-    # label, used to keep unfinished pieces out of the Explore/gallery browse. ON
-    # by default (rule 26 — the feature works out of the box). Gates only the
-    # LIVE import hook; the existing catalogue is caught by the operator-triggered
-    # "Scan existing posts" backfill (which runs regardless of this flag).
-    wip_title_tagging_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=True, server_default="true",
-    )
-    # Soft WIP title tier (#1474): also tag sketch/doodle/scribble titles, but with
-    # a PROVISIONAL source (`wip_title_soft`) that never trains the head, since these
-    # are lower-precision (a finished "sketch" isn't WIP). OFF by default — a lower-
-    # precision tier is opt-in (the ring-loud audit surfaces false positives).
-    wip_soft_title_tagging_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=False, server_default="false",
-    )
-
     @classmethod
     async def load(cls, session) -> ImportSettings:
         """The singleton settings row (id=1), via an async session."""

backend/app/models/ml_settings.py

@@ -10,7 +10,6 @@ from sqlalchemy import (
     Integer,
     String,
     func,
-    select,
 )
 from sqlalchemy.orm import Mapped, mapped_column
 
@@ -24,25 +23,46 @@ class MLSettings(Base):
     __table_args__ = (CheckConstraint("id = 1", name="singleton"),)
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    # CPU whole-image embedding (B3, operator 2026-07-02). The ml-worker's ONLY
-    # processing role is the embed fallback for stacks WITHOUT a GPU agent — ON
-    # by default so a fresh install works with no agent. Stacks that run the
-    # agent and drop the ml-worker container turn this OFF so import hooks stop
-    # queueing embed work nothing will consume (the daily GPU 'embed' backfill
-    # covers those images instead).
-    cpu_embed_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=True
+    suggestion_threshold_character: Mapped[float] = mapped_column(
+        Float, nullable=False, default=0.70
     )
-    # Video embedding (#747). Sample one frame every N seconds (fixed CADENCE, not
-    # a fixed count) so coverage reflects real screen time regardless of length;
-    # cap the total so a long video can't explode into hundreds of embeds. The
-    # per-frame SigLIP embeddings are mean-pooled. Operator-tunable.
+    # Default raised 0.50 → 0.70 on 2026-06-02 — operator-flagged 0.50
+    # surfaced too many low-confidence picks; 0.70 keeps the rail
+    # signal-rich while still surfacing more than the original 0.95
+    # which hid almost everything. Operator-tunable via Settings → ML.
+    suggestion_threshold_general: Mapped[float] = mapped_column(
+        Float, nullable=False, default=0.70
+    )
+    centroid_similarity_threshold: Mapped[float] = mapped_column(
+        Float, nullable=False, default=0.55
+    )
+    # Ingest floor: tagger predictions below this confidence are not stored
+    # (tagger.Tagger.infer). Default 0.70 — the suggestion path already
+    # filters at 0.70 and the centroid/learned path covers low-confidence
+    # preferred tags, so the sub-0.70 tail is redundant weight (it had
+    # bloated image_record's TOAST to ~100 GB; plan-task #764). Operator-
+    # tunable via Settings → ML; must stay ≤ the suggestion thresholds.
+    tagger_store_floor: Mapped[float] = mapped_column(
+        Float, nullable=False, default=0.70
+    )
+    min_reference_images: Mapped[int] = mapped_column(
+        Integer, nullable=False, default=5
+    )
+    # Video tagging (#747). Sample one frame every N seconds (fixed CADENCE, not a
+    # fixed count) so a tag's frame-presence reflects real screen time regardless
+    # of video length; cap the total so a long video can't explode into hundreds
+    # of inferences (the cadence stretches past the cap). A tag is kept only if it
+    # appears in >= video_min_tag_frames sampled frames (≈ that many × interval
+    # seconds on screen) — duration-independent noise rejection. Operator-tunable.
     video_frame_interval_seconds: Mapped[float] = mapped_column(
         Float, nullable=False, default=4.0
     )
     video_max_frames: Mapped[int] = mapped_column(
         Integer, nullable=False, default=64
     )
+    video_min_tag_frames: Mapped[int] = mapped_column(
+        Integer, nullable=False, default=3
+    )
     # Tagging-v2 head training (#114). The head is the suggestion source that
     # LEARNS from the operator's tags (replacing Camie + centroid). A concept
     # needs >= head_min_positives labelled images before a head is trained;
@@ -64,163 +84,14 @@ class MLSettings(Base):
         Boolean, nullable=False, default=True
     )
     head_auto_apply_min_positives: Mapped[int] = mapped_column(
-        # Support floor raised 30→50 (operator-asked 2026-07-06): a head needs
-        # more human labels before it may fire without a human.
-        Integer, nullable=False, default=50
+        Integer, nullable=False, default=30
     )
-    # CCIP character-match cosine cut (#114). 0.85 default — the v1 flat 0.75
-    # over-fired (high-reference characters matched a scatter of images); 0.85
-    # keeps the confident single-character matches. Tunable from the agent card.
-    ccip_match_threshold: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.85
+    tagger_model_version: Mapped[str] = mapped_column(
+        String(128), nullable=False, default="camie-tagger-v2"
     )
-    # CCIP auto-apply (#114). Confident matches (>= ccip_auto_apply_threshold,
-    # above the suggest cut) auto-tag on a daily sweep. ON by default (opt-out);
-    # single-character references + the high bar keep it safe, every tag reversible.
-    ccip_auto_apply_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=True
-    )
-    ccip_auto_apply_threshold: Mapped[float] = mapped_column(
-        # Raised 0.92→0.95 (operator-asked 2026-07-06) so only very confident
-        # character matches auto-tag.
-        Float, nullable=False, default=0.95
-    )
-    # -- Presentation chrome auto-hide (#141) -------------------------------
-    # `banner` (chrome — clusters on UI, not content) auto-applies on the sweep
-    # with its OWN flat threshold (decoupled from content-head graduation) and is
-    # HIDDEN from the gallery. Hiding is consequential so it runs HIGH. When an
-    # image would be auto-hidden but ALSO scores >= presentation_conflict_threshold
-    # on a content head, it's still hidden but flagged for review
-    # (PresentationReview, mode='chrome') instead of buried silently. ON by default
-    # (opt-out); every auto-tag is reversible. NOTE (#1464): `wip` + `editor
-    # screenshot` are no longer chrome — they went to the PROCESS path below.
-    presentation_auto_apply_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=True
-    )
-    presentation_auto_apply_threshold: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.90
-    )
-    presentation_conflict_threshold: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.50
-    )
-    # -- Process auto-apply (#1464) ----------------------------------------
-    # `wip` / `editor screenshot` are PROCESS art — unfinished pieces + program
-    # screenshots that must stay OUT of head/CCIP training but, unlike chrome,
-    # remain VISIBLE in the gallery (operator 2026-07-12). They auto-apply on the
-    # sweep with their OWN flat threshold and a PROVISIONAL source (`process_auto`,
-    # in training_data._AUTO_SOURCES) so the head NEVER trains on its own output —
-    # it learns only from title (`wip_title`) + manual labels, which breaks the
-    # runaway loop. When a process tag would be applied but the image ALSO scores
-    # >= process_conflict_threshold on a content head, it's flagged for review
-    # (PresentationReview, mode='process') rather than silently marked. OFF by
-    # default — a new whole-library auto-tagger is opt-in; every auto-tag reversible.
-    process_auto_apply_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=False
-    )
-    process_auto_apply_threshold: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.90
-    )
-    process_conflict_threshold: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.50
-    )
-    # Default = SigLIP 2 (so400m, 512px) for new installs (migration 0069);
-    # existing libraries keep their stored value until the operator re-embeds.
     embedder_model_version: Mapped[str] = mapped_column(
-        String(128), nullable=False, default="siglip2-so400m-patch16-512"
-    )
-    # The HF model NAME the embedder loads (server CPU embed + announced to the
-    # GPU agent in the lease). Operator-settable so the embedder is a choice, not
-    # a hardcode (#1190): set name + version together, then re-embed + retrain.
-    embedder_model_name: Mapped[str] = mapped_column(
-        String(128), nullable=False, default="google/siglip2-so400m-patch16-512"
-    )
-    # -- Crop proposers / detectors (#1202, #134) --------------------------
-    # WHERE-to-crop YOLO detectors feeding the crop→SigLIP bag + CCIP. Config
-    # lives HERE (DB) and is announced to the GPU agent in the lease — same as
-    # the embedder model — so it is UI-tunable with NO restart, and the agent's
-    # env is bootstrap-only. Each weights spec is an ultralytics builtin name,
-    # an http(s) URL, or "hf_repo::file" (agent's _resolve). enabled off (or an
-    # empty weights) skips that proposer. All ON by default (operator 2026-07-05)
-    # so a fresh install crops out-of-the-box.
-    # person: general COCO figure detector for Western/realistic art the anime
-    # person-detector misses → NMS-merged with imgutils → CCIP + concept.
-    detector_person_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=True
-    )
-    detector_person_weights: Mapped[str] = mapped_column(
-        String(512), nullable=False, default="yolo11n.pt"
-    )
-    detector_person_conf: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.35
-    )
-    # anatomy: booru_yolo anime/furry/NSFW torso components → concept crops.
-    # Default = yolov11m_aa22 (26 classes, best mAP50-95 0.96), committed in the
-    # upstream repo so the URL resolves. License UNSTATED — fine for a private
-    # homelab (operator accepted #1202).
-    detector_anatomy_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=True
-    )
-    detector_anatomy_weights: Mapped[str] = mapped_column(
-        String(512), nullable=False,
-        default=(
-            "https://github.com/aperveyev/booru_yolo/raw/main/models/"
-            "yolov11m_aa22.pt"
-        ),
-    )
-    detector_anatomy_conf: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.30
-    )
-    # panel: comic page → panel regions → concept crops (Apache-2.0, YOLOv12x).
-    detector_panel_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=True
-    )
-    detector_panel_weights: Mapped[str] = mapped_column(
-        String(512), nullable=False,
-        default="mosesb/best-comic-panel-detection::best.pt",
-    )
-    detector_panel_conf: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.30
-    )
-    # Per-frame caps bound the crop→embed explosion; max_regions is the hard
-    # per-job backstop; dedupe_iou drops near-duplicate crops before the embed.
-    detector_max_figures: Mapped[int] = mapped_column(
-        Integer, nullable=False, default=8
-    )
-    detector_max_components: Mapped[int] = mapped_column(
-        Integer, nullable=False, default=8
-    )
-    detector_max_panels: Mapped[int] = mapped_column(
-        Integer, nullable=False, default=8
-    )
-    detector_max_regions: Mapped[int] = mapped_column(
-        Integer, nullable=False, default=128
-    )
-    detector_dedupe_iou: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.85
-    )
-    # -- CCIP character prototypes (#1317) ---------------------------------
-    # The per-character reference set is precomputed + refreshed INCREMENTALLY
-    # (services.ml.character_prototypes) instead of rebuilt on the request path.
-    # ccip_ref_signature is the cheap GLOBAL gate — when it's unchanged the
-    # refresh no-ops; ccip_prototype_cap bounds the reference vectors kept per
-    # character so MATCH cost doesn't grow with a character's popularity.
-    ccip_ref_signature: Mapped[str | None] = mapped_column(
-        String(128), nullable=True
-    )
-    ccip_prototype_cap: Mapped[int] = mapped_column(
-        Integer, nullable=False, default=64
+        String(128), nullable=False, default="siglip-so400m-patch14-384"
     )
     updated_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )
-
-    @classmethod
-    async def load(cls, session) -> MLSettings:
-        """The singleton settings row (id=1), via an async session. Mirrors
-        ImportSettings.load — the shared singleton-loader pattern."""
-        return (await session.execute(select(cls).where(cls.id == 1))).scalar_one()
-
-    @classmethod
-    def load_sync(cls, session) -> MLSettings:
-        """The singleton settings row (id=1), via a sync session."""
-        return session.execute(select(cls).where(cls.id == 1)).scalar_one()

backend/app/models/pixiv_failed_media.py

@@ -1,45 +0,0 @@
-"""PixivFailedMedia — per-source dead-letter ledger of Pixiv media that keeps
-failing to download/validate.
-
-Mirror of PatreonFailedMedia/SubscribeStarFailedMedia. Media that fails every
-walk (404'd pximg URL, deleted work, persistently-corrupt bytes) would
-otherwise re-error forever and re-burn backfill chunks. After ``attempts``
-reaches the dead-letter threshold the ingester skips it on routine
-tick/backfill walks (recovery still re-attempts). A later clean download
-clears the row.
-
-`filehash` is the same synthesized ``<illust_id>:p<num>`` /
-``<illust_id>:ugoira`` key the seen-ledger uses. UNIQUE (source_id, filehash)
-is the upsert key.
-"""
-
-from datetime import datetime
-
-from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func
-from sqlalchemy.orm import Mapped, mapped_column
-from sqlalchemy.types import DateTime
-
-from .base import Base
-
-
-class PixivFailedMedia(Base):
-    __tablename__ = "pixiv_failed_media"
-    __table_args__ = (
-        UniqueConstraint(
-            "source_id", "filehash", name="uq_pixiv_failed_media_source_id"
-        ),
-    )
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    source_id: Mapped[int] = mapped_column(
-        ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
-    )
-    filehash: Mapped[str] = mapped_column(String(128), nullable=False)
-    attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
-    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
-    first_failed_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    last_failed_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )

backend/app/models/pixiv_seen_media.py

@@ -1,42 +0,0 @@
-"""PixivSeenMedia — per-source ledger of Pixiv media already
-downloaded+processed.
-
-Mirror of PatreonSeenMedia/SubscribeStarSeenMedia for the Pixiv native
-ingester (replacing gallery-dl). One queryable row per (source, media) so
-routine walks skip media we've already ingested; recovery mode bypasses the
-ledger to re-walk.
-
-Pixiv original URLs carry no content hash, so `filehash` is always the
-synthesized ``<illust_id>:p<num>`` (page) / ``<illust_id>:ugoira`` (frame
-zip) key — stable across any URL-shape drift. String(128) matches the sibling
-ledgers.
-"""
-
-from datetime import datetime
-
-from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
-from sqlalchemy.orm import Mapped, mapped_column
-from sqlalchemy.types import DateTime
-
-from .base import Base
-
-
-class PixivSeenMedia(Base):
-    __tablename__ = "pixiv_seen_media"
-    __table_args__ = (
-        # Dedup key the downloader upserts against: one ledger row per
-        # (source, media). A second sighting of the same media is a no-op.
-        UniqueConstraint(
-            "source_id", "filehash", name="uq_pixiv_seen_media_source_id"
-        ),
-    )
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    source_id: Mapped[int] = mapped_column(
-        ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
-    )
-    filehash: Mapped[str] = mapped_column(String(128), nullable=False)
-    post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
-    seen_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )

backend/app/models/post.py

@@ -8,17 +8,7 @@ artist-filter queries don't depend on the Source detour).
 
 from datetime import datetime
 
-from sqlalchemy import (
-    JSON,
-    CheckConstraint,
-    DateTime,
-    ForeignKey,
-    Integer,
-    String,
-    Text,
-    UniqueConstraint,
-    func,
-)
+from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
 from sqlalchemy.orm import Mapped, mapped_column
 
 from .base import Base
@@ -33,10 +23,6 @@ class Post(Base):
         # (created in alembic 0030) covers that case via
         # (artist_id, external_post_id).
         UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
-        CheckConstraint(
-            "translation_override IN ('auto', 'force', 'original')",
-            name="ck_post_translation_override",
-        ),
     )
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
@@ -61,34 +47,6 @@ class Post(Base):
     description: Mapped[str | None] = mapped_column(Text, nullable=True)
     attachment_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
 
-    # -- Post-text translation (milestone 143). Filled by the translate_posts
-    # sweep via the Interpreter LAN service so viewing is instant.
-    # translated_source_lang is the DETECTED original language; "en" (or a
-    # passthrough) means nothing to translate and the *_translated columns stay
-    # NULL. engine_version keys the Interpreter cache — re-runs are ~1ms and a
-    # model upgrade re-translates instead of serving stale.
-    post_title_translated: Mapped[str | None] = mapped_column(Text, nullable=True)
-    description_translated: Mapped[str | None] = mapped_column(Text, nullable=True)
-    translated_source_lang: Mapped[str | None] = mapped_column(
-        String(8), nullable=True
-    )
-    translation_engine_version: Mapped[str | None] = mapped_column(
-        String(128), nullable=True
-    )
-    translated_at: Mapped[datetime | None] = mapped_column(
-        DateTime(timezone=True), nullable=True
-    )
-    # Sticky per-post override of the translation decision (milestone 155):
-    # 'auto' = the acceptance gate decides; 'force' = always store Interpreter's
-    # translation even below the confidence floor (rescue a skipped legit-foreign
-    # title); 'original' = never translate, keep the original (kill a confidently
-    # mis-flagged one the floor can't catch). The sweep reads this on every run,
-    # and re-translate leaves 'original' posts alone, so the choice survives a
-    # Re-translate-all.
-    translation_override: Mapped[str] = mapped_column(
-        String(16), nullable=False, default="auto", server_default="auto",
-    )
-
     downloaded_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )

backend/app/models/presentation_review.py

@@ -1,48 +0,0 @@
-"""PresentationReview — a system-tag the auto-apply sweep applied that ALSO looked
-like real content, flagged for operator review (milestone 141 + #1464).
-
-When a sweep applies a system tag but the image ALSO scores highly on a content
-head, it still applies the tag but records this row so a review strip can surface
-it ("⚠ also looks like <conflict tag>"). Two modes (#1464): 'chrome' (banner —
-image is HIDDEN, review is keep-hidden / un-hide) and 'process' (wip / editor
-screenshot — image stays VISIBLE, review is confirm / remove-tag). Resolved rows
-are pruned by retention.
-"""
-
-from datetime import datetime
-
-from sqlalchemy import DateTime, Float, ForeignKey, String, func
-from sqlalchemy.orm import Mapped, mapped_column
-
-from .base import Base
-
-
-class PresentationReview(Base):
-    __tablename__ = "presentation_review"
-
-    image_record_id: Mapped[int] = mapped_column(
-        ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True
-    )
-    # The presentation tag that was auto-applied (banner / editor screenshot).
-    tag_id: Mapped[int] = mapped_column(
-        ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
-    )
-    # The content tag the image ALSO scored high on — the "concerning" signal.
-    # SET NULL (not CASCADE): losing the conflict tag shouldn't erase the flag.
-    conflict_tag_id: Mapped[int | None] = mapped_column(
-        ForeignKey("tag.id", ondelete="SET NULL"), nullable=True
-    )
-    conflict_score: Mapped[float] = mapped_column(Float, nullable=False)
-    # Which sweep flagged this (#1464): 'chrome' (banner, hidden) or 'process'
-    # (wip / editor screenshot, shown). Drives which review strip surfaces it and
-    # what "resolve" means (un-hide vs remove-tag). Existing rows backfill 'chrome'.
-    mode: Mapped[str] = mapped_column(
-        String(16), nullable=False, default="chrome", server_default="chrome"
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    # Set when the operator keeps-hidden or un-hides; retention prunes resolved.
-    resolved_at: Mapped[datetime | None] = mapped_column(
-        DateTime(timezone=True), nullable=True
-    )

backend/app/models/tag.py

@@ -10,7 +10,6 @@ from datetime import datetime
 from enum import StrEnum
 
 from sqlalchemy import (
-    Boolean,
     CheckConstraint,
     Column,
     DateTime,
@@ -18,7 +17,6 @@ from sqlalchemy import (
     Integer,
     String,
     Table,
-    false,
     func,
 )
 from sqlalchemy import (
@@ -43,21 +41,6 @@ class TagKind(StrEnum):
     # to keep historic tag rows queryable.
 
 
-# The seeded system tags (migration 0075). Two behavior groups (#1464):
-#   CHROME (banner): clusters on UI chrome, not content → HIDDEN from the default
-#     gallery + from similarity; auto-applied via the sweep's chrome mode.
-#   PROCESS (wip, editor screenshot): real-but-unfinished art / program screenshots
-#     → SHOWN in the gallery (operator 2026-07-12), but excluded from the Explore
-#     rabbit-hole; auto-applied via the sweep's process mode (provisional source,
-#     ring-loud review guard).
-# ALL three are excluded from OTHER concepts' head/CCIP training (training-hygiene,
-# keyed on is_system); a system tag's OWN head trains on them — that's what makes
-# auto-flagging work.
-SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot")
-CHROME_SYSTEM_TAGS = ("banner",)
-PROCESS_SYSTEM_TAGS = ("wip", "editor screenshot")
-WIP_SYSTEM_TAG = "wip"
-
 image_tag = Table(
     "image_tag",
     Base.metadata,
@@ -91,14 +74,6 @@ class Tag(Base):
     fandom_id: Mapped[int | None] = mapped_column(
         ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, index=True
     )
-    # System tags ship with FC (wip / banner / editor screenshot, seeded in
-    # migration 0075) and drive the training-hygiene exclusions: images
-    # carrying one are excluded from OTHER concepts' head training and from
-    # CCIP identity references. The mechanism keys on these exact rows, so
-    # they're protected from rename/merge-away/re-fandom in TagService.
-    is_system: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=False, server_default=false()
-    )
 
     created_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()

backend/app/models/tag_allowlist.py

@@ -0,0 +1,32 @@
+"""TagAllowlist — tags the operator opted-in to auto-apply via maintenance."""
+
+from datetime import datetime
+
+from sqlalchemy import CheckConstraint, DateTime, Float, ForeignKey, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from .base import Base
+
+
+class TagAllowlist(Base):
+    __tablename__ = "tag_allowlist"
+    # Bare name — Base.metadata's naming convention prepends ck_<table>_,
+    # producing the final ck_tag_allowlist_confidence_range (matches migration 0003).
+    __table_args__ = (
+        CheckConstraint(
+            "min_confidence > 0 AND min_confidence <= 1",
+            name="confidence_range",
+        ),
+    )
+
+    tag_id: Mapped[int] = mapped_column(
+        ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
+    )
+    # Default auto-apply threshold for a newly-accepted tag. 0.90 (lowered from
+    # 0.95 on operator evidence 2026-06-07: 0.95 was too strict and skipped
+    # confident-enough applications). Per-tag value is still tunable in the
+    # allowlist table; existing rows keep whatever they were stored with.
+    min_confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.90)
+    added_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )

backend/app/models/tag_eval_run.py

@@ -0,0 +1,45 @@
+"""TagEvalRun — persisted lifecycle of a head-vs-centroid tagging eval (#1130).
+
+Mirrors LibraryAuditRun so the result SURVIVES navigation: the run + its full
+report live in this row, and the admin card rehydrates from it on mount instead
+of holding the report in transient frontend state. State machine:
+running → ready / error. The async ml-queue task writes `report` (JSONB) when
+done; a maintenance recovery sweep flips a stalled `running` row to `error`.
+"""
+
+from datetime import datetime
+from typing import Any
+
+from sqlalchemy import DateTime, Integer, String, Text, func
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column
+
+from .base import Base
+
+
+class TagEvalRun(Base):
+    __tablename__ = "tag_eval_run"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    # The eval parameters: {concepts: [...], curve_points: [...], neg_ratio,
+    # cv_folds, ...} — echoed back so the report is self-describing.
+    params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
+    status: Mapped[str] = mapped_column(
+        String(16), nullable=False, default="running", index=True,
+    )
+    # running | ready | error
+    started_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now(),
+    )
+    finished_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True,
+    )
+    # The full result: per-concept metrics (head vs centroid), learning-curve
+    # points, and example image ids. Null until the task finishes.
+    report: Mapped[dict[str, Any] | None] = mapped_column(JSONB, nullable=True)
+    error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    # Last time the task made progress — the recovery sweep tells a live run
+    # from a SIGKILL'd one by this (mirrors LibraryAuditRun).
+    last_progress_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True,
+    )

backend/app/models/tag_head.py

@@ -73,12 +73,5 @@ class TagHead(Base):
     trained_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )
-    # Training-data fingerprint (positives + rejections) at last fit — the
-    # incremental-retrain change detector (#1317 p2). A manual Retrain refits only
-    # heads whose fingerprint moved; the nightly run ignores it (full reconcile).
-    # NULL forces a refit (pre-fingerprint heads).
-    train_fingerprint: Mapped[str | None] = mapped_column(
-        String(128), nullable=True
-    )
     # Extra detail (auto-apply operating point, F1, etc.) — non-load-bearing.
     metrics: Mapped[dict[str, Any] | None] = mapped_column(JSONB, nullable=True)

backend/app/models/tag_reference_embedding.py

@@ -0,0 +1,23 @@
+"""TagReferenceEmbedding — per-tag centroid (mean SigLIP embedding of members)."""
+
+from datetime import datetime
+
+from pgvector.sqlalchemy import Vector
+from sqlalchemy import DateTime, ForeignKey, Integer, String, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from .base import Base
+
+
+class TagReferenceEmbedding(Base):
+    __tablename__ = "tag_reference_embedding"
+
+    tag_id: Mapped[int] = mapped_column(
+        ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
+    )
+    embedding: Mapped[list[float]] = mapped_column(Vector(1152), nullable=False)
+    reference_count: Mapped[int] = mapped_column(Integer, nullable=False)
+    model_version: Mapped[str] = mapped_column(String(128), nullable=False)
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )

backend/app/scripts/download_models.py

@@ -7,6 +7,7 @@ import sys
 from pathlib import Path
 
 MODEL_ROOT = Path(os.environ.get("ML_MODEL_DIR", "/models"))
+CAMIE_REPO = os.environ.get("CAMIE_HF_REPO", "Camais03/camie-tagger-v2")
 SIGLIP_REPO = os.environ.get(
     "SIGLIP_HF_REPO", "google/siglip-so400m-patch14-384"
 )
@@ -23,6 +24,34 @@ def _snapshot(repo_id: str, dest: Path, allow_patterns: list[str] | None) -> Non
     )
 
 
+def ensure_camie() -> None:
+    """Fetch Camie v2 weights + metadata.
+
+    v2 layout (HuggingFace Camais03/camie-tagger-v2): the ONNX file is
+    named camie-tagger-v2.onnx (not model.onnx) and tags ship inside
+    camie-tagger-v2-metadata.json (not selected_tags.csv). Both at root.
+    The repo also contains app/, game/, training/, images/ subdirs full
+    of setup/demo files we don't need — allow_patterns scopes the fetch
+    to just the inference essentials (~790 MB instead of ~2 GB).
+    """
+    dest = MODEL_ROOT / "camie"
+    model_file = dest / "camie-tagger-v2.onnx"
+    meta_file = dest / "camie-tagger-v2-metadata.json"
+    if model_file.is_file() and meta_file.is_file():
+        print(f"[download_models] Camie present at {dest}")
+        return
+    print(f"[download_models] Fetching {CAMIE_REPO} -> {dest}")
+    _snapshot(
+        CAMIE_REPO, dest,
+        [
+            "camie-tagger-v2.onnx",
+            "camie-tagger-v2-metadata.json",
+            "config.json",
+            "config.yaml",
+        ],
+    )
+
+
 def ensure_siglip() -> None:
     dest = MODEL_ROOT / "siglip"
     if (dest / "config.json").is_file() and any(dest.glob("*.safetensors")):
@@ -33,6 +62,7 @@ def ensure_siglip() -> None:
 
 
 def main() -> int:
+    ensure_camie()
     ensure_siglip()
     print("[download_models] Done.")
     return 0

backend/app/services/artist_service.py

@@ -283,23 +283,6 @@ class ArtistService:
             await self.session.commit()
         return artist, created
 
-    async def rename(self, artist_id: int, name: str) -> Artist | None:
-        """Change the display NAME only (#130). The slug — and every on-disk path
-        keyed off it — is IMMUTABLE, so a rename never moves files or risks a path
-        collision. Name is free text and NON-unique (migration 0077). Returns the
-        updated Artist, None if not found; raises ValueError on empty name."""
-        cleaned = (name or "").strip()
-        if not cleaned:
-            raise ValueError("artist name must not be empty")
-        artist = (await self.session.execute(
-            select(Artist).where(Artist.id == artist_id)
-        )).scalar_one_or_none()
-        if artist is None:
-            return None
-        artist.name = cleaned
-        await self.session.commit()
-        return artist
-
     async def autocomplete(self, prefix: str, limit: int = 20) -> list[Artist]:
         cleaned = (prefix or "").strip()
         if not cleaned:

backend/app/services/audits/single_color.py

@@ -1,9 +1,8 @@
 """Single-color audit: matches images where one color dominates beyond
-the threshold (within the given Euclidean RGB tolerance). The canonical
-predicate for BOTH surfaces: FC-Cleanup's retroactive audit and — since
-2026-07-02 — the import-side filter (Importer._single_color_hit /
-SkipReason.single_color), so what the audit flags and what the import
-skips can never disagree.
+the threshold (within the given Euclidean RGB tolerance). The first
+canonical implementation — the import-side filter (SkipReason.single_color)
+was never wired; FC-Cleanup's audit module is the source of truth and a
+future spec can adopt it on the import path too.
 """
 
 from PIL import Image

backend/app/services/cleanup_service.py

@@ -17,7 +17,7 @@ from datetime import UTC, datetime, timedelta
 from pathlib import Path
 from typing import Any
 
-from sqlalchemy import and_, delete, func, or_, select, update
+from sqlalchemy import delete, func, or_, select, update
 from sqlalchemy.orm import Session, aliased
 
 from ..models import (
@@ -203,9 +203,6 @@ def _unused_tag_conditions() -> list:
         Tag.id.not_in(used_via_series),
         Tag.id.not_in(used_via_chapter),
         Tag.id.not_in(used_via_fandom),
-        # System tags (#128) ship with zero applications and must survive a
-        # prune — the training-hygiene machinery keys on the rows.
-        Tag.is_system.is_(False),
     ]
 
 
@@ -398,15 +395,14 @@ def delete_images(
 def delete_tag(session: Session, *, tag_id: int) -> dict:
     """Simple DELETE FROM tag WHERE id=?.
 
-    Postgres cascades the rest (image_tag, tag_alias, tag_suggestion_rejection,
-    series_page). Returns counts BEFORE delete so the caller can surface them.
+    Postgres cascades the rest (image_tag, tag_alias, tag_allowlist,
+    tag_reference_embedding, tag_suggestion_rejection, series_page).
+    Returns counts BEFORE delete so the caller can surface them.
     Raises LookupError if tag_id not found.
     """
     tag = session.get(Tag, tag_id)
     if tag is None:
         raise LookupError(f"tag id not found: {tag_id}")
-    if tag.is_system:
-        raise ValueError(f"'{tag.name}' is a system tag and cannot be deleted")
     associations_count = count_tag_associations(session, tag_id=tag_id)
     info = {"id": tag.id, "name": tag.name, "kind": tag.kind.value}
     session.delete(tag)
@@ -723,27 +719,89 @@ def reconcile_duplicate_posts(
     return {"groups": len(groups), "merged": losers_total, "sample": sample}
 
 
-# The CONTENT vocabulary. "Reset content tagging" wipes these so the operator
-# can re-tag from scratch. fandom + series (and series_page ordering) are
-# deliberately NOT here — they're kept.
+# Legacy tags FC no longer uses, in two shapes:
+#   (1) kinds the tag input never produces — archive/post/artist.
+#       provenance (post grouping) + archive membership are their own
+#       systems now, and artists are first-class Artist/Source rows.
+#       meta/rating were already hard-deleted by alembic 0023.
+#   (2) name prefixes from IR kinds FC never adopted — `source:*`.
+#       ImageRepo had a `source` kind; FC's enum doesn't, so ir_ingest
+#       fell those back to `general` (kind=general, name="source:patreon"
+#       etc.). They can't be caught by kind, so we match the name prefix.
+PURGEABLE_TAG_KINDS = ("archive", "post", "artist")
+LEGACY_NAME_PREFIXES = ("source:",)
+
+
+def _legacy_tag_predicate():
+    name_clauses = [Tag.name.like(f"{p}%") for p in LEGACY_NAME_PREFIXES]
+    return or_(Tag.kind.in_(PURGEABLE_TAG_KINDS), *name_clauses)
+
+
+def purge_legacy_tags(session: Session, *, dry_run: bool = False) -> dict:
+    """Count (dry_run) or delete legacy IR-migration tags: archive/post/
+    artist-kind tags PLUS general tags whose name matches a legacy
+    prefix (source:*).
+
+    CASCADE on image_tag / tag_alias / tag_allowlist /
+    tag_reference_embedding / tag_suggestion_rejection / series_page
+    clears the related rows on the parent DELETE.
+
+    Returns:
+      {"by_kind": {kind: count, ...},        # kind-matched rows
+       "by_prefix": {"source:*": count},     # name-prefix-matched rows
+       "count": total, "sample_names": [first 50],
+       and on live runs "deleted": total}
+    """
+    predicate = _legacy_tag_predicate()
+    rows = session.execute(
+        select(Tag.id, Tag.name, Tag.kind).where(predicate)
+    ).all()
+    by_kind: dict[str, int] = {}
+    by_prefix: dict[str, int] = {}
+    for _id, name, kind in rows:
+        # Classify by name-prefix first so a source:* row counts once,
+        # under the prefix bucket, regardless of its (general) kind.
+        matched_prefix = next(
+            (p for p in LEGACY_NAME_PREFIXES if name.startswith(p)), None,
+        )
+        if matched_prefix is not None:
+            label = f"{matched_prefix}*"
+            by_prefix[label] = by_prefix.get(label, 0) + 1
+        else:
+            key = kind.value if hasattr(kind, "value") else str(kind)
+            by_kind[key] = by_kind.get(key, 0) + 1
+    sample = [name for _id, name, _kind in rows[:50]]
+    total = len(rows)
+    result = {
+        "by_kind": by_kind, "by_prefix": by_prefix,
+        "count": total, "sample_names": sample,
+    }
+    if dry_run:
+        return result
+    if total:
+        session.execute(Tag.__table__.delete().where(predicate))
+        session.commit()
+    result["deleted"] = total
+    return result
+
+
+# The Camie-suggestable CONTENT vocabulary. "Reset content tagging" wipes
+# these so the operator can re-tag from scratch via auto-suggest. fandom +
+# series (and series_page ordering) are deliberately NOT here — they're kept.
 RESETTABLE_TAG_KINDS = ("general", "character")
 
 
 def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict:
     """Count (dry_run) or DELETE every general + character tag so the operator
-    can re-tag from scratch. NB: the deleted applications include the tagging
-    heads' training positives — suggestions do NOT repopulate on their own; the
-    heads retrain from whatever the operator re-tags. (The API route gates the
-    live run behind a preview-derived confirm token for exactly this reason.)
+    can re-tag from scratch via the Camie auto-suggest.
 
-    PRESERVED: fandom + series tags and their series_page ordering, AND the
-    system hygiene tags (#128) WITH their applications — the reset re-tags
-    CONTENT concepts, while wip/banner flags describe the file itself and
-    re-flagging hundreds of banners by hand would be pure loss. CASCADE on
-    image_tag / tag_alias / tag_suggestion_rejection clears each deleted tag's
-    applications + metadata. Tag.fandom_id is SET NULL, so deleting character
-    tags never touches the fandom rows. Irreversible except via DB backup
-    restore.
+    PRESERVED: fandom + series tags and their series_page ordering, plus every
+    image's image_prediction rows (untouched) so suggestions
+    repopulate immediately. CASCADE on image_tag / tag_alias / tag_allowlist /
+    tag_reference_embedding / tag_suggestion_rejection clears each deleted
+    tag's applications + metadata. Tag.fandom_id is SET NULL, so deleting
+    character tags never touches the fandom rows. Irreversible except via DB
+    backup restore.
 
     Returns:
       {"by_kind": {"general": N, "character": M},
@@ -752,9 +810,7 @@ def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict:
        "sample_names": [first 50],
        and on live runs "deleted": total}
     """
-    predicate = and_(
-        Tag.kind.in_(RESETTABLE_TAG_KINDS), Tag.is_system.is_(False)
-    )
+    predicate = Tag.kind.in_(RESETTABLE_TAG_KINDS)
     rows = session.execute(
         select(Tag.id, Tag.name, Tag.kind).where(predicate)
     ).all()
@@ -1018,7 +1074,7 @@ def reextract_archive_attachments(
     still an archive on disk, so the cursor is what guarantees forward progress.
     """
     from ..models import ImportSettings, Post, PostAttachment, Source
-    from ..tasks.ml import cpu_embed_enabled, embed_image
+    from ..tasks.ml import tag_and_embed
     from ..tasks.thumbnail import generate_thumbnail
     from .archive_extractor import is_archive
     from .importer import Importer
@@ -1099,12 +1155,10 @@ def reextract_archive_attachments(
 
     # Thumbnails + ML for the newly-imported members (best-effort; off the
     # critical path — a Redis hiccup must not fail the whole re-extract).
-    do_embed = cpu_embed_enabled()
     for img_id in enqueue_ids:
         try:
             generate_thumbnail.delay(img_id)
-            if do_embed:
-                embed_image.delay(img_id)
+            tag_and_embed.delay(img_id)
         except Exception as exc:
             log.warning("re-extract enqueue failed for image %s: %s", img_id, exc)
     return summary

backend/app/services/download_backends.py

@@ -24,17 +24,15 @@ import asyncio
 from pathlib import Path
 
 from .gallery_dl import DownloadResult, ErrorType
+from .native_ingest_common import NativeIngestError
 from .patreon_ingester import PatreonIngester
 from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
-from .pixiv_client import user_id_from_url
-from .pixiv_ingester import PixivIngester
 from .subscribestar_ingester import SubscribeStarIngester
 
 # Platforms whose download + verify go through the native ingester rather than
-# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord,
-# deviantart — the latter slated for retirement, not migration) until they
-# migrate too.
-NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "pixiv"})
+# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord, pixiv,
+# deviantart) until they migrate too.
+NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})
 
 # Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure
 # messages so the operator sees the exact lookup endpoint that was hit.
@@ -48,7 +46,6 @@ def _native_ingester_cls(platform: str):
     dispatch pick up the replacement."""
     return {
         "patreon": PatreonIngester,
-        "pixiv": PixivIngester,
         "subscribestar": SubscribeStarIngester,
     }[platform]
 
@@ -101,34 +98,14 @@ async def _resolve_native_campaign_id(
     platform: str, url: str, cookies_path: str | None, overrides: dict,
 ) -> tuple[str | None, str | None]:
     """`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
-    feed id IS the creator URL; Pixiv's is the numeric user id parsed straight
-    from it (no lookup → resolved None either way). Patreon resolves the
+    feed id IS the creator URL (no lookup → resolved None). Patreon resolves the
     campaign id from the vanity URL (resolved non-None when a lookup actually ran,
     so phase 3 caches it)."""
     if platform == "subscribestar":
         return url, None
-    if platform == "pixiv":
-        return user_id_from_url(url), None
     return await resolve_campaign_id_for_source(url, cookies_path, overrides)
 
 
-def _campaign_resolution_error(platform: str, url: str) -> str:
-    """Operator-facing message for a native source whose campaign id could not
-    be resolved — names the platform's own lookup mechanism."""
-    if platform == "pixiv":
-        return (
-            f"Could not extract a pixiv user id. source_url={url!r} — expected "
-            "a URL like https://www.pixiv.net/users/<id>."
-        )
-    vanity = extract_vanity(url)
-    return (
-        f"Could not resolve Patreon campaign id. source_url={url!r}; "
-        f"vanity={vanity!r}; "
-        f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
-        "(vanity lookup failed — cookies expired or creator moved?)"
-    )
-
-
 async def _run_native_ingester(
     ctx: dict, source_config, mode: str | None, gdl, sync_session_factory,
 ) -> tuple[DownloadResult, str | None]:
@@ -146,9 +123,9 @@ async def _run_native_ingester(
         platform, ctx["url"], ctx["cookies_path"], overrides
     )
     if not campaign_id:
-        # Patreon: vanity lookup failed. Pixiv: no numeric user id in the URL.
-        # (SubscribeStar's campaign id is the URL itself — never lands here.)
+        # Only reachable for Patreon (SubscribeStar's campaign id is the URL).
         url = ctx["url"]
+        vanity = extract_vanity(url)
         return (
             DownloadResult(
                 success=False,
@@ -156,7 +133,12 @@ async def _run_native_ingester(
                 artist_slug=ctx["artist_slug"],
                 platform=platform,
                 error_type=ErrorType.NOT_FOUND,
-                error_message=_campaign_resolution_error(platform, url),
+                error_message=(
+                    f"Could not resolve Patreon campaign id. source_url={url!r}; "
+                    f"vanity={vanity!r}; "
+                    f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
+                    "(vanity lookup failed — cookies expired or creator moved?)"
+                ),
             ),
             None,
         )
@@ -179,10 +161,6 @@ async def _run_native_ingester(
         validate=gdl._validate_files,
         rate_limit=rate_limit,
         request_sleep=request_sleep,
-        # Uniform across adapters: token platforms (pixiv) authenticate with
-        # it, cookie platforms accept-and-ignore — so this construction stays
-        # platform-agnostic.
-        auth_token=ctx["auth_token"],
     )
     loop = asyncio.get_running_loop()
     dl_result = await loop.run_in_executor(
@@ -203,6 +181,54 @@ async def _run_native_ingester(
     return dl_result, resolved_campaign_id
 
 
+async def preview_source(
+    *,
+    platform: str,
+    url: str,
+    source_id: int,
+    config_overrides: dict | None,
+    cookies_path: str | None,
+    images_root: Path,
+    sync_session_factory,
+    page_limit: int = 3,
+) -> dict:
+    """Dry-run preview for a native platform (plan #708 B4): resolve the campaign
+    id, then walk a few pages counting media not already seen/dead — no download.
+
+    Returns the preview dict (total_new / posts_scanned / pages_scanned /
+    has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure.
+    Native-only — the caller gates on `uses_native_ingester`.
+    """
+    import asyncio
+
+    campaign_id, _ = await _resolve_native_campaign_id(
+        platform, url, cookies_path, config_overrides or {}
+    )
+    if not campaign_id:
+        vanity = extract_vanity(url)
+        return {
+            "error": (
+                f"Couldn't resolve the campaign id. source_url={url!r}; "
+                f"vanity={vanity!r}; lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
+                "(cookies expired, or the creator moved/renamed?)."
+            )
+        }
+    ingester = _native_ingester_cls(platform)(
+        images_root=images_root,
+        cookies_path=cookies_path,
+        session_factory=sync_session_factory,
+    )
+    loop = asyncio.get_running_loop()
+    try:
+        result = await loop.run_in_executor(
+            None,
+            lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit),
+        )
+    except NativeIngestError as exc:
+        return {"error": f"Couldn't preview: {exc}"}
+    return result
+
+
 async def verify_source_credential(
     *,
     platform: str,
@@ -219,18 +245,13 @@ async def verify_source_credential(
     this and render the result.
     """
     if uses_native_ingester(platform):
-        # Native ingester platforms verify via their own lightweight auth probe.
-        # SubscribeStar's probe takes the creator URL directly; Patreon's
-        # resolves the campaign id first; Pixiv's is one OAuth refresh (the
-        # exact call that fails when the token is bad — no feed walk).
+        # Native ingester platforms verify via their own lightweight auth probe
+        # (one authenticated feed fetch). SubscribeStar's probe takes the creator
+        # URL directly; Patreon's resolves the campaign id first.
         if platform == "subscribestar":
             from .subscribestar_ingester import verify_subscribestar_credential
 
             return await verify_subscribestar_credential(url, cookies_path, config_overrides)
-        if platform == "pixiv":
-            from .pixiv_ingester import verify_pixiv_credential
-
-            return await verify_pixiv_credential(auth_token)
         from .patreon_ingester import verify_patreon_credential
 
         return await verify_patreon_credential(url, cookies_path, config_overrides)

backend/app/services/download_service.py

@@ -326,16 +326,14 @@ class DownloadService:
                 # for hours after a download landed. Lazy import to avoid
                 # circular-import risk between this service and the
                 # tasks/* modules that import it.
-                from ..tasks.ml import cpu_embed_enabled, embed_image
+                from ..tasks.ml import tag_and_embed
                 from ..tasks.thumbnail import generate_thumbnail
-                do_embed = cpu_embed_enabled()
                 ids = list(result.member_image_ids)
                 if result.image_id is not None and result.image_id not in ids:
                     ids.append(result.image_id)
                 for img_id in ids:
                     generate_thumbnail.delay(img_id)
-                    if do_embed:
-                        embed_image.delay(img_id)
+                    tag_and_embed.delay(img_id)
             elif result.status == "attached":
                 # Non-media or extracted archive captured as PostAttachment
                 # (FC-2d-iii). The canonical copy lives in the attachments
@@ -418,8 +416,7 @@ class DownloadService:
         # the duplicate file). Empty outside recapture mode.
         relink_pairs = getattr(dl_result, "relink_source_paths", None) or []
         relinked = 0
-        post_linked = 0
-        for rel_str, rel_url, rel_post_id in relink_pairs:
+        for rel_str, rel_url in relink_pairs:
             rel_path = Path(rel_str)
             if not rel_path.exists():  # noqa: ASYNC240
                 continue
@@ -429,27 +426,13 @@ class DownloadService:
 
             if await loop.run_in_executor(None, _relink):
                 relinked += 1
-
-            # #1288: link the on-disk image to its Post. Recapture disk-skips the
-            # media (never re-imported), so a pre-existing image (e.g. one pulled
-            # under the old gallery-dl path) otherwise stays orphaned even after
-            # the post record is written. Idempotent for already-linked images.
-            def _link(p=rel_path, pid=rel_post_id):
-                return self.importer.link_existing_image_to_post(
-                    p, pid, source=source_row, artist=artist,
-                )
-
-            if await loop.run_in_executor(None, _link):
-                post_linked += 1
         if relink_pairs:
             # recapture diagnostic: how many on-disk images got their
-            # source_filehash backfilled (inline-image localization) and how many
-            # got (re)linked to their Post. < total for source_filehash is normal
-            # (files already carrying a filehash are skipped, NULL-only).
+            # source_filehash backfilled (inline-image localization). < total is
+            # normal — files already carrying a filehash are skipped (NULL-only).
             log.info(
-                "recap: relinked source_filehash on %d and linked %d/%d on-disk "
-                "image(s) to their post",
-                relinked, post_linked, len(relink_pairs),
+                "recap: relinked source_filehash on %d/%d on-disk image(s)",
+                relinked, len(relink_pairs),
             )
 
         # Kick the off-platform file-host downloader for any links this run

backend/app/services/extension_service.py

@@ -8,7 +8,6 @@ and returns a JSON-shaped dict for the API layer.
 
 from __future__ import annotations
 
-import logging
 import re
 
 from sqlalchemy import select
@@ -19,8 +18,6 @@ from ..utils.slug import slugify
 from .db_helpers import get_or_create
 from .source_service import BACKFILL_MAX_CHUNKS
 
-log = logging.getLogger(__name__)
-
 
 class UnknownPlatformError(Exception):
     """URL didn't match any platform pattern."""
@@ -35,14 +32,9 @@ class InvalidUrlError(Exception):
 # reviewers catch drift.
 _PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
     ("patreon", re.compile(
-        # Three creator URL shapes — bare (patreon.com/Atole), `c/`, and `cw/`
-        # (the "creator workspace" URL served once subscribed, see
-        # patreon_resolver._VANITY_RE). A trailing sub-path is allowed so a
-        # creator's inner page still derives the slug. Nav pages stay excluded.
         r"^https?://(?:www\.)?patreon\.com/"
-        r"(?:cw/|c/)?"
-        r"(?!(?:home|search|messages|notifications|library|settings|posts)(?:[/?#]|$))"
-        r"(?P<slug>[^/?#]+)",
+        r"(?!home$|search\b|messages\b|notifications\b|library\b|settings\b|posts\b|c/)"
+        r"(?P<slug>[^/?#]+)/?$",
         re.IGNORECASE,
     )),
     ("subscribestar", re.compile(
@@ -69,38 +61,15 @@ _PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
 
 
 class ExtensionService:
-    def __init__(self, session: AsyncSession, crypto=None) -> None:
+    def __init__(self, session: AsyncSession) -> None:
         self.session = session
-        # Optional decryptor for resolving a token-auth platform's display name
-        # (pixiv) at add-time. None → skip resolution, fall back to the handle.
-        self._crypto = crypto
 
     async def quick_add_source(self, url: str) -> dict:
         platform, raw_slug = self._derive(url)
-        # Identity by SOURCE handle (#130): an existing (platform, url) source
-        # keeps its artist on re-add — even if that artist was since renamed (its
-        # frozen slug no longer matches the current name). Only a genuinely new
-        # source resolves/creates an artist.
-        existing = (await self.session.execute(
-            select(Source).where(Source.platform == platform, Source.url == url)
-        )).scalar_one_or_none()
-        if existing is not None:
-            artist = (await self.session.execute(
-                select(Artist).where(Artist.id == existing.artist_id)
-            )).scalar_one()
-            return self._shape(existing, artist, created_source=False, created_artist=False)
-
-        # New source → name the artist properly by resolving the real display
-        # name from the platform (falls back to the URL handle).
-        name = await self._resolve_artist_name(platform, raw_slug, url)
-        artist, created_artist = await self._find_or_create_artist(name)
+        artist, created_artist = await self._find_or_create_artist(raw_slug)
         source, created_source = await self._find_or_create_source(
             artist_id=artist.id, platform=platform, url=url,
         )
-        return self._shape(source, artist, created_source, created_artist)
-
-    @staticmethod
-    def _shape(source, artist, created_source: bool, created_artist: bool) -> dict:
         return {
             "source": {
                 "id": source.id,
@@ -118,50 +87,6 @@ class ExtensionService:
             "created_artist": created_artist,
         }
 
-    async def _resolve_artist_name(
-        self, platform: str, raw_slug: str, url: str
-    ) -> str:
-        """The real display name for a new artist, resolved from the platform at
-        add-time (#130). Our native platforms each have a name source — pixiv the
-        app API (token), patreon the campaigns API, subscribestar the profile
-        page (both cookies). Other platforms (and any failure — no credential,
-        network error) fall back to the URL handle, which is already readable.
-        The resolvers are sync, so they run in an executor."""
-        if self._crypto is None or platform not in ("pixiv", "patreon", "subscribestar"):
-            return raw_slug
-        import asyncio
-
-        from .credential_service import CredentialService
-        cred = CredentialService(self.session, self._crypto)
-        loop = asyncio.get_running_loop()
-        try:
-            if platform == "pixiv":
-                token = await cred.get_token("pixiv")
-                if not token:
-                    return raw_slug
-                from .pixiv_client import PixivClient
-                name = await loop.run_in_executor(
-                    None, PixivClient(token).resolve_display_name, raw_slug
-                )
-            elif platform == "patreon":
-                cookies = await cred.get_cookies_path("patreon")
-                from .patreon_resolver import resolve_display_name
-                name = await loop.run_in_executor(
-                    None, resolve_display_name, raw_slug,
-                    str(cookies) if cookies else None,
-                )
-            else:  # subscribestar
-                cookies = await cred.get_cookies_path("subscribestar")
-                from .subscribestar_client import SubscribeStarClient
-                client = SubscribeStarClient(str(cookies) if cookies else None)
-                name = await loop.run_in_executor(
-                    None, client.resolve_display_name, url
-                )
-        except Exception as exc:      # resolution is best-effort — never block the add
-            log.warning("artist display-name resolution failed (%s): %s", platform, exc)
-            return raw_slug
-        return name or raw_slug
-
     async def probe(self, url: str) -> dict:
         """Read-only resolution of a creator-page URL against the FC DB.
         Returns one of:

backend/app/services/external_links.py

@@ -1,52 +0,0 @@
-"""Surgical re-fetch of a post's external file-host links.
-
-The normal download cadence never re-walks deep back-catalogue posts (the
-seen-gates exist precisely to keep old items from resurfacing), so when a
-file that CAME from an ExternalLink is deleted — e.g. the failure-triage
-recovery flow removing a corrupt original — a plain source re-check will
-never bring it back. The link ROW is the durable, per-post handle: resetting
-it to pending and dispatching the fetch re-downloads exactly that link's
-payload, and sha-dedupe at import discards anything that still exists — so
-only the missing file actually lands. (Operator 2026-07-03: recovery must
-not require artist-wide deep scans.)
-"""
-import logging
-
-from sqlalchemy import select
-from sqlalchemy.orm import Session
-
-from ..models import ExternalLink
-
-log = logging.getLogger(__name__)
-
-
-def refetch_links_for_post(session: Session, post_id: int) -> dict:
-    """Reset every settled ExternalLink on a post to pending (fresh attempt
-    budget) and dispatch its fetch. In-flight ('downloading') links are left
-    alone. Commits. Returns {links_total, links_reset}."""
-    links = session.execute(
-        select(ExternalLink).where(ExternalLink.post_id == post_id)
-    ).scalars().all()
-    reset_ids = []
-    for link in links:
-        if link.status == "downloading":
-            continue
-        link.status = "pending"
-        link.attempts = 0
-        link.last_error = None
-        link.completed_at = None
-        reset_ids.append(link.id)
-    session.commit()
-    if reset_ids:
-        # Lazy import (services -> tasks would cycle at module load). The
-        # 10-min extdl sweep would pick pending rows up anyway — dispatching
-        # directly just skips the wait.
-        from ..tasks.external import fetch_external_link
-
-        for lid in reset_ids:
-            fetch_external_link.delay(lid)
-        log.info(
-            "external refetch: post %s — %d/%d link(s) reset + dispatched",
-            post_id, len(reset_ids), len(links),
-        )
-    return {"links_total": len(links), "links_reset": len(reset_ids)}

backend/app/services/gallery_dl.py

@@ -157,7 +157,7 @@ class DownloadResult:
     # pairs for already-present media whose ImageRecord.source_filehash should be
     # backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
     # on the gallery-dl path and outside recapture.
-    relink_source_paths: list[tuple[str, str, str]] = field(default_factory=list)
+    relink_source_paths: list[tuple[str, str]] = field(default_factory=list)
     stdout: str = ""
     stderr: str = ""
     return_code: int = 0
@@ -298,9 +298,8 @@ class GalleryDLService:
     # removed at the plan-#697 cutover — it now uses the native ingester
     # (services/patreon_ingester.py), not gallery-dl.
     PLATFORM_DEFAULTS = {
-        # subscribestar removed — native-ingester platform now (#71); pixiv
-        # removed likewise (#129). The remaining entries are the gallery-dl
-        # platforms not yet migrated.
+        # subscribestar removed — it's a native-ingester platform now (#71); the
+        # remaining entries are the gallery-dl platforms not yet migrated.
         "hentaifoundry": {
             "content_types": ["all"],
             "directory": [],
@@ -316,6 +315,12 @@ class GalleryDLService:
             "reactions": False,
             "threads": True,
         },
+        "pixiv": {
+            "content_types": ["all"],
+            "directory": ["{category}"],
+            "filename": "{id}_{title[:50]}_{num:>02}.{extension}",
+            "ugoira": True,
+        },
         "deviantart": {
             "content_types": ["all"],
             "directory": [],
@@ -689,6 +694,9 @@ class GalleryDLService:
         if auth_token and platform == "discord":
             config["extractor"].setdefault("discord", {})
             config["extractor"]["discord"]["token"] = auth_token
+        if auth_token and platform == "pixiv":
+            config["extractor"].setdefault("pixiv", {})
+            config["extractor"]["pixiv"]["refresh-token"] = auth_token
 
         with tempfile.NamedTemporaryFile(
             mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
@@ -874,6 +882,8 @@ class GalleryDLService:
             config["extractor"]["cookies"] = cookies_path
         if auth_token and platform == "discord":
             config["extractor"].setdefault("discord", {})["token"] = auth_token
+        if auth_token and platform == "pixiv":
+            config["extractor"].setdefault("pixiv", {})["refresh-token"] = auth_token
 
         with tempfile.NamedTemporaryFile(
             mode="w", suffix=".json", delete=False, dir=str(self._config_dir),

backend/app/services/gallery_service.py

@@ -22,16 +22,8 @@ from sqlalchemy import Select, and_, distinct, exists, func, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import aliased
 
-from ..models import (
-    Artist,
-    ImageProvenance,
-    ImageRecord,
-    Post,
-    Source,
-    Tag,
-    TagPositiveConfirmation,
-)
-from ..models.tag import CHROME_SYSTEM_TAGS, PROCESS_SYSTEM_TAGS, image_tag
+from ..models import Artist, ImageProvenance, ImageRecord, Post, Source, Tag
+from ..models.tag import image_tag
 from .pagination import decode_cursor, encode_cursor
 from .tag_query import (
     fandom_join_alias,
@@ -63,25 +55,6 @@ def _effective_date_col():
     return ImageRecord.effective_date
 
 
-# Sort key -> the materialized column the gallery orders + cursors on. Both are
-# indexed (DESC, id DESC), so every sort is a forward index range scan.
-#   newest/oldest    → effective_date    (primary post's date, else download)
-#   posted_new/_old  → earliest_post_date (earliest publish across ALL posts)
-_SORT_COLUMNS = {
-    "newest": ImageRecord.effective_date,
-    "oldest": ImageRecord.effective_date,
-    "posted_new": ImageRecord.earliest_post_date,
-    "posted_old": ImageRecord.earliest_post_date,
-}
-_ASCENDING_SORTS = {"oldest", "posted_old"}
-
-
-def _sort_column(sort: str):
-    """The materialized date column a gallery sort orders/cursors on (falls back
-    to effective_date for any unknown sort)."""
-    return _SORT_COLUMNS.get(sort, ImageRecord.effective_date)
-
-
 def _outer_join_primary_post(stmt: Select) -> Select:
     """LEFT JOIN Post on ImageRecord.primary_post_id so the COALESCE
     above sees Post.post_date when available. Images without a post
@@ -187,7 +160,7 @@ def _apply_scope(
     stmt, *, tag_ids, post_id, artist_id, media_type,
     tag_or_groups=None, tag_exclude=None,
     platform=None, untagged=False, no_artist=False,
-    date_from=None, date_to=None, hidden_tag_ids=None,
+    date_from=None, date_to=None,
 ):
     """Apply the composable gallery filters to a statement.
 
@@ -224,12 +197,6 @@ def _apply_scope(
         stmt = stmt.where(image_in_any_tag_scope(group))
     if tag_exclude:
         stmt = stmt.where(~image_in_any_tag_scope(tag_exclude))
-    # Presentation chrome (banner / editor screenshot) is hidden from the default
-    # gallery — an implicit exclude the caller supplies unless the operator asked
-    # to include hidden or is explicitly filtering for a presentation tag
-    # (milestone 141). `wip` is NOT hidden. Resolved to ids by _hidden_tag_ids.
-    if hidden_tag_ids:
-        stmt = stmt.where(~image_in_any_tag_scope(hidden_tag_ids))
     prov = _provenance_clause(post_id, artist_id)
     if prov is not None:
         stmt = stmt.where(prov)
@@ -322,99 +289,6 @@ def _gallery_images(rows, artists: dict[int, dict]) -> list[GalleryImage]:
     ]
 
 
-def _diversify_similar(src, rows, limit, *, dup_threshold=8, lam=0.40):
-    """Trim a nearest-cosine candidate pool down to `limit` diverse picks.
-
-    1. pHash collapse: drop any candidate whose perceptual hash is within
-       `dup_threshold` Hamming bits of the anchor or an already-kept candidate —
-       so a reposted banner (and the anchor's own clones) appears at most once.
-    2. MMR (Maximal Marginal Relevance): greedily pick the candidate maximising
-       `lam * sim_to_anchor - (1 - lam) * max_sim_to_already_picked`. This keeps
-       the most relevant up top but pushes the selection to SPAN clusters
-       instead of returning 40 variations of one image.
-
-    `lam` is the variance dial: lower = weight the diversity penalty harder, so
-    the rail reaches further across clusters (operator wanted MORE variance,
-    2026-07-01 — dropped 0.55→0.40, dup 6→8, paired with a wider pool in
-    `similar()`).
-
-    Falls back to nearest-order (`rows[:limit]`) on any failure or a small pool.
-    """
-    if len(rows) <= 1:
-        return rows[:limit]
-    try:
-        import imagehash
-        import numpy as np
-    except Exception:
-        return rows[:limit]
-
-    # --- 1. pHash near-duplicate collapse (videos/NULL phash pass through) ---
-    kept = []
-    seen = []
-    if src.phash:
-        try:
-            seen.append(imagehash.hex_to_hash(src.phash))
-        except Exception:
-            pass
-    for row in rows:
-        ph = row[0].phash
-        if ph:
-            try:
-                h = imagehash.hex_to_hash(ph)
-                if any((h - k) <= dup_threshold for k in seen):
-                    continue
-                seen.append(h)
-            except Exception:
-                pass
-        kept.append(row)
-    if len(kept) <= limit:
-        return kept
-
-    # --- 2. MMR re-rank on the L2-normalised SigLIP embeddings ---
-    try:
-        a = np.asarray(src.siglip_embedding, dtype=np.float32)
-        a = a / (np.linalg.norm(a) or 1.0)
-        V = np.vstack([
-            np.asarray(row[0].siglip_embedding, dtype=np.float32) for row in kept
-        ])
-        V = V / np.clip(np.linalg.norm(V, axis=1, keepdims=True), 1e-8, None)
-    except Exception:
-        return kept[:limit]
-
-    rel = V @ a                                  # (N,) cosine to the anchor
-    n = len(kept)
-    picked_mask = np.zeros(n, dtype=bool)
-    max_sim = np.zeros(n, dtype=np.float32)      # max sim to anything picked yet
-    order = []
-    for _ in range(min(limit, n)):
-        scores = lam * rel - (1.0 - lam) * max_sim
-        scores[picked_mask] = -np.inf
-        i = int(np.argmax(scores))
-        order.append(i)
-        picked_mask[i] = True
-        max_sim = np.maximum(max_sim, V @ V[i])
-    return [kept[i] for i in order]
-
-
-def _reach_sample(rows, limit, reach):
-    """From a distance-sorted candidate pool (nearest first), pick a spread of ranks
-    that MIXES near (tag the current cluster) and mid-far (escape it) BEFORE dedup +
-    MMR — the Explore "reach" dial (#1476).
-
-    reach in (0, 1]: the sampled span grows outward from the anchor (0.25→1.0 of the
-    pool), evenly strided from rank 0 so the nearest are still represented. In a
-    dense signature the nearest ranks are near-identical, so reaching farther is the
-    only way to hand MMR genuinely different content — MMR alone can't escape a pool
-    that's already all-near. reach<=0 or a small pool passes through unchanged."""
-    n = len(rows)
-    want = max(limit * 8, 100)
-    if reach <= 0 or n <= want:
-        return rows
-    span = int(min(1.0, 0.25 + 0.75 * reach) * n)
-    idx = sorted({min(int(i * span / want), n - 1) for i in range(want)})
-    return [rows[i] for i in idx]
-
-
 async def _artists_for(session, image_ids: list[int]) -> dict[int, dict]:
     """Map image_id -> {"name","slug"} via the canonical
     image_record.artist_id (FC-2d-vii-c). Bounded by page size."""
@@ -435,32 +309,6 @@ class GalleryService:
     def __init__(self, session: AsyncSession):
         self.session = session
 
-    async def _hidden_tag_ids(
-        self, include_hidden, tag_ids, tag_or_groups,
-    ) -> list[int] | None:
-        """Chrome (banner) tag ids to implicitly exclude from a gallery query, or
-        None. None when the caller asked to include hidden, when the operator is
-        explicitly filtering FOR a chrome tag (they clearly want to see it), or when
-        no chrome tags exist. (milestone 141; #1464: editor screenshot is now PROCESS
-        — shown — so only `banner` hides here.)"""
-        if include_hidden:
-            return None
-        rows = await self.session.execute(
-            select(Tag.id).where(
-                Tag.is_system.is_(True),
-                Tag.name.in_(CHROME_SYSTEM_TAGS),
-            )
-        )
-        pres = [r[0] for r in rows]
-        if not pres:
-            return None
-        explicit = set(tag_ids or [])
-        for group in tag_or_groups or []:
-            explicit.update(group)
-        if explicit & set(pres):
-            return None
-        return pres
-
     async def scroll(
         self,
         cursor: str | None,
@@ -477,21 +325,14 @@ class GalleryService:
         no_artist: bool = False,
         date_from: datetime | None = None,
         date_to: datetime | None = None,
-        include_hidden: bool = False,
     ) -> GalleryPage:
         if limit < 1 or limit > 200:
             raise ValueError("limit must be between 1 and 200")
         _require_single_filter(
             tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
         )
-        hidden = await self._hidden_tag_ids(
-            include_hidden, tag_ids, tag_or_groups,
-        )
 
-        # eff is the ACTIVE sort column (effective_date or earliest_post_date);
-        # the cursor, ordering and year/month grouping all key off it, so the
-        # 'post date' sort paginates + buckets by original publish transparently.
-        eff = _sort_column(sort)
+        eff = _effective_date_col()
         stmt = select(ImageRecord, Post.post_date, eff.label("eff"))
         stmt = _outer_join_primary_post(stmt)
         stmt = _apply_scope(
@@ -499,10 +340,10 @@ class GalleryService:
             artist_id=artist_id, media_type=media_type,
             tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
             platform=platform, untagged=untagged, no_artist=no_artist,
-            date_from=date_from, date_to=date_to, hidden_tag_ids=hidden,
+            date_from=date_from, date_to=date_to,
         )
 
-        descending = sort not in _ASCENDING_SORTS
+        descending = sort != "oldest"
         if cursor:
             cur_ts, cur_id = decode_cursor(cursor)
             # The cursor is just (last eff, last id); the request's sort
@@ -552,7 +393,6 @@ class GalleryService:
         no_artist: bool = False,
         date_from: datetime | None = None,
         date_to: datetime | None = None,
-        include_hidden: bool = False,
     ) -> list[TimelineBucket]:
         eff = _effective_date_col()
         year_col = func.date_part("year", eff).label("yr")
@@ -564,15 +404,12 @@ class GalleryService:
         _require_single_filter(
             tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
         )
-        hidden = await self._hidden_tag_ids(
-            include_hidden, tag_ids, tag_or_groups,
-        )
         stmt = _apply_scope(
             stmt, tag_ids=tag_ids, post_id=post_id,
             artist_id=artist_id, media_type=media_type,
             tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
             platform=platform, untagged=untagged, no_artist=no_artist,
-            date_from=date_from, date_to=date_to, hidden_tag_ids=hidden,
+            date_from=date_from, date_to=date_to,
         )
         stmt = stmt.group_by(year_col, month_col).order_by(year_col.desc(), month_col.desc())
         rows = (await self.session.execute(stmt)).all()
@@ -586,7 +423,7 @@ class GalleryService:
         tag_exclude: list[int] | None = None,
         platform: str | None = None, untagged: bool = False,
         no_artist: bool = False, date_from: datetime | None = None,
-        date_to: datetime | None = None, include_hidden: bool = False,
+        date_to: datetime | None = None,
     ) -> str | None:
         """Returns a cursor that, when passed to scroll() with the same sort,
         positions at the first image of the given year-month. None if the
@@ -603,15 +440,12 @@ class GalleryService:
         _require_single_filter(
             tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
         )
-        hidden = await self._hidden_tag_ids(
-            include_hidden, tag_ids, tag_or_groups,
-        )
         stmt = _apply_scope(
             stmt, tag_ids=tag_ids, post_id=post_id,
             artist_id=artist_id, media_type=media_type,
             tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
             platform=platform, untagged=untagged, no_artist=no_artist,
-            date_from=date_from, date_to=date_to, hidden_tag_ids=hidden,
+            date_from=date_from, date_to=date_to,
         )
         descending = sort != "oldest"
         if descending:
@@ -636,7 +470,6 @@ class GalleryService:
         platform: str | None = None,
         untagged: bool = False, no_artist: bool = False,
         date_from: datetime | None = None, date_to: datetime | None = None,
-        include_hidden: bool = False,
     ) -> GalleryFacets:
         """Live facet counts scoped to the current filter. Each facet GROUP is
         computed with all OTHER active filters applied but its OWN selection
@@ -647,14 +480,10 @@ class GalleryService:
         _require_single_filter(
             tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
         )
-        hidden = await self._hidden_tag_ids(
-            include_hidden, tag_ids, tag_or_groups,
-        )
         common = {
             "tag_ids": tag_ids, "post_id": post_id,
             "artist_id": artist_id, "media_type": media_type,
             "tag_or_groups": tag_or_groups, "tag_exclude": tag_exclude,
-            "hidden_tag_ids": hidden,
         }
 
         # total — the full active filter (the headline result count).
@@ -735,23 +564,15 @@ class GalleryService:
         platform: str | None = None,
         untagged: bool = False, no_artist: bool = False,
         date_from: datetime | None = None, date_to: datetime | None = None,
-        exclude_wip: bool = False,
-        reach: float = 0.0, exclude_ids: list[int] | None = None,
     ) -> list[GalleryImage] | None:
-        """Visual "more like this": images near `image_id`'s SigLIP embedding
-        (pgvector, HNSW-indexed — alembic 0036), then DIVERSIFIED so the result
-        doesn't collapse into one cluster. No ML inference here.
+        """Visual "more like this": images ranked by cosine distance to
+        `image_id`'s SigLIP embedding (pgvector, HNSW-indexed — alembic 0036).
+        No ML inference here; the embedding was computed at import.
 
-        Pure nearest-cosine piles up near-identical images — a reposted banner
-        fills the whole grid, and once you wander into a B&W / comic-panel
-        cluster every neighbour is more of the same with no way back to colour
-        (operator-reported 2026-06-30). So we pull a WIDER candidate pool, then:
-          1. collapse near-duplicate pHashes (and drop clones of the anchor),
-          2. MMR re-rank — pick for closeness-to-anchor but penalise similarity
-             to what's already picked, so the result SPANS clusters.
-
-        Returns None if the source doesn't exist (→ 404), [] if it has no
-        embedding. Composes with the scope filters (AND); REPLACES the date sort.
+        Returns None if the source image doesn't exist (→ 404), [] if it has
+        no embedding (a video / not-yet-embedded). Composes with the Phase-1/2
+        scope filters (AND) but REPLACES the date sort — always nearest-first,
+        bounded to `limit` (no cursor; distance-ranking has no date cursor).
         """
         if limit < 1 or limit > 200:
             raise ValueError("limit must be between 1 and 200")
@@ -761,48 +582,14 @@ class GalleryService:
         if src.siglip_embedding is None:
             return []
 
-        # Over-fetch so diversification has clusters to spread across — without a
-        # wide pool there's nothing but the near-dupes to choose from. Widened
-        # (5×→8×, cap 200→400) so the stronger MMR has genuinely distinct
-        # neighbourhoods to reach into for more variance (operator, 2026-07-01).
-        # Explore's reach>0 (#1476) widens it a LOT more: in a dense signature the
-        # nearest few hundred are all near-identical, so far-enough candidates only
-        # exist deeper in the ranked pool. _reach_sample then strides across them.
-        if reach > 0:
-            pool_n = min(1000, max(limit * 25, 100))
-        else:
-            pool_n = min(400, max(limit * 8, 100))
         distance = ImageRecord.siglip_embedding.cosine_distance(src.siglip_embedding)
         eff = _effective_date_col()
         stmt = select(ImageRecord, Post.post_date, eff.label("eff"))
         stmt = _outer_join_primary_post(stmt)
-        # Chrome (banner, #128) clusters on UI rather than content, so near any one
-        # of them they'd fill the grid → excluded from CANDIDATES always (the anchor
-        # itself may be a banner). PROCESS art (wip / editor screenshot) stays
-        # surfaced here by default (real content; only the training pipelines exclude
-        # it), but the Explore rabbit-hole passes exclude_wip to also drop the whole
-        # process group so a browse doesn't keep surfacing work-in-progress
-        # (operator, 2026-07-08; #1464 — editor now rides with wip here).
-        excluded_system_tags = CHROME_SYSTEM_TAGS
-        if exclude_wip:
-            excluded_system_tags = (*CHROME_SYSTEM_TAGS, *PROCESS_SYSTEM_TAGS)
-        presentation = (
-            select(image_tag.c.image_record_id)
-            .join(Tag, Tag.id == image_tag.c.tag_id)
-            .where(
-                Tag.is_system.is_(True),
-                Tag.name.in_(excluded_system_tags),
-            )
-        )
         stmt = stmt.where(
             ImageRecord.siglip_embedding.is_not(None),
             ImageRecord.id != image_id,
-            ImageRecord.id.not_in(presentation),
         )
-        # Anti-revisit (#1476): the Explore walk passes its breadcrumb so already-
-        # walked images aren't re-served as neighbours — → can't loop you back in.
-        if exclude_ids:
-            stmt = stmt.where(ImageRecord.id.not_in(exclude_ids))
         stmt = _apply_scope(
             stmt, tag_ids=tag_ids, post_id=None,
             artist_id=artist_id, media_type=media_type,
@@ -810,13 +597,8 @@ class GalleryService:
             platform=platform, untagged=untagged, no_artist=no_artist,
             date_from=date_from, date_to=date_to,
         )
-        stmt = stmt.order_by(distance.asc()).limit(pool_n)
+        stmt = stmt.order_by(distance.asc()).limit(limit)
         rows = (await self.session.execute(stmt)).all()
-        # Explore reach: stride across an outward-growing distance span so the pool
-        # handed to MMR spans near→mid-far, not just the tight cluster (#1476).
-        if reach > 0:
-            rows = _reach_sample(rows, limit, reach)
-        rows = _diversify_similar(src, rows, limit)
         artists = await _artists_for(self.session, [r[0].id for r in rows])
         return _gallery_images(rows, artists)
 
@@ -827,14 +609,8 @@ class GalleryService:
         # Self-join Tag to resolve a character's fandom NAME (not just id) so the
         # modal chip can label it without an N+1 (shared tag_query helpers).
         fandom_alias = fandom_join_alias()
-        # source drives the auto-applied badge; confirmed = operator affirmed the
-        # tag (positive + retraction-shielded, milestone 139).
-        confirmed = exists().where(
-            TagPositiveConfirmation.image_record_id == image_id,
-            TagPositiveConfirmation.tag_id == Tag.id,
-        ).label("confirmed")
         tag_stmt = (
-            select(*tag_columns(fandom_alias), image_tag.c.source, confirmed)
+            select(*tag_columns(fandom_alias))
             .select_from(
                 Tag.__table__
                 .join(image_tag, image_tag.c.tag_id == Tag.id)

backend/app/services/importer.py

@@ -17,7 +17,7 @@ from enum import StrEnum
 from pathlib import Path
 
 from PIL import Image
-from sqlalchemy import func, select, update
+from sqlalchemy import select, update
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
@@ -44,24 +44,11 @@ from ..utils.sidecar import find_sidecar, parse_sidecar
 from ..utils.slug import slugify
 from .archive_extractor import extract_archive, is_archive
 from .attachment_store import AttachmentStore
-from .audits import single_color
 from .link_extract import extract_external_links
 from .thumbnailer import Thumbnailer
-from .wip_title import (
-    WIP_TITLE_SOFT_SOURCE,
-    WIP_TITLE_SOURCE,
-    apply_wip_image_tags,
-    matches_soft_wip_title,
-    matches_wip_title,
-    resolve_wip_tag_id,
-)
 
 log = logging.getLogger(__name__)
 
-# Sentinel for the lazily-resolved wip tag id (distinguishes "not resolved yet"
-# from a genuine None = tag absent, so absence is cached and not re-queried).
-_UNSET = object()
-
 
 class SkipReason(StrEnum):
     too_small = "too_small"
@@ -195,10 +182,6 @@ class Importer:
         # invalidated mid-Importer (Importer instances are per-task /
         # per-archive-import so cross-instance staleness is harmless).
         self._phash_candidates: list[tuple] | None = None
-        # Lazily-resolved `wip` system tag id for title-based WIP auto-tagging
-        # (task #1458). Sentinel _UNSET so a genuine None (tag absent) is cached
-        # and not re-queried per media. Importer is per-task, so this can't stale.
-        self._wip_tag_id: int | None = _UNSET
 
     def _phash_candidates_cache(self) -> list[tuple]:
         """Cached `(phash, width, height, id)` rows from image_record.
@@ -807,13 +790,6 @@ class Importer:
                         error=f"{pct:.2%} transparent",
                     )
 
-            if self.settings.skip_single_color and self._single_color_hit(source):
-                return ImportResult(
-                    status="skipped", skip_reason=SkipReason.single_color,
-                    error=(f"one color dominates >"
-                           f"{self.settings.single_color_threshold:.0%}"),
-                )
-
         # Artist anchored to the attribution path (folder→artist), resolved
         # UP-FRONT so the enrich-on-duplicate branches link provenance with the
         # right artist even when the sidecar carries none — which is now the norm
@@ -949,10 +925,6 @@ class Importer:
         # Thumbnail is queued separately by the calling task; the importer
         # does not generate thumbnails inline so the import queue stays moving.
 
-        # Title-based WIP auto-tag (task #1458): fresh import only, after the
-        # sidecar has linked the post so record.primary_post_id / its title exist.
-        self._maybe_apply_wip_title(record)
-
         self.session.commit()
         return ImportResult(status="imported", image_id=record.id)
 
@@ -996,47 +968,6 @@ class Importer:
         self.session.commit()
         return ImportResult(status="refreshed", image_id=existing.id)
 
-    def _maybe_apply_wip_title(self, record: ImageRecord) -> None:
-        """Auto-apply the `wip` system tag to a FRESHLY-imported image when its
-        primary post's TITLE explicitly declares work-in-progress (task #1458 —
-        the artist's own "WIP" / "work in progress" label).
-
-        Called ONLY from the two new-record paths (never deep-scan / supersede),
-        so a manually-removed WIP tag is never re-applied by a routine re-scan —
-        removal sticks. The existing catalogue is covered separately by the
-        operator-triggered backfill sweep. Gated by the settings toggle, and
-        best-effort: any failure is logged, never allowed to fail the import."""
-        hard_on = self.settings.wip_title_tagging_enabled
-        soft_on = self.settings.wip_soft_title_tagging_enabled
-        if not (hard_on or soft_on):
-            return
-        if record.primary_post_id is None:
-            return
-        try:
-            title = self.session.execute(
-                select(Post.post_title).where(Post.id == record.primary_post_id)
-            ).scalar_one_or_none()
-            # HARD tier ("WIP"/"work in progress") wins — higher precision, and it
-            # trains the head; SOFT (sketch/doodle, #1474) is the provisional fallback
-            # that never trains (source wip_title_soft).
-            if hard_on and matches_wip_title(title):
-                source = WIP_TITLE_SOURCE
-            elif soft_on and matches_soft_wip_title(title):
-                source = WIP_TITLE_SOFT_SOURCE
-            else:
-                return
-            if self._wip_tag_id is _UNSET:
-                self._wip_tag_id = resolve_wip_tag_id(self.session)
-            if self._wip_tag_id is None:
-                return
-            apply_wip_image_tags(
-                self.session, [record.id], self._wip_tag_id, source=source
-            )
-        except Exception as exc:  # noqa: BLE001 — a tag must never fail an import
-            log.warning(
-                "wip-title auto-tag failed for image %s: %s", record.id, exc
-            )
-
     def _apply_post_fields(self, post: Post, sd) -> None:
         """Write a parsed sidecar's post-level fields onto a Post — the SINGLE
         predicate shared by BOTH ingest paths: the per-media path (_apply_sidecar)
@@ -1192,13 +1123,6 @@ class Importer:
                         status="skipped", skip_reason=SkipReason.too_transparent,
                         error=f"{pct:.2%} transparent",
                     )
-
-            if self.settings.skip_single_color and self._single_color_hit(path):
-                return ImportResult(
-                    status="skipped", skip_reason=SkipReason.single_color,
-                    error=(f"one color dominates >"
-                           f"{self.settings.single_color_threshold:.0%}"),
-                )
         else:
             # Best-effort probe for dims + duration so downloaded videos can dedup
             # (#871). LENIENT: unlike _import_media this path does not reject on a
@@ -1314,10 +1238,6 @@ class Importer:
         # per-post Source row.
         self._apply_sidecar(record, path, artist, explicit_source=source)
 
-        # Title-based WIP auto-tag (task #1458): fresh import only, see the
-        # matching call in _import_media.
-        self._maybe_apply_wip_title(record)
-
         self.session.commit()
         return ImportResult(status="imported", image_id=record.id)
 
@@ -1378,113 +1298,6 @@ class Importer:
         self.session.commit()
         return True
 
-    def link_existing_image_to_post(
-        self, path: Path, external_post_id: str, *,
-        source: Source | None = None, artist: Artist | None = None,
-    ) -> bool:
-        """Recapture back-link: associate an ALREADY-on-disk image (matched by
-        its stored path) with its post — the link _apply_sidecar normally makes
-        at per-media import time.
-
-        Recapture disk-skips downloaded media, so a pre-existing image (e.g. one
-        pulled under the old gallery-dl path, imported as a bare record with no
-        post) never gets its `image_provenance` row / `primary_post_id`. This
-        backfills it from the walk's (on-disk path, post external id) pairing:
-        find the ImageRecord by path, find/attach the Post by (source,
-        external_post_id), upsert provenance. Idempotent (issue #1288). Returns
-        True when a record matched and was linked; no-op when the file has no
-        record or no id is given."""
-        if not external_post_id:
-            return False
-        record = self.session.execute(
-            select(ImageRecord).where(ImageRecord.path == str(path))
-        ).scalar_one_or_none()
-        if record is None:
-            return False
-        artist_id = record.artist_id or (artist.id if artist else None)
-        if artist_id is None:
-            return False
-        if record.artist_id is None:
-            record.artist_id = artist_id
-        post = self._find_or_create_post(
-            source_id=source.id if source else None,
-            external_post_id=str(external_post_id),
-            artist_id=artist_id,
-        )
-        self._attach_provenance(
-            record, post, source_id=source.id if source else None,
-        )
-        self.session.commit()
-        return True
-
-    def _attach_provenance(
-        self, record: ImageRecord, post: Post, *,
-        source_id: int | None, captured_metadata: dict | None = None,
-    ) -> None:
-        """Upsert the (image, post) `image_provenance` link + keep
-        `primary_post_id` and the denormalized gallery sort keys aligned. Shared
-        by _apply_sidecar (fresh per-media import) and link_existing_image_to_post
-        (recapture back-link, #1288) so the two paths can't diverge. Idempotent.
-
-        Race-safe (image_record_id, post_id) upsert — mirrors the
-        _find_or_create_source/post savepoint pattern. The plain
-        SELECT-then-INSERT pattern lost a race when two workers ran on the same
-        (image, post) pair (e.g. the 5-min recovery sweep re-enqueued a
-        still-running long import), planting duplicates that then broke
-        .scalar_one_or_none() on every later deep-scan rederive
-        (MultipleResultsFound). Alembic 0021's uq_image_provenance_image_post
-        UNIQUE makes this savepoint trip on collision."""
-        exists = self.session.execute(
-            select(ImageProvenance.id).where(
-                ImageProvenance.image_record_id == record.id,
-                ImageProvenance.post_id == post.id,
-            )
-        ).scalar_one_or_none()
-        if exists is None:
-            sp = self.session.begin_nested()
-            try:
-                self.session.add(
-                    ImageProvenance(
-                        image_record_id=record.id,
-                        post_id=post.id,
-                        source_id=source_id,
-                        captured_metadata=captured_metadata,
-                    )
-                )
-                self.session.flush()
-                sp.commit()
-            except IntegrityError:
-                sp.rollback()
-        if record.primary_post_id is None:
-            record.primary_post_id = post.id
-        # Keep the denormalized gallery sort key (alembic 0035) aligned with
-        # the primary post's publish date so /scroll orders off
-        # ix_image_record_effective_date instead of COALESCE-ing across the
-        # post join. Only override when THIS post is the primary AND carries
-        # a date; otherwise the column keeps its created_at-equivalent server
-        # default (matches the old COALESCE(post_date, created_at) fallback).
-        if record.primary_post_id == post.id and post.post_date is not None:
-            record.effective_date = post.post_date
-        # earliest_post_date (alembic 0071) = MIN(post_date) across ALL of this
-        # image's provenance posts, not just the primary — so the gallery can
-        # sort by original publish rather than the download/repost the primary
-        # points at. Recompute from provenance whenever a dated post is linked;
-        # the provenance row for THIS post was committed above, so the MIN
-        # includes it. Leaves the created_at default when no linked post is dated.
-        if post.post_date is not None:
-            earliest = self.session.execute(
-                select(func.min(Post.post_date))
-                .select_from(ImageProvenance)
-                .join(Post, Post.id == ImageProvenance.post_id)
-                .where(
-                    ImageProvenance.image_record_id == record.id,
-                    Post.post_date.is_not(None),
-                )
-            ).scalar_one_or_none()
-            if earliest is not None:
-                record.earliest_post_date = earliest
-        self.session.flush()
-
     def _apply_sidecar(
         self,
         record: ImageRecord,
@@ -1558,12 +1371,47 @@ class Importer:
         if not self.post_first:
             self._apply_post_fields(post, sd)
 
-        # Link the (image, post) provenance + keep primary_post_id / the gallery
-        # sort keys aligned — shared with the recapture back-link path (#1288).
-        self._attach_provenance(
-            record, post, source_id=src.id if src else None,
-            captured_metadata=sd.raw,
-        )
+        # Race-safe (image_record_id, post_id) upsert — mirrors the
+        # _find_or_create_source/post savepoint pattern. The plain
+        # SELECT-then-INSERT pattern lost a race when two workers ran
+        # _apply_sidecar on the same (image, post) pair (e.g. the 5-min
+        # recovery sweep re-enqueued a still-running long import), planting
+        # duplicates that then broke .scalar_one_or_none() on every later
+        # deep-scan rederive (MultipleResultsFound). Alembic 0021 adds the
+        # uq_image_provenance_image_post UNIQUE so this savepoint actually
+        # trips on collision.
+        exists = self.session.execute(
+            select(ImageProvenance.id).where(
+                ImageProvenance.image_record_id == record.id,
+                ImageProvenance.post_id == post.id,
+            )
+        ).scalar_one_or_none()
+        if exists is None:
+            sp = self.session.begin_nested()
+            try:
+                self.session.add(
+                    ImageProvenance(
+                        image_record_id=record.id,
+                        post_id=post.id,
+                        source_id=src.id if src else None,
+                        captured_metadata=sd.raw,
+                    )
+                )
+                self.session.flush()
+                sp.commit()
+            except IntegrityError:
+                sp.rollback()
+        if record.primary_post_id is None:
+            record.primary_post_id = post.id
+        # Keep the denormalized gallery sort key (alembic 0035) aligned with
+        # the primary post's publish date so /scroll orders off
+        # ix_image_record_effective_date instead of COALESCE-ing across the
+        # post join. Only override when THIS post is the primary AND carries
+        # a date; otherwise the column keeps its created_at-equivalent server
+        # default (matches the old COALESCE(post_date, created_at) fallback).
+        if record.primary_post_id == post.id and post.post_date is not None:
+            record.effective_date = post.post_date
+        self.session.flush()
 
     def _copy_to_library(
         self, source: Path, sha: str, attribution_path: Path
@@ -1627,8 +1475,20 @@ class Importer:
         existing.duration_seconds = duration  # #871: keep the kept copy's duration
         existing.thumbnail_path = None
         existing.integrity_status = "unknown"
+        existing.tagger_model_version = None
         existing.siglip_embedding = None
         existing.siglip_model_version = None
+        existing.centroid_scores = None
+        # #768: predictions also live in the normalized image_prediction table
+        # now — clear them so a re-imported file re-derives a fresh set.
+        from sqlalchemy import delete as _delete
+
+        from ..models import ImagePrediction as _ImagePrediction
+        self.session.execute(
+            _delete(_ImagePrediction).where(
+                _ImagePrediction.image_record_id == existing.id
+            )
+        )
         # created_at intentionally preserved; updated_at auto-bumps.
         self.session.flush()
         self.session.commit()
@@ -1672,33 +1532,6 @@ class Importer:
                 # Benign orphan; the DB swap already committed. Don't undo it.
                 pass
 
-    # Matches the Cleanup audit card's default tolerance: the import-side
-    # filter and the retroactive audit must agree on what "single color" MEANS
-    # (Euclidean RGB distance to the dominant color); only the match threshold
-    # is operator-tunable per surface.
-    _SINGLE_COLOR_TOLERANCE = 30
-
-    def _single_color_hit(self, source: Path) -> bool:
-        """True when one color dominates beyond the configured threshold — the
-        same canonical predicate the Cleanup audit runs (audits.single_color,
-        whose docstring anticipated this adoption; the skip_single_color
-        setting existed but was never wired until 2026-07-02). Never raises:
-        unreadable files were already rejected by verify() upstream, and a
-        residual decode error just declines to match (the import proceeds)."""
-        try:
-            with Image.open(source) as im:
-                if getattr(im, "is_animated", False):
-                    # Frame 0 only would misjudge animations; skip like the
-                    # transparency check does.
-                    return False
-                return single_color.evaluate(
-                    im,
-                    threshold=self.settings.single_color_threshold,
-                    tolerance=self._SINGLE_COLOR_TOLERANCE,
-                )
-        except Exception:
-            return False
-
     def _transparency_pct(self, source: Path) -> float:
         """Fraction of fully-transparent pixels in the image. 0.0 if no alpha.
 

backend/app/services/ingest_core.py

@@ -90,7 +90,6 @@ class Ingester:
         platform: str,
         error_base: type[Exception],
         drift_label: str | None = None,
-        body_canary: bool = True,
     ):
         self.client = client
         self.downloader = downloader
@@ -106,12 +105,6 @@ class Ingester:
         # update"). Defaults to the platform name; adapters pass a richer phrase
         # (e.g. "Patreon API", "SubscribeStar markup").
         self._drift_label = drift_label or platform
-        # #862 canary opt-out: platforms whose posts legitimately have empty
-        # bodies across large samples (pixiv — caption-less artists are common)
-        # would false-positive the zero-bodies-means-drift alarm; their clients
-        # catch drift structurally (response-shape checks) instead. The
-        # "bodies X/N" summary line still surfaces the ratio either way.
-        self._body_canary = body_canary
 
     # -- public ------------------------------------------------------------
 
@@ -179,11 +172,10 @@ class Ingester:
         written: list[str] = []
         post_records: list[str] = []
         quarantined_paths: list[str] = []
-        # #830 recapture: (on-disk path, CDN source_url, post_id) triples for
-        # already-present media, so phase 3 can (a) backfill the ImageRecord's
-        # source_filehash and (b) link the on-disk image to its Post (#1288) —
-        # WITHOUT re-downloading or unlinking the file. Empty outside recapture.
-        relink: list[tuple[str, str, str]] = []
+        # #830 recapture: (on-disk path, CDN source_url) pairs for already-present
+        # media, so phase 3 can backfill the ImageRecord's source_filehash WITHOUT
+        # re-downloading or unlinking the file. Empty outside recapture mode.
+        relink: list[tuple[str, str]] = []
         downloaded = 0
         errors = 0
         quarantined = 0
@@ -411,15 +403,12 @@ class Ingester:
                         to_clear.append(key)
                         skipped_count += 1
                         consecutive_seen += 1
-                        # #830/#1288 recapture: surface (on-disk path, CDN url,
-                        # post_id) so phase 3 can backfill source_filehash AND link
-                        # the on-disk image to its Post — a SEPARATE non-deleting
-                        # channel, never the import list (which would unlink the
-                        # file, per above).
+                        # #830 recapture: surface (on-disk path, CDN url) so phase
+                        # 3 can backfill source_filehash for inline-image
+                        # localization — a SEPARATE non-deleting channel, never the
+                        # import list (which would unlink the file, per above).
                         if recapture and outcome.path is not None:
-                            relink.append(
-                                (str(outcome.path), media_item.url, media_item.post_id)
-                            )
+                            relink.append((str(outcome.path), media_item.url))
                     elif outcome.status == "skipped_seen":
                         skipped_count += 1
                         consecutive_seen += 1
@@ -547,11 +536,7 @@ class Ingester:
         # creds") so the breakage screams instead of silently archiving empties.
         # Only reached on an otherwise-clean walk (timeout/stop/error returned
         # above), so it never masks a more specific failure.
-        if (
-            self._body_canary
-            and posts_recorded >= _CANARY_MIN_SAMPLE
-            and posts_with_body == 0
-        ):
+        if posts_recorded >= _CANARY_MIN_SAMPLE and posts_with_body == 0:
             msg = (
                 f"Post-body canary: extracted a body from 0 of {posts_recorded} "
                 "posts — Patreon's body field shape likely changed; the ingester "
@@ -577,6 +562,73 @@ class Ingester:
             error_type=None, error_message=None,
         )
 
+    # -- preview (dry-run) -------------------------------------------------
+
+    def preview(
+        self,
+        source_id: int,
+        campaign_id: str,
+        *,
+        page_limit: int = 3,
+        sample_size: int = 10,
+    ) -> dict:
+        """Dry-run (plan #708 B4): walk up to `page_limit` pages and count media
+        NOT already in the seen/dead ledgers, WITHOUT downloading anything.
+
+        Read-only — only the seen/dead SELECTs touch the DB (short sessions). Lets
+        an operator gauge "is this source worth a backfill?" cheaply. Returns:
+            {total_new, posts_scanned, pages_scanned, has_more,
+             sample: [{title, date, new}, ...]}  # sample = posts with new media
+        A client-level failure (auth/drift) propagates to the caller.
+        """
+        total_new = 0
+        posts_scanned = 0
+        pages_scanned = 0
+        has_more = False
+        sample: list[dict] = []
+        unset = object()
+        last_page: object = unset
+        # #874: same gated-post gate as run() — the preview must not count
+        # blurred locked-preview media as "new", or it would overstate a gated
+        # source's backlog (preview/apply parity, rule 93).
+        post_is_gated = getattr(self.client, "post_is_gated", None)
+        for post, included, page_cursor in self.client.iter_posts(
+            campaign_id, cursor=None
+        ):
+            if page_cursor != last_page:
+                last_page = page_cursor
+                pages_scanned += 1
+                if pages_scanned > page_limit:
+                    has_more = True
+                    pages_scanned = page_limit
+                    break
+            posts_scanned += 1
+            if post_is_gated and post_is_gated(post):
+                continue
+            media = self.client.extract_media(post, included)
+            if not media:
+                continue
+            keys = [self._ledger_key(m) for m in media]
+            skip = self._seen_keys(source_id, keys) | self._dead_keys(source_id, keys)
+            new_count = sum(1 for m in media if self._ledger_key(m) not in skip)
+            total_new += new_count
+            if new_count > 0 and len(sample) < sample_size:
+                meta = self.client.post_meta(post)
+                sample.append(
+                    {
+                        "title": meta.get("title") or "(untitled)",
+                        "date": meta.get("date"),
+                        "new": new_count,
+                    }
+                )
+        return {
+            "total_new": total_new,
+            "posts_scanned": posts_scanned,
+            "pages_scanned": pages_scanned,
+            "has_more": has_more,
+            "sample": sample,
+        }
+
     # -- failure mapping (adapter overrides) -------------------------------
 
     def _failure_result(self, exc: Exception, _result) -> DownloadResult:

backend/app/services/interpreter_client.py

@@ -1,152 +0,0 @@
-"""Interpreter translation client (milestone 143) — a thin SYNC wrapper over the
-self-hosted Interpreter LAN service (LibreTranslate-compatible `/v1/translate`).
-
-Sync (requests) because the only caller is the sync celery translate sweep, and
-it mirrors FC's other platform clients. The service knows nothing about Curator —
-all Curator logic stays here. base_url is operator-configured (empty until set;
-behind a reverse proxy). Full API contract: Scribe note #1347.
-"""
-from __future__ import annotations
-
-from datetime import UTC, datetime
-from email.utils import parsedate_to_datetime
-
-import requests
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
-
-
-class InterpreterUnavailable(Exception):
-    """The translation engine is down / unreachable / draining — a connection
-    error, HTTP 429, or a 5xx (commonly 502/503/504 through a reverse proxy while
-    the service restarts). Retry later, don't drop the item. ``retry_after`` holds
-    the server's Retry-After hint in seconds when it sent one, so the caller can
-    back off exactly as long as it's asked to."""
-
-    def __init__(self, message: str, *, retry_after: float | None = None):
-        super().__init__(message)
-        self.retry_after = retry_after
-
-
-class InterpreterBadRequest(Exception):
-    """Bad request params (HTTP 400)."""
-
-
-def _url(base_url: str, path: str) -> str:
-    return f"{base_url.rstrip('/')}{path}"
-
-
-def _parse_retry_after(resp) -> float | None:
-    """Parse a Retry-After header (RFC 7231: delta-seconds or an HTTP-date) into
-    non-negative seconds, or None if absent/unparseable — so a gracefully-
-    draining Interpreter can tell Curator exactly how long to wait before it
-    tries again."""
-    raw = (resp.headers.get("Retry-After") or "").strip()
-    if not raw:
-        return None
-    try:
-        return max(0.0, float(int(raw)))       # delta-seconds form
-    except ValueError:
-        pass
-    try:                                        # HTTP-date form
-        when = parsedate_to_datetime(raw)
-    except (TypeError, ValueError):
-        return None
-    if when is None:
-        return None
-    if when.tzinfo is None:
-        when = when.replace(tzinfo=UTC)
-    return max(0.0, (when - datetime.now(UTC)).total_seconds())
-
-
-# A shared session pools the keep-alive connection across the per-post sweep
-# calls, and retries CONNECT failures only (connect=2, short backoff) — smoothing
-# the instant a reverse proxy reloads. Status codes are deliberately NOT retried
-# (status=0, raise_on_status=False): translate() maps 429/5xx → InterpreterUnavailable
-# itself, and letting urllib3 retry a draining 503 would defeat the Retry-After
-# backoff we honour upstream.
-_retry = Retry(
-    total=None, connect=2, read=0, redirect=0, status=0,
-    backoff_factor=0.3, raise_on_status=False,
-)
-session = requests.Session()
-_adapter = HTTPAdapter(max_retries=_retry)
-session.mount("http://", _adapter)
-session.mount("https://", _adapter)
-
-
-def health(base_url: str, *, timeout: float = 5.0) -> bool:
-    """True iff the Interpreter LLM engine is up. Any error (unset URL, network,
-    non-200, engine down) → False, so the sweep just no-ops rather than raising."""
-    if not base_url:
-        return False
-    try:
-        r = session.get(_url(base_url, "/v1/health"), timeout=timeout)
-    except requests.RequestException:
-        return False
-    if r.status_code != 200:
-        return False
-    engines = (r.json() or {}).get("engines") or {}
-    return bool(engines.get("llm"))
-
-
-def translate(
-    texts: list[str], *, base_url: str, target: str = "en",
-    source: str = "auto", timeout: float = 120.0,
-) -> dict:
-    """Translate a batch. Returns::
-
-        {"translations": [str, ...],        # SAME order & length as `texts`
-         "detected_lang": str | None,       # aggregate: describes the FIRST item
-         "detected_confidence": float | None,  # detector's confidence, if given
-         "engine": str | None,
-         "engine_version": str | None}
-
-    Interpreter batch metadata is aggregate (first item only) — fine for one
-    post's ``[title, description]`` batch since they share a language. Passthrough
-    items (already target-language / emoji-only) come back UNCHANGED in their
-    slot. Raises InterpreterUnavailable on a connection error / 429 / 5xx (retry
-    later, honouring Retry-After), InterpreterBadRequest on 400. (Scribe note
-    #1347.)
-    """
-    if not texts:
-        return {"translations": [], "detected_lang": None,
-                "detected_confidence": None,
-                "engine": None, "engine_version": None}
-    try:
-        r = session.post(
-            _url(base_url, "/v1/translate"),
-            json={"q": list(texts), "source": source,
-                  "target": target, "engine": "auto"},
-            timeout=timeout,
-        )
-    except requests.RequestException as e:
-        raise InterpreterUnavailable(str(e)) from e
-    if r.status_code == 400:
-        raise InterpreterBadRequest((r.json() or {}).get("error", "bad request"))
-    # A gracefully-draining service — often behind a reverse proxy — returns 429
-    # or a 5xx gateway error (502/503/504), not always a clean 503. Treat every
-    # one as "unavailable, retry later" and honour any Retry-After it sends, so a
-    # rolling Interpreter restart interrupts the sweep cleanly (instead of raising
-    # an opaque HTTPError) and resumes exactly when the service says it's ready.
-    if r.status_code == 429 or r.status_code >= 500:
-        raise InterpreterUnavailable(
-            f"interpreter unavailable (HTTP {r.status_code})",
-            retry_after=_parse_retry_after(r),
-        )
-    r.raise_for_status()
-    body = r.json() or {}
-    out = body.get("translatedText")
-    # q was an array → translatedText is an array (same order & length). Guard a
-    # scalar reply just in case (shouldn't happen for an array q).
-    if not isinstance(out, list):
-        out = [out]
-    interp = body.get("interpreter") or {}
-    detected = body.get("detectedLanguage") or {}
-    return {
-        "translations": out,
-        "detected_lang": detected.get("language"),
-        "detected_confidence": detected.get("confidence"),
-        "engine": interp.get("engine"),
-        "engine_version": interp.get("engine_version"),
-    }

backend/app/services/ml/__init__.py

@@ -1,3 +1 @@
-"""ML pipeline services: embedders, heads (the learning suggester), suggestions,
-GPU-job queue + failure triage, CCIP characters, crops/regions, allowlist and
-aliases."""
+"""ML pipeline services: tagger, embedder, suggestions, centroids, allowlist, aliases."""

backend/app/services/ml/allowlist.py

@@ -1,31 +1,59 @@
-"""Suggestion actions: accept applies the canonical tag to an image (which
-feeds head training); dismiss / reject record a per-image rejection.
-
-(The Camie allowlist bulk-apply was retired #1189 — heads + CCIP are the tag
-source, and head auto-apply is the earned propagation. Accept no longer
-allowlists or fans a tag out across the library.)
+"""Allowlist semantics: accepting a suggestion adds the canonical tag to
+image_tag AND to tag_allowlist; per-image removal/dismiss writes a rejection.
 """
 
-from sqlalchemy import delete
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from sqlalchemy import and_, delete, distinct, func, or_, select
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from ...models import TagSuggestionRejection
+from ...models import (
+    ImagePrediction,
+    MLSettings,
+    Tag,
+    TagAlias,
+    TagAllowlist,
+    TagSuggestionRejection,
+)
 from ...models.tag import image_tag
+from .aliases import AliasService
+
+
+@dataclass(frozen=True)
+class AllowlistRow:
+    tag_id: int
+    tag_name: str
+    tag_kind: str
+    min_confidence: float
+    applied_count: int      # image_tag rows currently carrying this tag
+    coverage_count: int     # images a sweep WOULD cover at min_confidence
 
 
 class AllowlistService:
     def __init__(self, session: AsyncSession):
         self.session = session
+        self.aliases = AliasService(session)
 
     async def _apply_image_tag(self, image_id: int, tag_id: int, source: str):
         stmt = insert(image_tag).values(
             image_record_id=image_id, tag_id=tag_id, source=source
-        ).on_conflict_do_nothing(
+        )
+        stmt = stmt.on_conflict_do_nothing(
             index_elements=["image_record_id", "tag_id"]
         )
         await self.session.execute(stmt)
 
+    async def _add_to_allowlist(self, tag_id: int) -> bool:
+        """Returns True if newly added (caller should kick off retro-apply)."""
+        exists = await self.session.get(TagAllowlist, tag_id)
+        if exists is not None:
+            return False
+        self.session.add(TagAllowlist(tag_id=tag_id))
+        await self.session.flush()
+        return True
+
     async def _clear_rejection(self, image_id: int, tag_id: int):
         await self.session.execute(
             delete(TagSuggestionRejection)
@@ -33,16 +61,30 @@ class AllowlistService:
             .where(TagSuggestionRejection.tag_id == tag_id)
         )
 
-    async def accept(self, image_id: int, tag_id: int) -> None:
-        """Apply the accepted tag to this image (source='ml_accepted', a head
-        training positive) and clear any prior rejection."""
+    async def accept(self, image_id: int, tag_id: int) -> bool:
+        """Accept a suggestion. Returns True if the tag was newly added to
+        the allowlist (the API layer enqueues apply_allowlist_tags then)."""
         await self._apply_image_tag(image_id, tag_id, source="ml_accepted")
         await self._clear_rejection(image_id, tag_id)
+        return await self._add_to_allowlist(tag_id)
+
+    async def add_alias_and_accept(
+        self,
+        image_id: int,
+        alias_string: str,
+        alias_category: str,
+        canonical_tag_id: int,
+    ) -> bool:
+        await self.aliases.create(
+            alias_string, alias_category, canonical_tag_id
+        )
+        return await self.accept(image_id, canonical_tag_id)
 
     async def dismiss(self, image_id: int, tag_id: int) -> None:
         stmt = insert(TagSuggestionRejection).values(
             image_record_id=image_id, tag_id=tag_id
-        ).on_conflict_do_nothing(
+        )
+        stmt = stmt.on_conflict_do_nothing(
             index_elements=["image_record_id", "tag_id"]
         )
         await self.session.execute(stmt)
@@ -54,11 +96,118 @@ class AllowlistService:
         await self._clear_rejection(image_id, tag_id)
 
     async def reject_applied_tag(self, image_id: int, tag_id: int) -> None:
-        """Operator removed an applied tag from an image. Remove the image_tag
-        row AND record a rejection so head auto-apply won't re-apply it."""
+        """Operator removed an applied tag from an image. Remove the
+        image_tag row AND record a rejection so the allowlist won't
+        re-apply it on the next maintenance sweep."""
         await self.session.execute(
             image_tag.delete()
             .where(image_tag.c.image_record_id == image_id)
             .where(image_tag.c.tag_id == tag_id)
         )
         await self.dismiss(image_id, tag_id)
+
+    async def _store_floor(self) -> float:
+        return (
+            await self.session.execute(
+                select(MLSettings.tagger_store_floor).where(MLSettings.id == 1)
+            )
+        ).scalar_one()
+
+    async def update_threshold(
+        self, tag_id: int, min_confidence: float
+    ) -> None:
+        row = await self.session.get(TagAllowlist, tag_id)
+        if row is not None:
+            # An allowlist tag can't auto-apply more permissively than the
+            # ingest store floor — predictions below tagger_store_floor aren't
+            # stored, so a lower min_confidence would behave identically to the
+            # floor. Clamp so the stored threshold matches actual behavior
+            # (#764).
+            floor = await self._store_floor()
+            row.min_confidence = max(min_confidence, floor)
+
+    async def remove(self, tag_id: int) -> None:
+        await self.session.execute(
+            delete(TagAllowlist).where(TagAllowlist.tag_id == tag_id)
+        )
+
+    async def _coverage_match(self, tag: Tag):
+        """The predicate over image_prediction rows that resolve to `tag`,
+        mirroring tasks.ml._confidence_for_tag's resolution: a prediction whose
+        raw_name equals the tag name (any category), OR an alias maps
+        (raw_name, category) -> this tag. Returns a SQLAlchemy boolean clause.
+        """
+        alias_rows = (
+            await self.session.execute(
+                select(TagAlias.alias_string, TagAlias.alias_category).where(
+                    TagAlias.canonical_tag_id == tag.id
+                )
+            )
+        ).all()
+        name_clause = ImagePrediction.raw_name == tag.name
+        alias_clauses = [
+            and_(
+                ImagePrediction.raw_name == a,
+                ImagePrediction.category == c,
+            )
+            for a, c in alias_rows
+        ]
+        return or_(name_clause, *alias_clauses) if alias_clauses else name_clause
+
+    async def coverage(self, tag_id: int, threshold: float) -> int:
+        """How many distinct images a sweep WOULD cover for this tag at
+        `threshold`: images with a resolving prediction scoring >= threshold.
+        The gross candidate pool (NOT minus already-applied/rejected) — it's
+        the tuning signal for "lower the threshold and ~N more images qualify".
+        """
+        tag = await self.session.get(Tag, tag_id)
+        if tag is None:
+            return 0
+        match = await self._coverage_match(tag)
+        stmt = select(
+            func.count(distinct(ImagePrediction.image_record_id))
+        ).where(ImagePrediction.score >= threshold, match)
+        return (await self.session.execute(stmt)).scalar_one()
+
+    async def list_all(self) -> Sequence[AllowlistRow]:
+        stmt = (
+            select(
+                TagAllowlist.tag_id,
+                Tag.name,
+                Tag.kind,
+                TagAllowlist.min_confidence,
+            )
+            .join(Tag, Tag.id == TagAllowlist.tag_id)
+            .order_by(Tag.name.asc())
+        )
+        rows = (await self.session.execute(stmt)).all()
+        tag_ids = [r[0] for r in rows]
+
+        # Applied counts in ONE grouped query (vs N per-row counts).
+        applied: dict[int, int] = {}
+        if tag_ids:
+            applied = dict(
+                (
+                    await self.session.execute(
+                        select(image_tag.c.tag_id, func.count())
+                        .where(image_tag.c.tag_id.in_(tag_ids))
+                        .group_by(image_tag.c.tag_id)
+                    )
+                ).all()
+            )
+
+        result = []
+        for r in rows:
+            # Coverage is per-tag (alias set differs); allowlist is small.
+            cov = await self.coverage(r[0], r[3])
+            result.append(
+                AllowlistRow(
+                    tag_id=r[0],
+                    tag_name=r[1],
+                    tag_kind=r[2].value if hasattr(r[2], "value") else str(r[2]),
+                    min_confidence=r[3],
+                    applied_count=applied.get(r[0], 0),
+                    coverage_count=cov,
+                )
+            )
+        return result

backend/app/services/ml/ccip.py

@@ -13,124 +13,28 @@ exact CCIP difference metric/threshold gets validated against the model during
 the hands-on eval. numpy is imported lazily (API worker has it via pgvector).
 """
 
-from sqlalchemy import exists, func, select
+from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from ...models import (
-    CcipPrototypeState,
-    CharacterPrototype,
-    ImageRegion,
-    MLSettings,
-    Tag,
-    TagKind,
-    TagPositiveConfirmation,
-)
+from ...models import ImageRegion, Tag, TagKind
 from ...models.tag import image_tag
-from .training_data import _AUTO_SOURCES
 
-# Cosine-similarity floor to call a figure the same character. The live setting
-# (ml_settings.ccip_match_threshold) drives it; this is only the fallback when no
-# threshold is supplied AND no settings row exists.
-DEFAULT_SIM_THRESHOLD = 0.85
+# Cosine-similarity floor to call a figure the same character. Conservative
+# default; tune from real matches (CCIP same-char clusters tightly).
+DEFAULT_SIM_THRESHOLD = 0.75
 _FIGURE_KINDS = ("face", "figure")
 
 
-async def _settings_threshold(session: AsyncSession) -> float:
-    val = (
-        await session.execute(
-            select(MLSettings.ccip_match_threshold).where(MLSettings.id == 1)
-        )
-    ).scalar_one_or_none()
-    return float(val) if val is not None else DEFAULT_SIM_THRESHOLD
-
-
 def _l2norm(mat, np):
     n = np.linalg.norm(mat, axis=1, keepdims=True)
     n[n == 0] = 1.0
     return mat / n
 
 
-# Single-shot cache of the (expensive) reference load, keyed on a cheap
-# signature that changes exactly when references could: a character tag added/
-# removed (n_char_tags) or a figure embedded (max/ n of ccip regions). Shared by
-# the live matcher (every modal open) and the auto-apply sweep.
-_REF_CACHE: dict = {"sig": None, "refs": None}
-
-
-def _single_character_images():
-    """Subquery of image ids carrying EXACTLY ONE character tag. References come
-    only from these — on a multi-character image the tag is image-level, so every
-    figure would otherwise pollute each character's prototype set (a 2-character
-    image tagged 'Velma' would make Daphne's figure a Velma reference)."""
-    return (
-        select(image_tag.c.image_record_id)
-        .join(Tag, Tag.id == image_tag.c.tag_id)
-        .where(Tag.kind == TagKind.character)
-        .group_by(image_tag.c.image_record_id)
-        .having(func.count() == 1)
-    )
-
-
-def _hygiene_tagged_images():
-    """Subquery of image ids carrying any SYSTEM tag (wip / banner / editor
-    screenshot). Training hygiene (#128): such images never contribute
-    reference prototypes — a faceless wip's figure region would otherwise
-    become an identity reference for the character it's tagged with."""
-    return (
-        select(image_tag.c.image_record_id)
-        .join(Tag, Tag.id == image_tag.c.tag_id)
-        .where(Tag.is_system.is_(True))
-    )
-
-
-async def _ref_signature(session: AsyncSession) -> tuple:
-    n_tags = (
-        await session.execute(
-            select(func.count())
-            .select_from(image_tag)
-            .join(Tag, Tag.id == image_tag.c.tag_id)
-            .where(Tag.kind == TagKind.character)
-        )
-    ).scalar_one()
-    n_regs, max_id = (
-        await session.execute(
-            select(func.count(), func.max(ImageRegion.id)).where(
-                ImageRegion.kind.in_(_FIGURE_KINDS),
-                ImageRegion.ccip_embedding.is_not(None),
-            )
-        )
-    ).one()
-    # Hygiene applications must invalidate too: tagging an image `wip` changes
-    # the reference set without touching character-tag or region counts.
-    n_hygiene = (
-        await session.execute(
-            select(func.count())
-            .select_from(image_tag)
-            .join(Tag, Tag.id == image_tag.c.tag_id)
-            .where(Tag.is_system.is_(True))
-        )
-    ).scalar_one()
-    return (n_tags, n_regs, max_id, n_hygiene)
-
-
-def _positive_char_tag():
-    """Condition on the joined character image_tag: HUMAN-applied or operator-
-    confirmed — NOT an unconfirmed auto-apply. Keeps an auto-tagged character from
-    self-seeding CCIP references, so a ccip_auto misfire can't reinforce itself
-    (milestone 139) — mirrors the head-training positive exclusion."""
-    return image_tag.c.source.not_in(_AUTO_SOURCES) | exists().where(
-        TagPositiveConfirmation.image_record_id == image_tag.c.image_record_id,
-        TagPositiveConfirmation.tag_id == image_tag.c.tag_id,
-    )
-
-
 async def character_references(session: AsyncSession) -> dict[int, list]:
     """Per character-tag CCIP reference vectors: figure/face-region CCIP
-    embeddings on UNAMBIGUOUS (single-character) images carrying that tag.
-    Multi-prototype — several vectors per character. Cached on a cheap signature."""
-    sig = await _ref_signature(session)
-    if _REF_CACHE["sig"] == sig and _REF_CACHE["refs"] is not None:
-        return _REF_CACHE["refs"]
+    embeddings on images that carry that character tag (the operator's examples).
+    Multi-prototype — several vectors per character."""
     rows = (
         await session.execute(
             select(image_tag.c.tag_id, ImageRegion.ccip_embedding)
@@ -141,19 +45,13 @@ async def character_references(session: AsyncSession) -> dict[int, list]:
             )
             .join(Tag, Tag.id == image_tag.c.tag_id)
             .where(Tag.kind == TagKind.character)
-            .where(_positive_char_tag())
             .where(ImageRegion.kind.in_(_FIGURE_KINDS))
             .where(ImageRegion.ccip_embedding.is_not(None))
-            .where(ImageRegion.image_record_id.in_(_single_character_images()))
-            .where(
-                ImageRegion.image_record_id.not_in(_hygiene_tagged_images())
-            )
         )
     ).all()
     refs: dict[int, list] = {}
     for tag_id, vec in rows:
         refs.setdefault(tag_id, []).append(vec)
-    _REF_CACHE.update(sig=sig, refs=refs)
     return refs
 
 
@@ -169,97 +67,27 @@ async def _tag_names(session: AsyncSession, tag_ids: list[int]) -> dict[int, str
     )
 
 
-# Per-character normalized prototype matrices, cached per process and refreshed
-# INCREMENTALLY: only characters whose ccip_prototype_state.updated_at advanced
-# are reloaded. This replaces the request-path rebuild of the ENTIRE reference
-# blob (the ~4s stall, #1317) — the prototypes are precomputed off the request
-# path by services.ml.character_prototypes (a beat + after each retrain).
-_PROTO_CACHE: dict = {"mats": {}, "ver": {}}
-
-
-async def _load_prototypes(session: AsyncSession) -> dict:
-    """{tag_id: (P, D) L2-normalized prototype matrix} from character_prototype,
-    served from the in-process cache and reloading ONLY the characters whose
-    updated_at changed. Empty dict when the store isn't populated yet (cold start
-    → match_image falls back to the legacy on-the-fly reference build)."""
-    import numpy as np
-
-    versions = dict(
-        (
-            await session.execute(
-                select(CcipPrototypeState.tag_id, CcipPrototypeState.updated_at)
-            )
-        ).all()
-    )
-    mats = _PROTO_CACHE["mats"]
-    ver = _PROTO_CACHE["ver"]
-    # Forget characters that no longer have prototypes.
-    for tag_id in [t for t in mats if t not in versions]:
-        mats.pop(tag_id, None)
-        ver.pop(tag_id, None)
-    # Reload only the characters whose prototypes changed since we cached them.
-    stale = [t for t, u in versions.items() if ver.get(t) != u]
-    if stale:
-        rows = (
-            await session.execute(
-                select(
-                    CharacterPrototype.tag_id, CharacterPrototype.ccip_embedding
-                ).where(CharacterPrototype.tag_id.in_(stale))
-            )
-        ).all()
-        by_tag: dict[int, list] = {}
-        for tag_id, vec in rows:
-            by_tag.setdefault(tag_id, []).append(
-                np.asarray(vec, dtype=np.float32)
-            )
-        for tag_id in stale:
-            vecs = by_tag.get(tag_id)
-            if vecs:
-                mats[tag_id] = _l2norm(np.vstack(vecs), np)
-                ver[tag_id] = versions[tag_id]
-            else:
-                mats.pop(tag_id, None)
-                ver.pop(tag_id, None)
-    return mats
-
-
 async def match_image(
-    session: AsyncSession, image_id: int, threshold: float | None = None
+    session: AsyncSession, image_id: int, threshold: float = DEFAULT_SIM_THRESHOLD
 ) -> list[dict]:
     """Character suggestions for one image from its figure-region CCIP vectors:
     [{tag_id, name, category:'character', score, source:'ccip'}], ranked.
     Already-applied character tags are excluded. Empty if the image has no figure
-    CCIP vectors or no character references exist yet. threshold defaults to the
-    live ml_settings.ccip_match_threshold."""
+    CCIP vectors or no character references exist yet."""
     import numpy as np
 
-    if threshold is None:
-        threshold = await _settings_threshold(session)
-
-    # Keep each figure region's bbox alongside its vector so a match can point at
-    # the figure that matched (#1206 grounding), not just the score.
-    fig_rows = (
+    qvecs = (
         await session.execute(
-            select(
-                ImageRegion.ccip_embedding,
-                ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh,
-                ImageRegion.kind, ImageRegion.detector_version,
-            ).where(
+            select(ImageRegion.ccip_embedding).where(
                 ImageRegion.image_record_id == image_id,
                 ImageRegion.kind.in_(_FIGURE_KINDS),
                 ImageRegion.ccip_embedding.is_not(None),
             )
         )
-    ).all()
-    if not fig_rows:
+    ).scalars().all()
+    if not qvecs:
         return []
-    # Prefer the precomputed prototype store (fast, incremental). On a cold start
-    # (store not yet populated post-deploy) fall back to the legacy on-the-fly
-    # reference build so character suggestions work immediately — the background
-    # refresh populates the store within ~15 min, after which this path is used
-    # and the per-accept ~4s rebuild is gone (#1317).
-    protos = await _load_prototypes(session)
-    refs = protos if protos else await character_references(session)
+    refs = await character_references(session)
     if not refs:
         return []
     applied = set(
@@ -273,25 +101,13 @@ async def match_image(
     )
     names = await _tag_names(session, [t for t in refs if t not in applied])
 
-    qvecs = [r[0] for r in fig_rows]
-    fig_meta = [
-        {"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector}
-        for _v, rx, ry, rw, rh, kind, detector in fig_rows
-    ]
     Q = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in qvecs]), np)
     out = []
     for tag_id, vecs in refs.items():
         if tag_id in applied:
             continue
-        # Prototype matrices are already L2-normalized; legacy refs are raw
-        # vector lists that still need stacking + normalizing.
-        R = vecs if protos else _l2norm(
-            np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np
-        )
-        sims = Q @ R.T                       # (n_query_figures, n_references)
-        per_figure = sims.max(axis=1)        # best reference cosine per figure
-        best_figure = int(per_figure.argmax())
-        best = float(per_figure[best_figure])
+        R = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np)
+        best = float((Q @ R.T).max())   # best (query figure, reference) cosine
         if best >= threshold:
             out.append({
                 "tag_id": tag_id,
@@ -299,8 +115,6 @@ async def match_image(
                 "category": "character",
                 "score": round(best, 4),
                 "source": "ccip",
-                # the figure region that matched → grounds the character tag.
-                "grounding": fig_meta[best_figure],
             })
     out.sort(key=lambda d: d["score"], reverse=True)
     return out

backend/app/services/ml/centroids.py

@@ -0,0 +1,163 @@
+"""Tag centroids: the mean SigLIP embedding of a tag's member images.
+
+Powers centroid-augmented suggestions (a tag whose centroid is close to an
+image's embedding becomes a suggestion even if Camie didn't predict it).
+"""
+
+from dataclasses import dataclass
+
+import numpy as np
+from sqlalchemy import func, select
+from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ...models import (
+    ImageRecord,
+    MLSettings,
+    Tag,
+    TagKind,
+    TagReferenceEmbedding,
+)
+from ...models.tag import image_tag
+
+ELIGIBLE_KINDS = {
+    TagKind.character,
+    TagKind.fandom,
+    TagKind.general,
+    TagKind.series,
+}
+
+
+@dataclass(frozen=True)
+class CentroidHit:
+    tag_id: int
+    similarity: float
+
+
+class CentroidService:
+    def __init__(self, session: AsyncSession):
+        self.session = session
+
+    async def _min_reference_images(self) -> int:
+        return (
+            await self.session.execute(
+                select(MLSettings.min_reference_images).where(MLSettings.id == 1)
+            )
+        ).scalar_one()
+
+    async def _model_version(self) -> str:
+        """Audit 2026-06-02: SigLIP model-version stamp comes from the
+        DB row, not the env constant. tag_and_embed (tasks/ml.py:110)
+        already reads from MLSettings.embedder_model_version, so by
+        sourcing centroid stamps + drift checks from the same row, we
+        eliminate the silent-drift case the audit flagged. env
+        SIGLIP_MODEL_VERSION still drives which model embedder.py
+        loads at runtime; the version stamp is purely the operator-
+        controlled identifier."""
+        return (
+            await self.session.execute(
+                select(MLSettings.embedder_model_version).where(MLSettings.id == 1)
+            )
+        ).scalar_one()
+
+    async def recompute_for_tag(self, tag_id: int) -> bool:
+        """Recompute one tag's centroid. Returns True if a centroid was
+        written, False if skipped (ineligible kind or too few members)."""
+        tag = await self.session.get(Tag, tag_id)
+        if tag is None or tag.kind not in ELIGIBLE_KINDS:
+            return False
+
+        min_refs = await self._min_reference_images()
+
+        stmt = (
+            select(ImageRecord.siglip_embedding)
+            .join(image_tag, image_tag.c.image_record_id == ImageRecord.id)
+            .where(image_tag.c.tag_id == tag_id)
+            .where(ImageRecord.siglip_embedding.is_not(None))
+        )
+        embeddings = [
+            np.array(e, dtype=np.float32)
+            for e in (await self.session.execute(stmt)).scalars().all()
+        ]
+        if len(embeddings) < min_refs:
+            return False
+
+        centroid = np.mean(np.stack(embeddings), axis=0).astype(np.float32)
+        model_version = await self._model_version()
+
+        stmt = insert(TagReferenceEmbedding).values(
+            tag_id=tag_id,
+            embedding=centroid.tolist(),
+            reference_count=len(embeddings),
+            model_version=model_version,
+        )
+        stmt = stmt.on_conflict_do_update(
+            index_elements=["tag_id"],
+            set_={
+                "embedding": centroid.tolist(),
+                "reference_count": len(embeddings),
+                "model_version": model_version,
+                "updated_at": func.now(),
+            },
+        )
+        await self.session.execute(stmt)
+        return True
+
+    async def list_drifted(self) -> list[int]:
+        """Tag ids whose centroid is stale: member count != reference_count,
+        OR no centroid row, OR centroid built on a different SigLIP version.
+        Only considers eligible-kind tags with embeddings present."""
+        current_model_version = await self._model_version()
+        member_counts = (
+            select(
+                image_tag.c.tag_id.label("tag_id"),
+                func.count(image_tag.c.image_record_id).label("members"),
+            )
+            .join(ImageRecord, ImageRecord.id == image_tag.c.image_record_id)
+            .where(ImageRecord.siglip_embedding.is_not(None))
+            .group_by(image_tag.c.tag_id)
+            .subquery()
+        )
+        stmt = (
+            select(Tag.id)
+            .join(member_counts, member_counts.c.tag_id == Tag.id)
+            .outerjoin(
+                TagReferenceEmbedding,
+                TagReferenceEmbedding.tag_id == Tag.id,
+            )
+            .where(Tag.kind.in_(ELIGIBLE_KINDS))
+            .where(
+                (TagReferenceEmbedding.tag_id.is_(None))
+                | (
+                    TagReferenceEmbedding.reference_count
+                    != member_counts.c.members
+                )
+                | (TagReferenceEmbedding.model_version != current_model_version)
+            )
+        )
+        return list((await self.session.execute(stmt)).scalars().all())
+
+    async def find_similar_tags(
+        self, image_id: int, limit: int = 20
+    ) -> list[CentroidHit]:
+        """Cosine similarity between an image's embedding and stored
+        centroids. Returns top-`limit` by similarity DESC. pgvector's
+        cosine_distance gives 1 - cosine_similarity."""
+        img = await self.session.get(ImageRecord, image_id)
+        if img is None or img.siglip_embedding is None:
+            return []
+        emb = img.siglip_embedding
+        distance = TagReferenceEmbedding.embedding.cosine_distance(emb)
+        stmt = (
+            select(
+                TagReferenceEmbedding.tag_id,
+                (1 - distance).label("similarity"),
+            )
+            .order_by(distance.asc())
+            .limit(limit)
+        )
+        rows = (await self.session.execute(stmt)).all()
+        return [
+            CentroidHit(tag_id=r.tag_id, similarity=float(r.similarity))
+            for r in rows
+        ]

backend/app/services/ml/character_prototypes.py

@@ -1,262 +0,0 @@
-"""Precomputed CCIP character prototypes — incremental builder (#1317, m138).
-
-Turns the per-character CCIP reference set into a precomputed artifact so the
-matcher never rebuilds it on the request path. Sync (runs in the celery ml
-worker via tasks.ml.refresh_character_prototypes); the async matcher only READS
-the character_prototype table.
-
-`refresh_character_prototypes`:
-  1. Cheap GLOBAL gate — a few COUNTs (`_global_signature`). Unchanged + not
-     `full` → no-op (nothing that affects references changed since last refresh).
-  2. Per-character fingerprint diff (one GROUP BY) → rebuild ONLY the characters
-     whose references changed (or are new); drop characters that lost all refs.
-Each rebuild loads just ONE character's reference vectors, caps them to
-MLSettings.ccip_prototype_cap, and replaces that character's prototype rows — so
-cost scales with WHAT changed, not with the library size.
-
-The reference PREDICATE (single-character, non-hygiene, figure CCIP) is imported
-from ccip so the prototypes match exactly what the legacy matcher selected.
-"""
-
-import random
-from datetime import UTC, datetime
-
-from sqlalchemy import delete, func, select
-from sqlalchemy.orm import Session
-
-from ...models import (
-    CcipPrototypeState,
-    CharacterPrototype,
-    ImageRegion,
-    MLSettings,
-    Tag,
-    TagKind,
-    TagPositiveConfirmation,
-)
-from ...models.tag import image_tag
-from .ccip import (
-    _FIGURE_KINDS,
-    _hygiene_tagged_images,
-    _l2norm,
-    _positive_char_tag,
-    _single_character_images,
-)
-
-# Deterministic per-tag capping so a rebuild of an UNCHANGED reference set
-# resamples identically (stable prototypes, no churn between refreshes).
-_SAMPLE_SEED = 1317
-
-
-def _global_signature(session: Session) -> str:
-    """Cheap 'could any references have changed' gate: character-tag count,
-    figure-CCIP region count + max id, hygiene-tag count. A few COUNTs — the same
-    quantities the legacy per-request signature used, now computed once per
-    refresh instead of on every /suggestions call."""
-    n_tags = session.execute(
-        select(func.count())
-        .select_from(image_tag)
-        .join(Tag, Tag.id == image_tag.c.tag_id)
-        .where(Tag.kind == TagKind.character)
-    ).scalar_one()
-    n_regs, max_id = session.execute(
-        select(func.count(), func.max(ImageRegion.id)).where(
-            ImageRegion.kind.in_(_FIGURE_KINDS),
-            ImageRegion.ccip_embedding.is_not(None),
-        )
-    ).one()
-    n_hygiene = session.execute(
-        select(func.count())
-        .select_from(image_tag)
-        .join(Tag, Tag.id == image_tag.c.tag_id)
-        .where(Tag.is_system.is_(True))
-    ).scalar_one()
-    # Character confirmations affect the reference set now that auto-tags only
-    # seed references once confirmed (milestone 139) — so a confirm must trip the
-    # gate, or the per-character diff (which reflects it) never runs.
-    n_conf = session.execute(
-        select(func.count())
-        .select_from(TagPositiveConfirmation)
-        .join(Tag, Tag.id == TagPositiveConfirmation.tag_id)
-        .where(Tag.kind == TagKind.character)
-    ).scalar_one()
-    return f"{n_tags}:{n_regs}:{max_id or 0}:{n_hygiene}:{n_conf}"
-
-
-def _current_fingerprints(session: Session) -> dict[int, str]:
-    """Per-character (reference count, max reference region id) over the SAME
-    predicate the matcher's references use. One GROUP BY → the change detector:
-    a character whose fingerprint moved (gained/lost a reference) needs a
-    rebuild; everyone else is left untouched."""
-    rows = session.execute(
-        select(
-            image_tag.c.tag_id,
-            func.count(ImageRegion.id),
-            func.max(ImageRegion.id),
-        )
-        .select_from(ImageRegion)
-        .join(
-            image_tag,
-            image_tag.c.image_record_id == ImageRegion.image_record_id,
-        )
-        .join(Tag, Tag.id == image_tag.c.tag_id)
-        .where(Tag.kind == TagKind.character)
-        .where(_positive_char_tag())
-        .where(ImageRegion.kind.in_(_FIGURE_KINDS))
-        .where(ImageRegion.ccip_embedding.is_not(None))
-        .where(ImageRegion.image_record_id.in_(_single_character_images()))
-        .where(ImageRegion.image_record_id.not_in(_hygiene_tagged_images()))
-        .group_by(image_tag.c.tag_id)
-    ).all()
-    return {tag_id: f"{cnt}:{mx}" for tag_id, cnt, mx in rows}
-
-
-def _rebuild_one(session: Session, tag_id: int, cap: int) -> int:
-    """Replace ONE character's prototype rows from its current references, capped
-    to `cap`. Loads only this character's vectors (bounded by its popularity)."""
-    rows = session.execute(
-        select(ImageRegion.id, ImageRegion.ccip_embedding)
-        .select_from(ImageRegion)
-        .join(
-            image_tag,
-            image_tag.c.image_record_id == ImageRegion.image_record_id,
-        )
-        .where(image_tag.c.tag_id == tag_id)
-        .where(_positive_char_tag())
-        .where(ImageRegion.kind.in_(_FIGURE_KINDS))
-        .where(ImageRegion.ccip_embedding.is_not(None))
-        .where(ImageRegion.image_record_id.in_(_single_character_images()))
-        .where(ImageRegion.image_record_id.not_in(_hygiene_tagged_images()))
-    ).all()
-    # Cap for bounded MATCH cost. A random sample (not most-recent) keeps the
-    # prototypes representative of the whole reference set; a fixed per-tag seed
-    # makes an unchanged set resample identically.
-    if cap > 0 and len(rows) > cap:
-        rows = random.Random(f"{_SAMPLE_SEED}:{tag_id}").sample(rows, cap)
-    session.execute(
-        delete(CharacterPrototype).where(CharacterPrototype.tag_id == tag_id)
-    )
-    for region_id, vec in rows:
-        session.add(
-            CharacterPrototype(
-                tag_id=tag_id, region_id=region_id, ccip_embedding=vec
-            )
-        )
-    return len(rows)
-
-
-def refresh_character_prototypes(
-    session: Session, *, full: bool = False
-) -> dict[str, int | bool]:
-    """Incrementally refresh the prototype store. `full=True` rebuilds every
-    character regardless of the gate/fingerprints (nightly reconcile). Returns
-    {skipped, rebuilt, removed}; commits."""
-    settings = MLSettings.load_sync(session)
-    sig = _global_signature(session)
-    if not full and settings.ccip_ref_signature == sig:
-        return {"skipped": True, "rebuilt": 0, "removed": 0}
-
-    cap = settings.ccip_prototype_cap
-    current = _current_fingerprints(session)
-    stored = dict(
-        session.execute(
-            select(CcipPrototypeState.tag_id, CcipPrototypeState.fingerprint)
-        ).all()
-    )
-    now = datetime.now(UTC)
-    rebuilt = 0
-    for tag_id, fp in current.items():
-        if full or stored.get(tag_id) != fp:
-            _rebuild_one(session, tag_id, cap)
-            state = session.get(CcipPrototypeState, tag_id)
-            if state is None:
-                state = CcipPrototypeState(tag_id=tag_id)
-                session.add(state)
-            state.fingerprint = fp
-            state.updated_at = now
-            rebuilt += 1
-    # Characters that lost every reference (refs removed / re-kinded / image now
-    # multi-character) → drop their prototypes + state so they stop matching.
-    removed = 0
-    for tag_id in set(stored) - set(current):
-        session.execute(
-            delete(CharacterPrototype).where(CharacterPrototype.tag_id == tag_id)
-        )
-        session.execute(
-            delete(CcipPrototypeState).where(CcipPrototypeState.tag_id == tag_id)
-        )
-        removed += 1
-
-    settings.ccip_ref_signature = sig
-    session.commit()
-    return {"skipped": False, "rebuilt": rebuilt, "removed": removed}
-
-
-def retract_auto_applied_ccip(session: Session) -> int:
-    """Soft auto-apply for CCIP character tags (milestone 139): re-score every
-    standing source='ccip_auto' character tag against that character's prototypes
-    and REMOVE the ones whose best figure match is now BELOW
-    ccip_auto_apply_threshold. Skips operator-confirmed tags. SILENT — a low score
-    isn't proof the tag was wrong (that's reserved for an operator removal). No-op
-    unless ccip_auto_apply_enabled. A character with no prototypes yet, or an image
-    with no figure vectors, is left alone (can't judge → keep). Returns
-    n_retracted."""
-    import numpy as np
-
-    settings = MLSettings.load_sync(session)
-    if not settings.ccip_auto_apply_enabled:
-        return 0
-    thr = float(settings.ccip_auto_apply_threshold)
-    pairs = session.execute(
-        select(image_tag.c.image_record_id, image_tag.c.tag_id)
-        .where(image_tag.c.source == "ccip_auto")
-    ).all()
-    if not pairs:
-        return 0
-    confirmed = {
-        (iid, tid) for iid, tid in session.execute(
-            select(
-                TagPositiveConfirmation.image_record_id,
-                TagPositiveConfirmation.tag_id,
-            )
-        ).all()
-    }
-    # Each involved character's normalized prototype matrix, loaded once.
-    proto: dict[int, object] = {}
-    for tid in {tid for _iid, tid in pairs}:
-        vecs = [
-            v for (v,) in session.execute(
-                select(CharacterPrototype.ccip_embedding)
-                .where(CharacterPrototype.tag_id == tid)
-            )
-        ]
-        if vecs:
-            proto[tid] = _l2norm(
-                np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np
-            )
-    retracted = 0
-    for iid, tid in pairs:
-        if (iid, tid) in confirmed or tid not in proto:
-            continue                                   # confirmed / no prototypes
-        qvecs = [
-            v for (v,) in session.execute(
-                select(ImageRegion.ccip_embedding)
-                .where(ImageRegion.image_record_id == iid)
-                .where(ImageRegion.kind.in_(_FIGURE_KINDS))
-                .where(ImageRegion.ccip_embedding.is_not(None))
-            )
-        ]
-        if not qvecs:
-            continue                                   # no figure vectors → keep
-        Q = _l2norm(
-            np.vstack([np.asarray(v, dtype=np.float32) for v in qvecs]), np
-        )
-        if float((Q @ proto[tid].T).max()) < thr:
-            session.execute(
-                image_tag.delete()
-                .where(image_tag.c.image_record_id == iid)
-                .where(image_tag.c.tag_id == tid)
-                .where(image_tag.c.source == "ccip_auto")
-            )
-            retracted += 1
-    session.commit()
-    return retracted

backend/app/services/ml/embedder.py

@@ -18,11 +18,9 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True
 # N_replicas × this within the cores allotted to ML to avoid oversubscription.
 _INTRA_OP_THREADS = 4
 
-DEFAULT_MODEL_NAME = os.environ.get(
+MODEL_NAME = os.environ.get(
     "SIGLIP_MODEL_NAME", "google/siglip-so400m-patch14-384"
 )
-# Back-compat alias (api/gpu imported this name as the fallback embedder id).
-MODEL_NAME = DEFAULT_MODEL_NAME
 MODEL_VERSION = os.environ.get(
     "SIGLIP_MODEL_VERSION", "siglip-so400m-patch14-384"
 )
@@ -31,42 +29,35 @@ _LOCAL_DIR = Path(os.environ.get("ML_MODEL_DIR", "/models")) / "siglip"
 
 
 class Embedder:
-    """Loads whatever SigLIP-family model it's given by HF NAME. For the default
-    model it prefers the pre-downloaded local dir (no re-download on existing
-    deploys); any other name resolves as an HF repo id (downloaded + cached on
-    first use), so an operator model swap (#1190) just works server-side."""
-
-    def __init__(self, model_name: str | None = None, model_dir: Path | None = None):
-        self.model_name = model_name or DEFAULT_MODEL_NAME
-        self._explicit_dir = model_dir
+    def __init__(self, model_dir: Path | None = None):
+        self._model_dir = model_dir or _LOCAL_DIR
         self._model = None
         self._processor = None
         self._torch = None
 
-    def _source(self) -> str:
-        if self._explicit_dir is not None:
-            return str(self._explicit_dir)
-        if self.model_name == DEFAULT_MODEL_NAME and _LOCAL_DIR.exists():
-            return str(_LOCAL_DIR)
-        return self.model_name
-
     def load(self) -> None:
         if self._model is not None:
             return
         import torch
-        from transformers import AutoImageProcessor, AutoModel
+        from transformers import AutoModel, SiglipImageProcessor
 
         self._torch = torch
         # Bound torch's CPU thread pool (see _INTRA_OP_THREADS) so each replica
         # stays a predictable core consumer on a shared node.
         torch.set_num_threads(_INTRA_OP_THREADS)
-        # IMAGE inference only — AutoImageProcessor loads just the image side
-        # (preprocessor_config.json), skipping the SigLIP tokenizer + its
-        # sentencepiece dep (operator hit that ImportError 2026-05-25). Works
-        # for any SigLIP-family model, keeping the embedder model-agnostic.
-        src = self._source()
-        self._processor = AutoImageProcessor.from_pretrained(src)
-        self._model = AutoModel.from_pretrained(src)
+        # FC's embedder only does IMAGE inference — never text. AutoProcessor
+        # loads the full processor including SiglipTokenizer, which requires
+        # the sentencepiece library at import time even if we never call it.
+        # SiglipImageProcessor loads ONLY preprocessor_config.json (image
+        # side) and skips the tokenizer config entirely. Operator hit the
+        # ImportError 2026-05-25 once the ml-worker started actually running
+        # tag_and_embed; switching to the image-only loader avoids the
+        # tokenizer dep without adding ~30 MB of unused C++ build to the
+        # lean ml-worker image.
+        self._processor = SiglipImageProcessor.from_pretrained(
+            str(self._model_dir)
+        )
+        self._model = AutoModel.from_pretrained(str(self._model_dir))
         self._model.eval()
 
     def infer(self, image_path: Path) -> np.ndarray:
@@ -83,12 +74,8 @@ class Embedder:
 _default_embedder: Embedder | None = None
 
 
-def get_embedder(model_name: str | None = None) -> Embedder:
-    """Cached embedder for `model_name` (default if None). Rebuilds the singleton
-    when the requested name changes, so an operator model swap takes effect
-    without restarting the worker."""
+def get_embedder() -> Embedder:
     global _default_embedder
-    name = model_name or DEFAULT_MODEL_NAME
-    if _default_embedder is None or _default_embedder.model_name != name:
-        _default_embedder = Embedder(model_name=name)
+    if _default_embedder is None:
+        _default_embedder = Embedder()
     return _default_embedder

backend/app/services/ml/gpu_jobs.py

@@ -12,9 +12,8 @@ and the lease itself reclaims expired leases as a final backstop. Result-writing
 
 from datetime import UTC, datetime, timedelta
 
-from sqlalchemy import and_, delete, exists, func, or_, select, update
+from sqlalchemy import and_, or_, select, update
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import aliased
 
 from ...models import GpuJob
 
@@ -25,107 +24,6 @@ DEFAULT_LEASE_TTL = 180     # seconds an agent holds a job before it can be re-l
 DEFAULT_BATCH = 8
 MAX_ATTEMPTS = 3
 
-# Poison-loop backstops. `attempts` counts LEASES GRANTED (incremented in
-# lease()), but fail()'s MAX_ATTEMPTS cap only fires when the agent reports a
-# failure — a job that keeps coming back via release() (transient handback) or
-# lease expiry (agent crash/wedge) never gets a verdict and would cycle forever.
-# The orphan sweep converts those to 'error': an expired lease that has already
-# been granted EXPIRED_POISON_CAP leases is presumed to kill/wedge the agent,
-# and a pending job granted PENDING_POISON_CAP leases without ever completing is
-# presumed poisoned (e.g. a transfer that stalls every time). Both stay
-# resurrectable via /retry_errors, which resets attempts.
-EXPIRED_POISON_CAP = MAX_ATTEMPTS + 2
-PENDING_POISON_CAP = 10
-
-
-def error_dedupe_statements():
-    """DELETEs enforcing: at most ONE error row per (image, task), and none that
-    a live or succeeded row makes moot. The 2026-07-02 tombstone loop (backfill
-    skip-lists lacked 'error') minted a duplicate error row per bad file per
-    hour; running these before every backfill and inside /retry_errors keeps the
-    error count reading as "distinct failing files" and stops a retry fanning
-    one file out into several duplicate pending jobs. Shared by the sync beat
-    task and the async API route so both prune by the SAME predicate.
-    Execution order matters: moot rows first, then older duplicates (the newest
-    error — the freshest reason — survives)."""
-    other = aliased(GpuJob)
-    same_pair = and_(
-        other.image_record_id == GpuJob.image_record_id,
-        other.task == GpuJob.task,
-    )
-    moot = (
-        delete(GpuJob)
-        .where(
-            GpuJob.status == "error",
-            exists().where(
-                same_pair, other.status.in_(["pending", "leased", "done"]),
-            ),
-        )
-        .execution_options(synchronize_session=False)
-    )
-    older_dupe = (
-        delete(GpuJob)
-        .where(
-            GpuJob.status == "error",
-            exists().where(
-                same_pair,
-                other.status == "error",
-                or_(
-                    other.updated_at > GpuJob.updated_at,
-                    and_(other.updated_at == GpuJob.updated_at,
-                         other.id > GpuJob.id),
-                ),
-            ),
-        )
-        .execution_options(synchronize_session=False)
-    )
-    return [moot, older_dupe]
-
-
-def recover_statements(now: datetime) -> dict:
-    """UPDATEs for the orphan sweep, keyed by outcome; insertion order IS the
-    required execution order ('recovered' must run after 'poison_expired', which
-    claims the crash-loopers out of the same expired-lease pool)."""
-    expired = and_(GpuJob.status == "leased", GpuJob.lease_expires_at < now)
-    unlease = {"lease_token": None, "leased_at": None, "lease_expires_at": None,
-               "updated_at": now}
-    return {
-        "poison_expired": (
-            update(GpuJob)
-            .where(expired, GpuJob.attempts >= EXPIRED_POISON_CAP)
-            .values(
-                status="error",
-                # Keep the job's last stored failure reason — it's the triage
-                # signal for WHY the loop happened.
-                error=func.concat(
-                    f"poisoned: lease expired after {EXPIRED_POISON_CAP}+ lease "
-                    "attempts (job repeatedly crashes or wedges the agent?); "
-                    "last error: ",
-                    func.coalesce(GpuJob.error, "none"),
-                ),
-                **unlease,
-            )
-        ),
-        "recovered": update(GpuJob).where(expired).values(
-            status="pending", **unlease,
-        ),
-        "poison_pending": (
-            update(GpuJob)
-            .where(GpuJob.status == "pending",
-                   GpuJob.attempts >= PENDING_POISON_CAP)
-            .values(
-                status="error",
-                error=func.concat(
-                    f"poisoned: {PENDING_POISON_CAP}+ lease attempts without "
-                    "ever completing (transfer stalls every time?); "
-                    "last error: ",
-                    func.coalesce(GpuJob.error, "none"),
-                ),
-                updated_at=now,
-            )
-        ),
-    }
-
 
 class GpuJobService:
     def __init__(self, session: AsyncSession):
@@ -153,33 +51,25 @@ class GpuJobService:
     async def lease(
         self, token: str, batch_size: int = DEFAULT_BATCH, ttl: int = DEFAULT_LEASE_TTL
     ) -> list[GpuJob]:
-        """Claim up to batch_size pending (or expired-leased) jobs for `token`.
-
-        Two phases so each hits a partial index (0070) and stays O(batch) no
-        matter how many done/error rows have accumulated: the pending pool is the
-        hot path; expired leases are reclaimed only when pending can't fill the
-        batch (a crashed agent's work — rare). The old single OR-query walked the
-        primary key past the whole done-prefix in id order → O(done), which is
-        why leasing crawled — and the DB saturated — as the run progressed."""
+        """Claim up to batch_size pending (or expired-leased) jobs for `token`."""
         now = datetime.now(UTC)
-
-        async def _claim(condition, limit: int) -> list[int]:
-            return list(
-                (
-                    await self.session.execute(
-                        select(GpuJob.id).where(condition)
-                        .order_by(GpuJob.id).limit(limit)
-                        .with_for_update(skip_locked=True)
+        picked = (
+            await self.session.execute(
+                select(GpuJob.id)
+                .where(
+                    or_(
+                        GpuJob.status == "pending",
+                        and_(
+                            GpuJob.status == "leased",
+                            GpuJob.lease_expires_at < now,
+                        ),
                     )
-                ).scalars().all()
-            )
-
-        picked = await _claim(GpuJob.status == "pending", batch_size)
-        if len(picked) < batch_size:     # pending exhausted → reclaim expired leases
-            picked += await _claim(
-                and_(GpuJob.status == "leased", GpuJob.lease_expires_at < now),
-                batch_size - len(picked),
+                )
+                .order_by(GpuJob.id)
+                .limit(batch_size)
+                .with_for_update(skip_locked=True)
             )
+        ).scalars().all()
         if not picked:
             return []
         await self.session.execute(
@@ -272,11 +162,16 @@ class GpuJobService:
 
     async def recover_orphaned(self) -> int:
         """Reset every expired lease back to pending — catches agents that died
-        mid-job (no graceful release) — and convert poison-loopers to 'error'
-        (see the *_POISON_CAP rationale above). Run on a short beat so the queue
-        recovers + reads honestly even when no worker is actively leasing.
-        Returns rows recovered to pending (poison conversions are extra)."""
-        counts = {}
-        for name, stmt in recover_statements(datetime.now(UTC)).items():
-            counts[name] = (await self.session.execute(stmt)).rowcount or 0
-        return counts["recovered"]
+        mid-job (no graceful release). Run on a short beat so the queue recovers
+        + reads honestly even when no worker is actively leasing. Returns rows
+        recovered."""
+        now = datetime.now(UTC)
+        res = await self.session.execute(
+            update(GpuJob)
+            .where(GpuJob.status == "leased", GpuJob.lease_expires_at < now)
+            .values(
+                status="pending", lease_token=None, leased_at=None,
+                lease_expires_at=None, updated_at=now,
+            )
+        )
+        return res.rowcount or 0

backend/app/services/ml/gpu_triage.py

@@ -1,179 +0,0 @@
-"""GPU-failure triage (#125): classify errored jobs, PROBE the file, recover.
-
-An errored GPU job is a tombstone with a stored reason, but the reason alone is
-a suspicion, not a verdict — a timeout can hit a perfectly fine file, and
-"moov atom not found" can mean a truncated download OR a one-off transfer
-fault. So triage EVALUATES: it runs the real integrity probe (sha256 recompute
-+ PIL/ffprobe — verify_integrity's own machinery) on each errored image ONCE
-and records both verdicts:
-
-  ImageRecord.integrity_status  <- file-level verdict (ok / corrupt / ...)
-  GpuJob.triage_status          <- 'defect'  (file is bad: recovery material,
-                                              excluded from /retry_errors)
-                                   'file_ok' (file passes: the failure was
-                                              operational, safe to retry)
-
-Recovery reuses established primitives: delete the defective copy + record
-(cleanup_service.delete_images — full cascade) and re-poll the image's
-subscription Source (the Layer-2 refetch pattern: gallery-dl re-fetches the
-now-absent file on the next source check). Images without a pollable Source
-report 'no_source' — manual remediation. Every classification is logged at
-WARNING so the operator notices in Logs / System Activity.
-"""
-import logging
-import time
-from datetime import UTC, datetime
-from pathlib import Path
-
-from sqlalchemy import select, update
-from sqlalchemy.orm import Session
-
-from ...models import GpuJob, ImageProvenance, ImageRecord, Source
-from ..cleanup_service import delete_images
-
-log = logging.getLogger(__name__)
-
-# Reason buckets for the triage overview (reporting only — the PROBE decides
-# 'defect', never the string). Ordered: first match wins.
-_REASON_BUCKETS = (
-    ("poisoned", ("poisoned:",)),
-    ("transient", ("gave up after repeated transient", "curator unreachable",
-                   "connection", "read timed out")),
-    ("timeout", ("timed out", "timeout")),
-    ("truncated_or_corrupt", ("moov atom", "invalid data", "end of file",
-                              "header missing", "error reading header",
-                              "truncated", "premature", "corrupt",
-                              "no frames sampled")),
-    ("decode", ("cannot identify", "decompression", "broken data stream",
-                "unrecognized data")),
-)
-
-
-def classify_reason(error: str | None) -> str:
-    """Bucket a stored job-error string for the overview table."""
-    text = (error or "").lower()
-    if not text:
-        return "other"
-    for bucket, needles in _REASON_BUCKETS:
-        if any(n in text for n in needles):
-            return bucket
-    return "other"
-
-
-def triage_errored_jobs(
-    session: Session, *, time_budget_seconds: float = 300.0,
-) -> dict:
-    """Probe every not-yet-triaged errored image and write both verdicts.
-
-    Time-boxed (sha256 of a large original over NFS can take tens of seconds)
-    and inherently resumable: rows are selected by `triage_status IS NULL`, so
-    the next sweep continues exactly where a budget cut stopped. Commits per
-    image so a mid-run crash keeps completed verdicts."""
-    image_ids = session.execute(
-        select(GpuJob.image_record_id)
-        .where(GpuJob.status == "error", GpuJob.triage_status.is_(None))
-        .group_by(GpuJob.image_record_id)
-        .order_by(GpuJob.image_record_id)
-    ).scalars().all()
-    counts = {"probed": 0, "defect": 0, "file_ok": 0, "partial": False}
-    if not image_ids:
-        return counts
-    # Lazy imports: the probe helper lives in the maintenance task module and
-    # the hasher in the importer — importing either at module load would pull
-    # celery into every service consumer.
-    from ...tasks.maintenance import _verify_one
-    from ..importer import _sha256_of
-
-    started = time.monotonic()
-    for image_id in image_ids:
-        if time.monotonic() - started > time_budget_seconds:
-            counts["partial"] = True
-            break
-        rec = session.get(ImageRecord, image_id)
-        if rec is None:      # record deleted since the job errored
-            continue
-        verdict = _verify_one(Path(rec.path), rec.sha256, rec.mime, _sha256_of)
-        # 'ok' means the failure was operational; anything else (corrupt /
-        # failed_verification = missing/unreadable) makes the file itself the
-        # problem — recovery material.
-        triage = "file_ok" if verdict == "ok" else "defect"
-        reason = session.execute(
-            select(GpuJob.error)
-            .where(GpuJob.image_record_id == image_id, GpuJob.status == "error")
-            .limit(1)
-        ).scalar_one_or_none()
-        rec.integrity_status = verdict
-        session.execute(
-            update(GpuJob)
-            .where(GpuJob.image_record_id == image_id, GpuJob.status == "error")
-            .values(triage_status=triage, updated_at=datetime.now(UTC))
-        )
-        session.commit()
-        counts["probed"] += 1
-        counts[triage] += 1
-        log.warning(
-            "gpu triage: image %s (%s) job error %r -> integrity probe %r -> %s",
-            image_id, rec.path, (reason or "")[:120], verdict, triage,
-        )
-    return counts
-
-
-def recover_defective_image(
-    session: Session, image_id: int, *, images_root: Path,
-) -> dict:
-    """Delete the defective copy + record and queue its re-fetch — surgically
-    where possible.
-
-    Two re-fetch layers (operator 2026-07-03: deep back-catalogue items are
-    NEVER re-walked by the normal cadence, so recovery can't rely on it):
-    1. SURGICAL: any ExternalLink rows on the image's post(s) are reset +
-       re-dispatched — this is how external-host files (the common defect
-       case: big videos) come back regardless of post age. Sha-dedupe at
-       import discards payload files that still exist.
-    2. BROAD: a source re-check, which re-fetches gallery-dl-NATIVE files the
-       walk still reaches (recent posts). A native file deeper than the walk
-       needs a per-source backfill/deep scan — reported via links_reset=0 so
-       the caller can say so.
-
-    The record delete cascades the error tombstones with it. 'no_source' when
-    no enabled, real-URL Source resolves via provenance — manual there."""
-    rec = session.get(ImageRecord, image_id)
-    if rec is None:
-        return {"status": "not_found"}
-    src_id = session.execute(
-        select(Source.id)
-        .join(ImageProvenance, ImageProvenance.source_id == Source.id)
-        .where(
-            ImageProvenance.image_record_id == image_id,
-            Source.enabled.is_(True),
-            ~Source.url.like("sidecar:%"),   # synthetic anchor — not pollable
-        )
-        .order_by(Source.id.asc())
-    ).scalars().first()
-    if src_id is None:
-        return {"status": "no_source"}
-    # Capture the post linkage BEFORE the delete cascades provenance away.
-    post_ids = session.execute(
-        select(ImageProvenance.post_id)
-        .where(ImageProvenance.image_record_id == image_id)
-    ).scalars().all()
-    path = rec.path
-    summary = delete_images(session, image_ids=[image_id], images_root=images_root)
-    from ..external_links import refetch_links_for_post
-
-    links_reset = 0
-    for pid in post_ids:
-        links_reset += refetch_links_for_post(session, pid)["links_reset"]
-    # Lazy import (services -> tasks would cycle at module load).
-    from ...tasks.download import download_source
-
-    download_source.delay(src_id)
-    log.warning(
-        "gpu triage recovery: deleted defective image %s (%s); reset %d "
-        "external link(s) and queued a re-check of source %s",
-        image_id, path, links_reset, src_id,
-    )
-    return {
-        "status": "refetch_queued", "source_id": src_id,
-        "links_reset": links_reset, **summary,
-    }