Merge pull request 'Release: GPU agent autoscaler (throughput hill-climb, Auto default-on)' (#158) from dev into main

pg_dump + zstd in runtime image, lift Quart body cap to 1 GiB. See PR #2.

.forgejo/workflows/ci.yml

@@ -9,15 +9,10 @@ name: CI
 on:
   push:
     branches: [dev, main]
-  # Renovate opens PRs from `renovate/*` branches into `dev`. Those branches
+  # pull_request trigger intentionally absent — with branches: [dev, main]
-  # never push to dev/main, so the push trigger above gives them NO pre-merge
+  # above, every PR commit already fires CI via the push event on dev. Adding
-  # CI — a bump could only be validated after it was already merged. This
+  # pull_request would duplicate runs on dev→main PRs. FC has no fork PRs
-  # pull_request trigger (base `dev` only) validates Renovate PRs before merge.
+  # (single-operator Forgejo repo) so push coverage is complete.
-  # It deliberately does NOT fire on dev→main PRs (base `main`), which still
-  # rely on the dev push run — so no duplicate runs. FC has no fork PRs
-  # (single-operator Forgejo repo), so secrets-on-PR is not a concern.
-  pull_request:
-    branches: [dev]
 
 jobs:
   # Fast-fail lint lane. ruff is pre-installed in the ci-python image, so
@@ -32,14 +27,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Ruff lint
-        # agent/ included so the GPU-agent is linted before its image is built
+        run: ruff check backend/ tests/ alembic/
-        # (build.yml only `docker build`s it — this is where it gets checked).
-        run: ruff check backend/ tests/ alembic/ agent/
-      - name: Agent syntax check
-        # The agent's runtime deps (torch/transformers/ultralytics) aren't in the
-        # CI image, so we can't import it — but compileall parses every module,
-        # catching syntax errors before the image build.
-        run: python -m compileall -q agent/fc_agent
 
   backend-lint-and-test:
     runs-on: python-ci
@@ -97,10 +85,10 @@ jobs:
       # If we want strict lockfile-based reproducibility later, commit a
       # package-lock.json and flip this back to `npm ci`.
       - run: npm install --no-audit --no-fund
-      # No type-check step: the frontend is pure JS (no .ts files, no JSDoc),
+      # `npm run check` (vue-tsc --noEmit) skipped: the frontend is pure JS
-      # so a type-checker has nothing to do. The vue-tsc devDep + its `check`
+      # with no .ts files and no JSDoc annotations, so vue-tsc has nothing
-      # script were dropped 2026-07-11 rather than bumped to v3. If we add
+      # to type-check. Re-enable once we add a tsconfig.json and either
-      # TS/JSDoc later, re-add a tsconfig.json + vue-tsc + a type-check step.
+      # convert to TS or add JSDoc.
       - run: npm run test:unit
       - run: npm run build
 

.forgejo/workflows/extension.yml

@@ -20,7 +20,7 @@ jobs:
   lint:
     runs-on: python-ci
     container:
-      image: node:24-bookworm-slim
+      image: node:22-bookworm-slim
     steps:
       - uses: actions/checkout@v4
       - name: Install web-ext

Dockerfile

@@ -1,6 +1,6 @@
-# syntax=docker/dockerfile:1.25
+# syntax=docker/dockerfile:1.7
 
-FROM node:24-alpine AS frontend-builder
+FROM node:22-alpine AS frontend-builder
 WORKDIR /build
 COPY frontend/package.json frontend/package-lock.json* ./
 # No package-lock.json is tracked yet (we don't run npm locally per

Dockerfile.ml

@@ -1,4 +1,4 @@
-# syntax=docker/dockerfile:1.25
+# syntax=docker/dockerfile:1.7
 
 FROM python:3.14-slim
 ENV PYTHONUNBUFFERED=1 \

agent/Dockerfile

@@ -1,24 +1,18 @@
 # FabledCurator GPU agent — runs on the desktop with the GPU.
-# CUDA 12.9 + cuDNN 9 runtime so onnxruntime-gpu can use the card (it needs
+# CUDA + cuDNN runtime so onnxruntime-gpu can use the card (it needs cuDNN 9 —
-# cuDNN 9 — the plain -runtime image lacks it: "libcudnn.so.9: cannot open
+# the plain -runtime image lacks it: "libcudnn.so.9: cannot open shared object
-# shared object file"); ffmpeg for video frames. Ubuntu 24.04 → Python 3.12.
+# file"); ffmpeg for video frames.
-# Stays on the CUDA-12 / cuDNN-9 line the default onnxruntime-gpu + torch are
+FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
-# built against (CUDA 13 has only nascent ONNX Runtime support).
-FROM nvidia/cuda:12.9.2-cudnn-runtime-ubuntu24.04
 
-# PIP_BREAK_SYSTEM_PACKAGES: Ubuntu 24.04 marks its system Python as externally
+ENV DEBIAN_FRONTEND=noninteractive PYTHONUNBUFFERED=1
-# managed (PEP 668), so a global `pip install` errors without this. It's a
-# single-purpose container — we own the whole environment, so installing into
-# the system site-packages is fine (and simplest — no venv on PATH to manage).
-ENV DEBIAN_FRONTEND=noninteractive PYTHONUNBUFFERED=1 PIP_BREAK_SYSTEM_PACKAGES=1
 RUN apt-get update \
     && apt-get install -y --no-install-recommends python3 python3-pip ffmpeg \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
-# torch from the CUDA-12.4 wheel index; its wheels bundle their own CUDA + cuDNN
+# torch from the CUDA-12.4 wheel index (matches the base image); its wheels
-# so they run on the 12.9 base and coexist with onnxruntime-gpu. Installed first
+# bundle their own CUDA + cuDNN and coexist with onnxruntime-gpu. Installed
-# + separately so the GPU build of torch is deterministic and layer-cached.
+# first + separately so the GPU build of torch is deterministic and layer-cached.
 RUN pip3 install --no-cache-dir torch==2.6.0 --index-url https://download.pytorch.org/whl/cu124
 COPY requirements.txt .
 RUN pip3 install --no-cache-dir -r requirements.txt

agent/docker-compose.yml

@@ -38,10 +38,6 @@ services:
       # spot + backs off under VRAM pressure). On by default; toggle live in the
       # control UI. Set to 0 to start in manual mode.
       AUTO_SCALE: ${AUTO_SCALE:-1}
-      # Aggregate download cap in MB/s (stills + video streams combined), so the
-      # agent can't saturate the desktop's network and wreck browsing — WiFi
-      # especially. 0 = unlimited; tunable live in the control UI.
-      BANDWIDTH_LIMIT_MB_S: ${BANDWIDTH_LIMIT_MB_S:-8}
       # Crop embedder (SigLIP concept bag): float16 keeps VRAM low on a shared
       # desktop GPU; the model itself is announced by the server.
       SIGLIP_DTYPE: ${SIGLIP_DTYPE:-float16}

agent/fc_agent/app.py

@@ -1,45 +1,21 @@
 """FastAPI control surface for the agent (served on localhost).
 
-Start / stop the download→GPU pipeline, tune the downloader count live (the
+Start / stop the worker pool, tune the worker count live (trades desktop
-workload is download-bound, so downloaders are the dial that trades desktop
+responsiveness for throughput), and watch GPU load + progress + the server-side
-bandwidth for throughput), and watch GPU load + buffer occupancy + progress +
+queue. Config is env-seeded; the worker count is adjustable here on the fly.
-the server-side queue. Config is env-seeded; the downloader count is adjustable
-here on the fly (GPU consumers autoscale between 1 and 2 on their own).
 """
-import logging
-
 from fastapi import FastAPI, Request
 from fastapi.responses import HTMLResponse, JSONResponse
 
-from . import logbuf
 from .config import Config
 from .gpu import read_gpu
 from .worker import Worker
 
-log = logging.getLogger("fc_agent.app")
-
-# Bump on every agent change. The page embeds this and /status reports it; the UI
-# warns to reload when they differ — so a stale browser-cached page can't be
-# mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.)
-VERSION = "2026-07-02.6 · sleep mode: an empty queue sheds to one downloader and backs the lease poll off to 15 min"
-
-logbuf.install()
 cfg = Config.from_env()
 worker = Worker(cfg)
 app = FastAPI(title="FabledCurator GPU agent")
 
 
-@app.middleware("http")
-async def _no_store(request, call_next):
-    # The control page is a static string and the status/gpu/logs polls are
-    # live data — never let the browser cache either, or a freshly-pulled agent
-    # image still shows the OLD UI until a hard refresh (operator-flagged
-    # 2026-06-30).
-    resp = await call_next(request)
-    resp.headers["Cache-Control"] = "no-store"
-    return resp
-
-
 @app.on_event("startup")
 def _maybe_autostart() -> None:
     # With AUTO_START set, a container restart (host reboot, or `restart:
@@ -52,19 +28,17 @@ def _maybe_autostart() -> None:
 
 @app.get("/", response_class=HTMLResponse)
 def index() -> str:
-    return _PAGE.replace("__BUILD__", VERSION)
+    return _PAGE
 
 
 @app.post("/start")
 def start():
-    log.info("UI: Start button pressed")   # the press; worker logs the transition
     worker.start()
     return JSONResponse(worker.status())
 
 
 @app.post("/stop")
 def stop():
-    log.info("UI: Stop button pressed")
     worker.stop()
     return JSONResponse(worker.status())
 
@@ -83,207 +57,65 @@ async def auto(request: Request):
     return JSONResponse(worker.status())
 
 
-@app.post("/bandwidth")
-async def bandwidth(request: Request):
-    body = await request.json()
-    worker.set_bandwidth(float(body.get("value", 0)))
-    return JSONResponse(worker.status())
-
-
-@app.get("/gpu")
-def gpu():
-    # GPU meters poll this on their own fast cadence. It only reads local
-    # nvidia-smi — no curator round-trip — so the util/VRAM bars stay live even
-    # when /status is slow waiting on the (sometimes busy) curator queue call.
-    g = read_gpu() or {}
-    us = worker.util_smooth()
-    if us is not None:
-        g["util_smooth"] = round(us, 1)   # autoscaler's EWMA — the UI bar tracks this
-    return JSONResponse(g)
-
-
-@app.get("/logs")
-def logs():
-    return JSONResponse({"lines": list(logbuf.LINES)})
-
-
 @app.get("/status")
 def status():
-    # Pure in-memory read: worker.status() is lock-free and the queue snapshot is
-    # kept fresh by a background poller — NO inline curator call, so this can't
-    # stall the status view when curator is buried under a big backlog.
-    worker.note_ui()          # a browser is watching → keep the queue snapshot warm
     s = worker.status()
     s["fc_url"] = cfg.fc_url
     s["configured"] = bool(cfg.token)
-    s["queue"] = worker.latest_queue()
+    s["gpu"] = read_gpu()
-    s["build"] = VERSION
+    try:
+        s["queue"] = worker.client.queue_status()
+    except Exception:
+        s["queue"] = None
     return JSONResponse(s)
 
 
 _PAGE = """<!doctype html><html><head><meta charset=utf-8>
-<meta name=viewport content="width=device-width,initial-scale=1">
+<title>FabledCurator GPU agent</title>
-<title>FabledCurator · GPU agent</title>
 <style>
- :root{--bg:#0f1216;--panel:#181c22;--panel2:#1e232b;--bd:#2a313b;--fg:#e9edf2;
+ body{font:14px system-ui;margin:2rem;max-width:680px;background:#14171a;color:#e8e8e8}
-  --mut:#8b97a6;--acc:#e8923a;--grn:#46c46a;--red:#e8584d;--amb:#e8b23a}
+ h1{font-size:18px} button{font:14px system-ui;padding:.5rem 1rem;border:0;border-radius:6px;
- *{box-sizing:border-box}
+ margin-right:.5rem;cursor:pointer;color:#fff} .start{background:#2e7d32}.stop{background:#b3261e}
- body{font:14px/1.5 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;margin:0;
+ .step{background:#33373b;padding:.4rem .7rem;font-weight:700}
-  background:radial-gradient(1200px 600px at 50% -10%,#1a2029,#0f1216);color:var(--fg)}
+ .stat{display:inline-block;margin-right:1.5rem;vertical-align:top}
- .wrap{max-width:820px;margin:0 auto;padding:28px 20px 28px;height:100vh;
+ .n{font-size:22px;font-weight:700} code{background:#222;padding:2px 6px;border-radius:4px}
-  box-sizing:border-box;overflow:hidden;display:flex;flex-direction:column}
+ .q,.gpu{margin-top:1rem;color:#9aa} .bar{height:8px;border-radius:4px;background:#222;overflow:hidden;
- header{display:flex;align-items:center;justify-content:space-between;margin-bottom:4px}
+ max-width:320px;margin-top:4px} .bar>i{display:block;height:100%;background:#3f7d3f}
- .brand{display:flex;align-items:center;gap:10px;font-size:19px;font-weight:700;letter-spacing:.2px}
+ .row{margin:.8rem 0}
- .logo{color:var(--acc);font-size:20px}
- .brand .sub{color:var(--mut);font-weight:600;font-size:13px;text-transform:uppercase;letter-spacing:.12em}
- .conn{display:flex;align-items:center;gap:8px;color:var(--mut);font-size:13px;font-weight:600}
- .dot{width:9px;height:9px;border-radius:50%;background:var(--mut);box-shadow:0 0 0 0 rgba(0,0,0,0)}
- .dot.green{background:var(--grn);box-shadow:0 0 10px 1px rgba(70,196,106,.5)}
- .dot.amber{background:var(--amb)} .dot.red{background:var(--red)}
- .meta{color:var(--mut);margin:0 0 18px;font-size:13px}
- code{background:#11151a;border:1px solid var(--bd);padding:2px 7px;border-radius:6px;
-  font:12px ui-monospace,SFMono-Regular,Menlo,monospace;color:#cdd6e0}
- .card{background:linear-gradient(180deg,var(--panel),var(--panel2));border:1px solid var(--bd);
-  border-radius:14px;padding:16px 18px;margin-bottom:14px;box-shadow:0 1px 0 rgba(255,255,255,.02) inset}
- .card-h{font-size:11px;font-weight:800;letter-spacing:.12em;text-transform:uppercase;
-  color:var(--mut);margin-bottom:14px}
- .controls{display:flex;align-items:center;gap:10px;flex-wrap:wrap}
- .spacer{flex:1}
- .btn{font:600 14px system-ui;padding:.5rem 1rem;border:1px solid transparent;border-radius:9px;
-  cursor:pointer;color:#fff;transition:.12s}
- .btn:hover{transform:translateY(-1px)}
- .btn[disabled]{opacity:.45;pointer-events:none;transform:none}
- @keyframes pulse{0%,100%{opacity:1}50%{opacity:.4}}
- .tile .n.busy{color:var(--acc);animation:pulse 1s ease-in-out infinite}
- .btn.start{background:linear-gradient(180deg,#2f9c4c,#247a3c)}
- .btn.stop{background:linear-gradient(180deg,#3a3f48,#2a2f37);color:#e9edf2;border-color:var(--bd)}
- .switch{display:inline-flex;align-items:center;gap:8px;cursor:pointer;font-weight:600;user-select:none}
- .switch input{display:none}
- .switch .track{width:38px;height:22px;border-radius:11px;background:#2a313b;position:relative;transition:.15s}
- .switch .track:after{content:"";position:absolute;top:2px;left:2px;width:18px;height:18px;border-radius:50%;
-  background:#cdd6e0;transition:.15s}
- .switch input:checked+.track{background:var(--acc)}
- .switch input:checked+.track:after{transform:translateX(16px);background:#fff}
- .stepper{display:inline-flex;align-items:center;gap:6px}
- .step{background:#262c34;color:var(--fg);border:1px solid var(--bd);border-radius:8px;
-  width:30px;height:32px;font:700 16px system-ui;cursor:pointer}
- .step:hover{border-color:var(--acc)}
- #conc,#bw{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a;
-  color:var(--fg);border:1px solid var(--bd);border-radius:8px}
- .unit{color:var(--mut);font-size:12px;font-weight:600}
- .hint{color:var(--mut);font-size:12px;margin-top:12px}
- .tiles{display:grid;grid-template-columns:repeat(6,1fr);gap:8px;margin-bottom:16px}
- .tile{background:#13171d;border:1px solid var(--bd);border-radius:10px;padding:12px 8px;text-align:center}
- .tile .n{font:800 22px ui-monospace,monospace;line-height:1.1}
- .tile .n.warn{color:var(--red)} .tile .n.ok{color:var(--grn)}
- .tile .l{font-size:10px;text-transform:uppercase;letter-spacing:.06em;color:var(--mut);margin-top:4px}
- .meters{display:flex;flex-direction:column;gap:10px;margin-bottom:14px}
- .meter-h{display:flex;justify-content:space-between;font-size:12px;color:var(--mut);margin-bottom:4px}
- .meter-h b{color:var(--fg);font-variant-numeric:tabular-nums}
- .bar{height:9px;border-radius:5px;background:#11151a;border:1px solid var(--bd);overflow:hidden}
- .bar>i{display:block;height:100%;width:0;background:linear-gradient(90deg,#3a7d57,var(--grn));transition:width .4s}
- #utilbar{background:linear-gradient(90deg,#9a5a1f,var(--acc))}
- #bufbar{background:linear-gradient(90deg,#2f5a9a,#4a86d8)}
- .queue{font:13px ui-monospace,monospace;color:var(--mut)}
- .banner{margin:0 0 14px;padding:.7rem .9rem;border-radius:10px;background:#3a2f12;
-  border:1px solid #5a4a17;color:#ffd98a;font-size:13px}
- .logs-h{display:flex;align-items:center;justify-content:space-between}
- .grow{flex:1;display:flex;flex-direction:column;min-height:0}
- .grow .logs{flex:1;min-height:0}
- .copybtn{font:600 11px system-ui;letter-spacing:.04em;text-transform:uppercase;
-  background:#262c34;color:var(--fg);border:1px solid var(--bd);border-radius:7px;
-  padding:5px 11px;cursor:pointer}
- .copybtn:hover{border-color:var(--acc)}
- .logs{margin:0;background:#0b0e12;border:1px solid var(--bd);border-radius:10px;padding:12px;
-  overflow:auto;font:12px/1.55 ui-monospace,SFMono-Regular,Menlo,monospace;
-  color:#b9c4d0;white-space:pre-wrap;word-break:break-word}
 </style></head><body>
-<div class=wrap>
+<h1>FabledCurator GPU agent</h1>
- <header>
+<p>FC: <code id=fc>—</code> · token <code id=cfg>—</code></p>
-  <div class=brand><span class=logo>◆</span> FabledCurator <span class=sub>GPU agent</span></div>
+<div class=row>
-  <div class=conn><span class="dot" id=dot></span><span id=connlbl>—</span></div>
+ <button class=start onclick=act('start')>Start</button>
- </header>
+ <button class=stop onclick=act('stop')>Stop</button>
- <p class=meta>Server <code id=fc>—</code> · token <code id=cfg>—</code> · build <code id=build>__BUILD__</code></p>
-
- <div id=verbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3">
-  a newer agent version is running — reload this page (Ctrl+Shift+R) to update the controls
- </div>
- <div id=banner class=banner style=display:none>
-  curator unreachable — holding work + retrying, resumes on its own (no restart needed)
- </div>
-
- <section class=card>
-  <div class=card-h>Control</div>
-  <div class=controls>
-   <button class="btn start" id=startbtn onclick=act('start')>▶ Start</button>
-   <button class="btn stop" id=stopbtn onclick=act('stop')>■ Stop</button>
-   <div class=spacer></div>
-   <label class=switch><input type=checkbox id=autochk onchange="setauto(this.checked)"><span class=track></span>Auto</label>
-   <div class=stepper>
-    <button class=step onclick=setc(-1)>−</button>
-    <input id=conc type=number min=1 value=1 onchange="setv(this.value)">
-    <button class=step onclick=setc(1)>+</button>
-   </div>
-   <div class=stepper title="aggregate download cap, downloads + video streams combined — 0 = unlimited">
-    <input id=bw type=number min=0 step=1 value=8 onchange="setbw(this.value)">
-    <span class=unit>MB/s</span>
-   </div>
-  </div>
-  <div class=hint id=conchint>auto-tuning downloaders to keep the GPU fed · max 8</div>
- </section>
-
- <section class=card>
-  <div class=card-h>Status</div>
-  <div class=tiles>
-   <div class=tile><div class=n id=state>—</div><div class=l>state</div></div>
-   <div class=tile><div class=n id=jpm>—</div><div class=l>jobs / min</div></div>
-   <div class=tile><div class=n id=dpm>—</div><div class=l>downloads / min</div></div>
-   <div class=tile><div class="n ok" id=done>0</div><div class=l>processed</div></div>
-   <div class=tile><div class=n id=err>0</div><div class=l>errors</div></div>
-   <div class=tile><div class=n id=waited>0</div><div class=l>waited out</div></div>
-  </div>
-  <div class=meters>
-   <div class=meter><div class=meter-h><span>GPU util</span><b id=utillbl>—</b></div>
-    <div class=bar><i id=utilbar></i></div></div>
-   <div class=meter><div class=meter-h><span>VRAM</span><b id=vramlbl>—</b></div>
-    <div class=bar><i id=gpubar></i></div></div>
-   <div class=meter><div class=meter-h><span>buffer occupancy</span><b id=buflbl>—</b></div>
-    <div class=bar><i id=bufbar></i></div></div>
-  </div>
-  <div class=queue id=pipe>downloaders — · consumers — · on GPU 0</div>
-  <div class=queue id=queue>queue —</div>
- </section>
-
- <section class="card grow">
-  <div class="card-h logs-h">Logs
-   <button class=copybtn id=copybtn onclick=copyLogs()>Copy</button>
-  </div>
-  <pre class=logs id=logs>waiting for activity…</pre>
- </section>
 </div>
+<div class=row>
+ <label style="margin-right:1rem"><input type=checkbox id=autochk onchange="setauto(this.checked)"> Auto</label>
+ workers
+ <button class=step onclick=setc(-1)>−</button>
+ <input id=conc type=number min=1 value=1
+  style="width:3.5rem;font:700 16px system-ui;text-align:center;background:#222;color:#e8e8e8;border:1px solid #444;border-radius:6px;padding:.3rem"
+  onchange="setv(this.value)">
+ <button class=step onclick=setc(1)>+</button>
+ <span class=cap style=color:#9aa id=conchint>auto-tuning to fill the GPU · max <b id=capn>8</b></span>
+</div>
+<div class=row>
+ <span class=stat><span class=n id=state>stopped</span><br>state</span>
+ <span class=stat><span class=n id=active>0</span><br>active now</span>
+ <span class=stat><span class=n id=done>0</span><br>processed</span>
+ <span class=stat><span class=n id=err>0</span><br>errors</span>
+ <span class=stat><span class=n id=wait>0</span><br>waited out</span>
+</div>
+<div id=banner style="display:none;margin:.6rem 0;padding:.5rem .8rem;border-radius:6px;background:#5a4a17;color:#ffe28a">
+ curator unreachable — holding work + retrying, will resume on its own (no restart needed)
+</div>
+<div class=gpu id=gpu>GPU — …</div>
+<div class=bar><i id=gpubar style=width:0%></i></div>
+<div class=q id=queue></div>
 <script>
- const PAGE_BUILD="__BUILD__"
  let CAP=8
- // Optimistic transitional state on click, then apply the POST's own status
+ async function act(p){await fetch('/'+p,{method:'POST'});refresh()}
- // response (it returns worker.status()) for instant feedback — don't wait on the
+ function setc(d){ setv((parseInt(conc.value||'1'))+d) }
- // separate /status poll, which can lag behind the curator queue call.
- async function act(p){
-  pending(p==='start'?'starting':'stopping')
-  // Abort a slow POST after 8s so the buttons never stay stuck — the periodic
-  // /status refresh (now always fast) recovers the true state either way.
-  const ac=new AbortController(); const to=setTimeout(()=>ac.abort(),8000)
-  try{ applyStatus(await (await fetch('/'+p,{method:'POST',signal:ac.signal})).json()) }
-  catch{ refresh() /* on abort/error, repaint the real state from /status */ }
-  finally{ clearTimeout(to) }
- }
- function pending(label){
-  // Instant optimistic feedback on click; applyStatus (POST response, then the
-  // periodic poll) then owns the real state + which buttons are enabled.
-  state.textContent=label; state.className='n busy'
-  dot.className='dot amber'
-  startbtn.disabled=true; stopbtn.disabled=true
- }
- function setc(d){ if(conc.disabled)return; setv((parseInt(conc.value||'1'))+d) }
  async function setv(v){
   v=Math.max(1,Math.min(CAP,parseInt(v)||1)); conc.value=v
   await fetch('/concurrency',{method:'POST',headers:{'Content-Type':'application/json'},
@@ -293,112 +125,27 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
   await fetch('/auto',{method:'POST',headers:{'Content-Type':'application/json'},
    body:JSON.stringify({value:on})});refresh()
  }
- async function setbw(v){
-  v=Math.max(0,parseFloat(v)||0); bw.value=v
-  await fetch('/bandwidth',{method:'POST',headers:{'Content-Type':'application/json'},
-   body:JSON.stringify({value:v})});refresh()
- }
  async function refresh(){
-  let s; try{ s=await (await fetch('/status')).json() }catch{ return }
+  const s=await (await fetch('/status')).json()
-  applyStatus(s)
+  CAP=s.max_concurrency||8; capn.textContent=CAP
- }
+  state.textContent=s.state; active.textContent=s.active; done.textContent=s.processed
- function applyStatus(s){
+  err.textContent=s.errors; fc.textContent=s.fc_url; wait.textContent=s.transient||0
-  // NB: don't write a separate `capn` element here — conchint.textContent below
+  // Running but the queue read failed → curator is unreachable; show we're
-  // rewrites the whole hint (incl. the max), and any child element nested in it
+  // riding it out rather than erroring.
-  // would be destroyed by that write, breaking the NEXT applyStatus call.
+  banner.style.display=(s.state==='running' && !s.queue)?'block':'none'
-  CAP=s.max_concurrency||8
+  // Auto on → the dial reflects the auto-chosen count (read-only); off → manual.
-  // The backend owns the state now (stopped|starting|running|stopping) and drives
-  // every transition, so the pill is always truthful — no client-side guessing
-  // from active>0, which used to wedge on "stopping" forever.
-  const st=s.state||'stopped'
-  const running=st==='running'
-  const busy=(st==='starting'||st==='stopping')
-  // Stale-page guard: if the server is a newer build than this page, the cached
-  // controls may misbehave — tell the operator to reload.
-  if(s.build && s.build!==PAGE_BUILD) verbanner.style.display='block'
-  state.textContent=st
-  state.className='n'+(busy?' busy':'')
-  // Buttons follow the real state so you can't fight a transition: Start only
-  // from stopped; Stop only while up; both disabled through "stopping" until the
-  // backend truthfully lands on "stopped".
-  startbtn.disabled=(st!=='stopped')
-  stopbtn.disabled=!(running||st==='starting')
-  // Throughput rates arrive READY from the backend (jobs/min ≈ GPU throughput,
-  // dl/min ≈ fetch throughput), computed there on a fixed cadence — so they show
-  // a real number no matter how often this tab polls (a backgrounded tab throttles
-  // its timers, which used to leave a client-side delta-rate blank forever).
-  jpm.textContent=(s.jobs_per_min!=null)?Math.round(s.jobs_per_min):'—'
-  dpm.textContent=(s.downloads_per_min!=null)?Math.round(s.downloads_per_min):'—'
-  done.textContent=s.processed
-  err.textContent=s.errors; err.className='n'+(s.errors>0?' warn':'')
-  waited.textContent=s.transient||0
-  // Instantaneous pool state → demoted to the sub-line, where its jumpiness reads
-  // as live churn rather than a "broken" headline metric.
-  pipe.textContent='downloaders '+(s.downloaders!=null?s.downloaders:'—')+' · consumers '+(s.consumers!=null?s.consumers:'—')+' · on GPU '+(s.active||0)
-   +' · net '+(s.net_mb_s!=null?s.net_mb_s.toFixed(1):'—')+' MB/s'
-   +(s.bandwidth_limit_mb_s>0?(' / cap '+s.bandwidth_limit_mb_s):'')
-  if(document.activeElement!==bw && s.bandwidth_limit_mb_s!=null) bw.value=s.bandwidth_limit_mb_s
-  // Buffer occupancy bar (also driven here so it tracks the /status cadence).
-  if(s.buffer!=null && s.buffer_max){ const p=Math.round(100*s.buffer/s.buffer_max)
-   buflbl.textContent=s.buffer+' / '+s.buffer_max; bufbar.style.width=p+'%' }
-  // Auto on → dial reflects the auto-chosen count (read-only); off → manual.
   if(document.activeElement!==autochk) autochk.checked=!!s.auto
-  conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.55:1
+  conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.6:1
-  conchint.textContent=(s.auto?('auto-tuning downloaders to keep the GPU fed · max '+CAP):('manual downloaders · max '+CAP))
+  conchint.textContent=s.auto?('auto-tuning to fill the GPU · max '+CAP):('manual · max '+CAP)
-   +(s.idle?' · idle — queue empty, lease poll backed off (new work noticed within ~15 min)'
+  capn.textContent=CAP
-     :(s.bw_capped?' · holding at the bandwidth cap (more downloaders would not go faster)':''))
   if(document.activeElement!==conc) conc.value=s.concurrency
   conc.max=CAP
-  // Connection pill + queue come only from the /status poll (the Start/Stop POST
+  cfg.textContent=s.configured?'set':'MISSING'
-  // responses skip the slow curator call to stay snappy) — guard so an action
+  if(s.gpu){
-  // response doesn't blank them.
+   gpu.textContent=`GPU — ${s.gpu.util_pct}% util · VRAM ${s.gpu.mem_used_mb}/${s.gpu.mem_total_mb} MB · ${s.gpu.temp_c}°C`
-  if('configured' in s){
+   gpubar.style.width=Math.round(100*s.gpu.mem_used_mb/s.gpu.mem_total_mb)+'%'
-   const ok=s.configured
+  } else { gpu.textContent='GPU — n/a (CPU fallback?)'; gpubar.style.width='0%' }
-   fc.textContent=s.fc_url; cfg.textContent=ok?'set':'MISSING'
+  queue.textContent=s.queue?`queue — pending ${s.queue.pending} · in flight ${s.queue.leased} · done ${s.queue.done} · errored ${s.queue.error}`:'queue — unreachable'
-   // Pill colour + label track the real state: green only when running AND
-   // curator is answering; amber for the transient states + a running-but-
-   // unreachable curator; grey when stopped; red with no token.
-   let dc='dot', lbl='stopped'
-   if(!ok){ dc='dot red'; lbl='no token' }
-   else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable' }
-   else if(st==='starting'){ dc='dot amber'; lbl='starting…' }
-   else if(st==='stopping'){ dc='dot amber'; lbl='stopping…' }
-   dot.className=dc; connlbl.textContent=lbl
-   banner.style.display=(st==='running' && !s.queue)?'block':'none'
-   queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
-  }
  }
- // GPU meters poll their OWN endpoint on a fast cadence — kept off /status so a
+ refresh(); setInterval(refresh,3000)
- // slow curator queue call can't freeze the bars (they only stale on refresh).
- let UAVG=null   // smoothed util for the bar (raw util swings 0↔99; show the trend)
- async function refreshGpu(){
-  let g; try{ g=await (await fetch('/gpu')).json() }catch{ return }
-  if(g && g.util_pct!=null){
-   // Prefer the agent's own EWMA (util_smooth) when running; otherwise smooth
-   // the raw reading here so a stopped agent's bar still glides, not jumps.
-   const raw=g.util_pct
-   UAVG = (g.util_smooth!=null) ? g.util_smooth
-         : (UAVG==null ? raw : 0.25*raw + 0.75*UAVG)
-   const used=g.mem_used_mb, tot=g.mem_total_mb||1
-   utillbl.textContent=Math.round(UAVG)+'% · '+g.temp_c+'°C'; utilbar.style.width=Math.round(UAVG)+'%'
-   vramlbl.textContent=used+' / '+tot+' MB'; gpubar.style.width=Math.round(100*used/tot)+'%'
-  } else { UAVG=null; utillbl.textContent='n/a'; vramlbl.textContent='n/a (CPU?)'; utilbar.style.width='0%'; gpubar.style.width='0%' }
- }
- async function refreshLogs(){
-  try{
-   const r=await (await fetch('/logs')).json()
-   const el=logs, atBottom=el.scrollHeight-el.scrollTop-el.clientHeight<40
-   el.textContent=(r.lines&&r.lines.length)?r.lines.join('\\n'):'waiting for activity…'
-   if(atBottom) el.scrollTop=el.scrollHeight
-  }catch{}
- }
- async function copyLogs(){
-  const txt=logs.textContent||''
-  try{ await navigator.clipboard.writeText(txt) }
-  catch{ const t=document.createElement('textarea'); t.value=txt; document.body.appendChild(t);
-   t.select(); try{document.execCommand('copy')}catch{}; t.remove() }
-  copybtn.textContent='Copied'; setTimeout(()=>{copybtn.textContent='Copy'},1200)
- }
- refresh(); refreshGpu(); refreshLogs()
- setInterval(refresh,3000); setInterval(refreshGpu,1500); setInterval(refreshLogs,2500)
 </script></body></html>"""

agent/fc_agent/client.py

@@ -5,69 +5,19 @@ bytes, all over HTTP with the bearer token. No DB/Redis.
 """
 import requests
 from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
 
 
 class FcClient:
     def __init__(self, base_url: str, token: str, agent_id: str):
         self.base = base_url.rstrip("/")
         self.agent_id = agent_id
-        # Main session: NO in-request retry — lease/fetch are cheap to redo and
+        self.s = requests.Session()
-        # the worker loop already backs off + re-leases on failure. (Auto-retrying
+        self.s.headers["Authorization"] = f"Bearer {token}"
-        # a lease could double-claim a batch if a response is lost.)
+        # Many worker threads share this Session; the default pool (10) would
-        self.s = self._session(token)
-        # Submit session: retry in-place, because by submit time the GPU work is
-        # already DONE — a momentary blip (dropped connection, gateway 5xx during
-        # a curator redeploy) must not throw that work away and force a full
-        # re-download + recompute on another agent. A duplicate submit after a
-        # lost response is harmless: the job is already closed, so it just returns
-        # 409 lease_invalid (a no-op). Idempotent enough to retry POST safely.
-        retry = Retry(
-            total=3, connect=3, read=3, status=3,
-            backoff_factor=0.5,                       # ~0.5s, 1s, 2s between tries
-            status_forcelist=(500, 502, 503, 504),    # transient server/gateway
-            allowed_methods=frozenset({"POST"}),
-            raise_on_status=False,                     # let raise_for_status decide
-        )
-        self._submit_s = self._session(token, retry)
-
-    @staticmethod
-    def _session(token: str, retry: Retry | None = None) -> requests.Session:
-        s = requests.Session()
-        s.headers["Authorization"] = f"Bearer {token}"
-        # Many worker threads share a Session; the default pool (10) would
         # throttle them + spam "connection pool is full". Size it for the cap.
-        adapter = HTTPAdapter(
+        adapter = HTTPAdapter(pool_connections=64, pool_maxsize=64)
-            pool_connections=64, pool_maxsize=64, max_retries=retry or 0
+        self.s.mount("http://", adapter)
-        )
+        self.s.mount("https://", adapter)
-        s.mount("http://", adapter)
-        s.mount("https://", adapter)
-        return s
-
-    def _submit(self, path: str, payload: dict) -> dict:
-        """POST to a submit endpoint on the RETRYING session (by submit time the
-        GPU work is done — a blip must not throw it away), raise on a hard error,
-        and return the parsed JSON. `agent_id` is added to every body."""
-        r = self._submit_s.post(
-            f"{self.base}{path}",
-            json={"agent_id": self.agent_id, **payload},
-            timeout=120,
-        )
-        r.raise_for_status()
-        return r.json()
-
-    def _post_quiet(self, path: str, payload: dict) -> None:
-        """Fire-and-forget POST on the main session — heartbeat/fail/release are
-        best-effort, so a transport error is swallowed (the worker's own retry and
-        the server's orphan-recovery cover a lost call). `agent_id` is added."""
-        try:
-            self.s.post(
-                f"{self.base}{path}",
-                json={"agent_id": self.agent_id, **payload},
-                timeout=30,
-            )
-        except requests.RequestException:
-            pass
 
     def lease(self, batch_size: int) -> list[dict]:
         r = self.s.post(
@@ -79,62 +29,70 @@ class FcClient:
         return r.json().get("jobs", [])
 
     def submit(self, job_id: int, regions: list[dict], replace_kinds: list[str]) -> dict:
-        return self._submit("/api/gpu/jobs/submit", {
+        r = self.s.post(
-            "job_id": job_id, "regions": regions, "replace_kinds": replace_kinds,
+            f"{self.base}/api/gpu/jobs/submit",
-        })
+            json={
+                "agent_id": self.agent_id, "job_id": job_id,
+                "regions": regions, "replace_kinds": replace_kinds,
+            },
+            timeout=120,
+        )
+        r.raise_for_status()
+        return r.json()
 
     def submit_embedding(self, job_id: int, embedding: list, version: str) -> dict:
         """Post a whole-image SigLIP embedding (the 'embed' task) → image_record."""
-        return self._submit("/api/gpu/jobs/submit_embedding", {
+        r = self.s.post(
-            "job_id": job_id, "embedding": embedding, "embedding_version": version,
+            f"{self.base}/api/gpu/jobs/submit_embedding",
-        })
+            json={
+                "agent_id": self.agent_id, "job_id": job_id,
+                "embedding": embedding, "embedding_version": version,
+            },
+            timeout=120,
+        )
+        r.raise_for_status()
+        return r.json()
 
     def heartbeat(self, job_ids: list[int]) -> None:
-        self._post_quiet("/api/gpu/jobs/heartbeat", {"job_ids": job_ids})
+        try:
+            self.s.post(
+                f"{self.base}/api/gpu/jobs/heartbeat",
+                json={"agent_id": self.agent_id, "job_ids": job_ids},
+                timeout=30,
+            )
+        except requests.RequestException:
+            pass
 
     def fail(self, job_id: int, error: str) -> None:
-        self._post_quiet("/api/gpu/jobs/fail", {"job_id": job_id, "error": error})
+        try:
+            self.s.post(
+                f"{self.base}/api/gpu/jobs/fail",
+                json={"agent_id": self.agent_id, "job_id": job_id, "error": error},
+                timeout=30,
+            )
+        except requests.RequestException:
+            pass
 
     def release(self, job_ids: list[int]) -> None:
         # Graceful hand-back on stop so orphaned work is re-leased at once.
         if not job_ids:
             return
-        self._post_quiet("/api/gpu/jobs/release", {"job_ids": job_ids})
-
-    def fetch_image(self, image_url: str, throttle=None) -> bytes:
-        # image_url is a server-relative path ("/images/...").
-        # timeout=(connect, read): the read timeout is BETWEEN-BYTES, not total,
-        # so a large-but-flowing download still completes — but a stuck/dead
-        # connection (curator overloaded) fails in 60s instead of hanging a
-        # downloader for 180s and piling up concurrent stuck requests on curator.
-        # With a throttle (the worker's shared TokenBucket), the body is streamed
-        # in chunks and each chunk is charged to the global bandwidth budget —
-        # pausing between reads lets TCP flow control pace curator's send side.
-        with self.s.get(
-            f"{self.base}{image_url}", timeout=(10, 60), stream=throttle is not None
-        ) as r:
-            r.raise_for_status()
-            if throttle is None:
-                return r.content
-            buf = bytearray()
-            for chunk in r.iter_content(chunk_size=262_144):
-                throttle.take(len(chunk))
-                buf.extend(chunk)
-            return bytes(buf)
-
-    def is_reachable(self) -> bool:
-        """Cheap 'is curator responding at all right now?' check. Used to decide,
-        when a video can't be sampled, between a transient outage (keep retrying —
-        survives a redeploy) and an unprocessable file (fail it, don't loop)."""
         try:
-            r = self.s.get(f"{self.base}/api/gpu/status", timeout=5)
+            self.s.post(
-            return r.status_code < 500
+                f"{self.base}/api/gpu/jobs/release",
+                json={"agent_id": self.agent_id, "job_ids": job_ids},
+                timeout=30,
+            )
         except requests.RequestException:
-            return False
+            pass
+
+    def fetch_image(self, image_url: str) -> bytes:
+        # image_url is a server-relative path ("/images/...").
+        r = self.s.get(f"{self.base}{image_url}", timeout=180)
+        r.raise_for_status()
+        return r.content
 
     def queue_status(self) -> dict:
-        # Short timeout: this backs the UI /status poll, so a busy curator must
+        r = self.s.get(f"{self.base}/api/gpu/status", timeout=15)
-        # not hang the page for long (the GPU meters poll /gpu separately).
-        r = self.s.get(f"{self.base}/api/gpu/status", timeout=5)
         r.raise_for_status()
         return r.json()

agent/fc_agent/config.py

@@ -1,18 +1,8 @@
 """Agent config, all from env (the control container is configured at run)."""
-# Lazy annotations so the `from_env(cls) -> Config` self-reference is a string,
-# not evaluated at class-definition time — otherwise it NameErrors on the agent's
-# Python 3.10 (CI lints on 3.14, where PEP 649 hides this).
-from __future__ import annotations
-
 import os
 from dataclasses import dataclass
 
 
-def _bool_env(name: str, default: str = "") -> bool:
-    """A boolean env var — present + truthy ('1'/'true'/'yes') → True."""
-    return os.environ.get(name, default).lower() in ("1", "true", "yes")
-
-
 @dataclass
 class Config:
     fc_url: str          # base URL of the FabledCurator web service
@@ -39,21 +29,9 @@ class Config:
     panel_conf: float
     max_components: int   # cap anatomy component crops per frame
     max_panels: int       # cap panel crops per frame
-    max_figures: int      # cap figure boxes per frame (each = a CCIP call + crop)
-    max_regions: int      # hard cap on total regions per JOB (submit-size backstop)
-    dedupe_iou: float     # crops overlapping >= this (same kind) are near-dupes,
-    #                       dropped before the embed; >=1.0 disables it
-    frame_dedupe_distance: int  # video frames whose dHash differs by < this many
-    #                             bits are near-duplicates, dropped before detect;
-    #                             higher keeps more frames, 0 disables
-    ffmpeg_timeout: float  # hard ceiling (s) for ffmpeg-from-URL video sampling;
-    #                        generous so a SLOW media link still completes
-    bandwidth_limit_mb_s: float  # aggregate download cap in MEGABYTES/s across
-    #                              all downloaders + video streams (0 = unlimited);
-    #                              tunable live from the agent UI
 
     @classmethod
-    def from_env(cls) -> Config:
+    def from_env(cls) -> "Config":
         return cls(
             fc_url=os.environ.get("FC_URL", "http://localhost:8000").rstrip("/"),
             token=os.environ.get("FC_TOKEN", ""),
@@ -65,8 +43,8 @@ class Config:
             poll_idle_seconds=float(os.environ.get("POLL_IDLE_SECONDS", "10")),
             embed_dtype=os.environ.get("SIGLIP_DTYPE", "float16"),
             embed_model_override=os.environ.get("EMBED_MODEL_NAME", ""),
-            auto_start=_bool_env("AUTO_START"),
+            auto_start=os.environ.get("AUTO_START", "").lower() in ("1", "true", "yes"),
-            auto_scale=_bool_env("AUTO_SCALE", "true"),
+            auto_scale=os.environ.get("AUTO_SCALE", "true").lower() in ("1", "true", "yes"),
             person_weights=os.environ.get("PERSON_WEIGHTS", "yolo11n.pt"),
             person_conf=float(os.environ.get("PERSON_CONF", "0.35")),
             anatomy_weights=os.environ.get("ANATOMY_WEIGHTS", ""),
@@ -75,16 +53,4 @@ class Config:
             panel_conf=float(os.environ.get("PANEL_CONF", "0.30")),
             max_components=int(os.environ.get("MAX_COMPONENTS", "8")),
             max_panels=int(os.environ.get("MAX_PANELS", "8")),
-            max_figures=int(os.environ.get("MAX_FIGURES", "8")),
-            max_regions=int(os.environ.get("MAX_REGIONS", "128")),
-            dedupe_iou=float(os.environ.get("DEDUPE_IOU", "0.85")),
-            frame_dedupe_distance=int(os.environ.get("FRAME_DEDUPE_DISTANCE", "8")),
-            ffmpeg_timeout=float(os.environ.get("FFMPEG_TIMEOUT", "1200")),
-            # Default 8 MB/s (~64 Mbit/s): ~20% of the measured ~300 Mbit/s home
-            # WiFi, so browsing stays snappy while the agent works — yet MORE
-            # sweep throughput than the self-inflicted congestion collapse this
-            # replaces (2026-07-02: 8 unthrottled downloaders bufferbloated the
-            # link to ~1-1.5 MB/s per stream, browser included). Raise it (or 0)
-            # from the agent UI on wired/faster networks.
-            bandwidth_limit_mb_s=float(os.environ.get("BANDWIDTH_LIMIT_MB_S", "8")),
         )

agent/fc_agent/detectors.py

@@ -17,7 +17,6 @@ cached under HF_HOME so the download happens once.
 import logging
 import os
 import threading
-import types
 from pathlib import Path
 
 log = logging.getLogger("fc_agent.detectors")
@@ -67,37 +66,6 @@ def nms_merge(boxes, iou_thresh: float = 0.6):
     return kept
 
 
-def dedupe_crops(pending, iou_thresh: float = 0.85):
-    """Greedy high-IoU dedupe over a list of (crop, region_template) pairs, run
-    just before the batched SigLIP embed so we never embed the same region twice.
-
-    Figure boxes are already NMS-merged and each YOLO self-NMSes, but the combined
-    per-frame pile (figure→concept ∪ anatomy component→concept ∪ panel) can still
-    carry genuine near-duplicates across proposers — e.g. a figure box that nearly
-    coincides with an anatomy component on a solo bust, or overlapping booru head
-    classes on one head. Those embed the same region twice, wasting GPU and a slot
-    against max_regions.
-
-    Boxes are compared ONLY within the same output kind and dropped when they
-    overlap at >= iou_thresh, keeping the highest-scoring one. The HIGH default
-    threshold is deliberate: it collapses only true near-identical boxes while
-    preserving intentional nested crops across scopes (a whole figure vs a small
-    head component sit well below it) and distinct kinds (concept vs panel). A
-    value >= 1.0 effectively disables it (nothing but an exact box matches)."""
-    kept = []
-    kept_boxes: dict = {}   # kind -> [bbox, ...] already kept
-    for crop, tmpl in sorted(
-        pending, key=lambda p: p[1].get("score") or 0.0, reverse=True
-    ):
-        bb = tmpl.get("bbox")
-        prior = kept_boxes.setdefault(tmpl.get("kind"), [])
-        if bb is not None and any(_iou(bb, kb) >= iou_thresh for kb in prior):
-            continue
-        prior.append(bb)
-        kept.append((crop, tmpl))
-    return kept
-
-
 class YoloProposer:
     """One lazily-loaded ultralytics YOLO. detect(image) → [(bbox_norm, score,
     label)] with bbox normalized (x, y, w, h) in [0,1]. Self-disables on any
@@ -125,18 +93,6 @@ class YoloProposer:
                     self._ok = False
                     return
                 self._model = YOLO(path)
-                # Disable ultralytics' load-time Conv+BN fusion. AutoBackend fuses
-                # the graph on the first predict; some checkpoints (yolo11n, the
-                # comic-panel model) crash that step with "'Conv' object has no
-                # attribute 'bn'" (a partially-fused / version-mismatched graph),
-                # which silently disabled those proposers (operator-flagged
-                # 2026-07-01). Unfused inference is correct — only marginally
-                # slower — and this is robust across ultralytics versions; if a
-                # future version ignores the override, the detect() guard below
-                # still self-disables the proposer instead of spamming per image.
-                inner = getattr(self._model, "model", None)
-                if inner is not None:
-                    inner.fuse = types.MethodType(lambda self, *a, **k: self, inner)
                 log.info("detector %s loaded (%s)", self.name, path)
             except Exception as exc:   # noqa: BLE001
                 log.warning("detector %s disabled (load failed): %s", self.name, exc)
@@ -149,12 +105,7 @@ class YoloProposer:
         try:
             res = self._model.predict(image, conf=self._conf, verbose=False)[0]
         except Exception as exc:   # noqa: BLE001
-            # Permanently self-disable on the FIRST inference failure rather than
+            log.warning("detector %s inference failed: %s", self.name, exc)
-            # re-throwing (and re-logging) on every image forever — an unfixable
-            # model fault degrades to "this proposer is off", logged once.
-            log.warning("detector %s disabled (inference failed): %s", self.name, exc)
-            self._ok = False
-            self._model = None
             return []
         iw, ih = image.size
         names = getattr(res, "names", None) or {}
@@ -193,26 +144,20 @@ class Proposers:
 
     def figures(self, image, base_boxes):
         """Merge imgutils person boxes (base_boxes: [(bbox, score)]) with the
-        general COCO person detector → NMS'd figure boxes [(bbox, score, label)],
+        general COCO person detector → NMS'd figure boxes [(bbox, score, label)]."""
-        capped to the highest-scoring max_figures. Uncapped, a busy/huge image
-        (many characters) yields hundreds of boxes → hundreds of per-figure CCIP
-        calls + crops → a 30s+ job and an oversized submit (operator-flagged)."""
         boxes = [(bb, sc if sc is not None else 1.0, "person") for bb, sc in base_boxes]
         if self._person is not None:
             boxes += self._person.detect(image)
-        return nms_merge(boxes)[: self.cfg.max_figures]   # nms_merge is score-desc
+        return nms_merge(boxes)
-
-    @staticmethod
-    def _top(detector, image, cap: int):
-        """Top-`cap` detections by score from an optional proposer (None → the
-        proposer is off → []). Shared by the anatomy + panel proposers, which
-        differ only in which detector and which cap."""
-        if detector is None:
-            return []
-        return sorted(detector.detect(image), key=lambda b: b[1], reverse=True)[:cap]
 
     def components(self, image):
-        return self._top(self._anatomy, image, self.cfg.max_components)
+        if self._anatomy is None:
+            return []
+        items = sorted(self._anatomy.detect(image), key=lambda b: b[1], reverse=True)
+        return items[: self.cfg.max_components]
 
     def panels(self, image):
-        return self._top(self._panel, image, self.cfg.max_panels)
+        if self._panel is None:
+            return []
+        items = sorted(self._panel.detect(image), key=lambda b: b[1], reverse=True)
+        return items[: self.cfg.max_panels]

agent/fc_agent/embedder.py

@@ -58,20 +58,12 @@ class CropEmbedder:
 
     def embed(self, image: Image.Image) -> list[float]:
         """A crop → its embedding as a plain float list, ready to POST."""
-        return self.embed_batch([image])[0]
-
-    def embed_batch(self, images: list) -> list[list[float]]:
-        """Embed many crops in ONE forward pass — far better GPU utilisation +
-        only one lock acquisition than embedding each crop separately (which
-        starved the GPU and serialised the whole pool)."""
-        if not images:
-            return []
         self.load()
         torch = self._torch
-        enc = self._processor(images=images, return_tensors="pt")
+        enc = self._processor(images=image, return_tensors="pt")
         pixel_values = enc["pixel_values"].to(self._device, self._dt)
         with self._infer_lock, torch.no_grad():
             out = self._model.get_image_features(pixel_values=pixel_values)
         pooled = out.pooler_output if hasattr(out, "pooler_output") else out
-        arr = pooled.float().cpu().numpy().astype(np.float32)
+        vec = pooled[0].float().cpu().numpy().astype(np.float32).reshape(-1)
-        return [row.reshape(-1).tolist() for row in arr]
+        return vec.tolist()

agent/fc_agent/gpu.py

@@ -1,24 +1,10 @@
 """GPU load readout via nvidia-smi (present in the container thanks to the
 NVIDIA Container Toolkit's `utility` capability). Returns None if unavailable —
-the UI just shows n/a (e.g. CPU-fallback run).
+the UI just shows n/a (e.g. CPU-fallback run)."""
-
-Reads are CACHED and de-duplicated: the UI meter polls fast, /status reads it,
-and the autoscaler samples it — if each spawned its own `nvidia-smi` (slow on a
-busy GPU) those blocking subprocesses would pile up in the server's thread pool
-and make the Start/Stop buttons feel dead. So a short TTL serves recent callers
-from cache, and only ONE probe runs at a time (others get the last value)."""
 import subprocess
-import threading
-import time
-
-_TTL = 1.0                       # seconds a sample is reused before re-probing
-_lock = threading.Lock()
-_cache: dict | None = None
-_cache_t = 0.0
-_probing = False
 
 
-def _probe() -> dict | None:
+def read_gpu() -> dict | None:
     try:
         out = subprocess.run(
             [
@@ -42,24 +28,3 @@ def _probe() -> dict | None:
         }
     except (ValueError, IndexError):
         return None
-
-
-def read_gpu(max_age: float = _TTL) -> dict | None:
-    """Latest GPU reading, cached. Serves from cache when fresh; when stale,
-    exactly one caller re-probes while the rest get the last value — so request
-    threads never block behind more than one `nvidia-smi`."""
-    global _cache, _cache_t, _probing
-    now = time.monotonic()
-    with _lock:
-        fresh = _cache is not None and (now - _cache_t) < max_age
-        if fresh or _probing:        # fresh, or a probe is already running
-            return _cache
-        _probing = True
-    try:
-        val = _probe()
-    finally:
-        with _lock:
-            _cache = val
-            _cache_t = time.monotonic()
-            _probing = False
-    return val

agent/fc_agent/logbuf.py

@@ -1,44 +0,0 @@
-"""In-memory log ring buffer so the control UI can show recent agent logs
-(detector loads, job errors, autoscaler decisions, outage back-offs) without
-needing `docker logs`. A bounded deque holds the last N formatted lines; a
-logging.Handler appends to it; the UI polls /logs."""
-import logging
-from collections import deque
-
-LINES: deque[str] = deque(maxlen=400)
-
-
-class RingHandler(logging.Handler):
-    def emit(self, record: logging.LogRecord) -> None:
-        try:
-            LINES.append(self.format(record))
-        except Exception:
-            pass
-
-
-_installed = False
-
-
-def install(level: int = logging.INFO) -> None:
-    """Attach the ring handler to the root logger once. fc_agent module loggers
-    propagate to root, so their records land here."""
-    global _installed
-    if _installed:
-        return
-    _installed = True
-    h = RingHandler()
-    h.setFormatter(
-        logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s", "%H:%M:%S")
-    )
-    root = logging.getLogger()
-    root.addHandler(h)
-    if root.level == logging.NOTSET or root.level > level:
-        root.setLevel(level)
-    # Keep the buffer signal-rich: silence the chatty HTTP/download libs (every
-    # HF model fetch logs per-request) so the console shows agent activity —
-    # detector loads, job errors, autoscale moves — not request spam.
-    for noisy in (
-        "uvicorn.access", "ultralytics", "httpx", "httpcore",
-        "huggingface_hub", "urllib3", "filelock",
-    ):
-        logging.getLogger(noisy).setLevel(logging.WARNING)

agent/fc_agent/media.py

@@ -2,77 +2,17 @@
 (ffmpeg) at the cadence FC sends — so a video becomes a bag of per-frame
 instances, each with a timestamp."""
 import io
-import logging
 import os
-import signal
 import subprocess
 import tempfile
-import time
 
-from PIL import Image, ImageFile
+from PIL import Image
-
-from .throttle import PidReadMeter
-
-log = logging.getLogger("fc_agent.media")
-
-# Load slightly-truncated images (a few missing trailing bytes) instead of
-# raising — matches the server embedder. These are common in scraped libraries
-# and would otherwise fail the job 3× then error (operator-flagged 2026-06-30).
-ImageFile.LOAD_TRUNCATED_IMAGES = True
-
-# Disable PIL's decompression-bomb guard: this is a TRUSTED local library, not an
-# untrusted upload surface, so a legitimately huge image (high-res scans/prints,
-# 90M+ pixels) must load. The default 89M-pixel limit only WARNS, but PIL raises
-# DecompressionBombError at 2× (~179M px) — which would fail those jobs outright
-# (operator-flagged 2026-06-30, images of 90–95M px).
-Image.MAX_IMAGE_PIXELS = None
 
 
 def is_video(mime: str) -> bool:
     return bool(mime) and (mime.startswith("video/") or mime in {"image/gif"})
 
 
-def _dhash(img: Image.Image, size: int = 8) -> int:
-    """Difference hash: compare adjacent pixels of a (size+1 × size) grayscale
-    thumbnail → a `size*size`-bit fingerprint. Cheap (64 comparisons on a 72-px
-    thumbnail) and robust to scaling/compression noise — near-identical frames
-    hash within a few bits, a real scene change moves many."""
-    small = img.convert("L").resize((size + 1, size))
-    px = list(small.getdata())
-    bits = 0
-    for row in range(size):
-        base = row * (size + 1)
-        for col in range(size):
-            bits = (bits << 1) | int(px[base + col] > px[base + col + 1])
-    return bits
-
-
-def dedupe_frames(
-    frames: list[tuple[float, Image.Image]], min_distance: int
-) -> list[tuple[float, Image.Image]]:
-    """Drop visually near-duplicate frames. A near-static video sampled into many
-    frames re-runs the WHOLE detect→CCIP→SigLIP chain on ~identical frames — the
-    dominant video load. Greedy perceptual-hash dedup: keep a frame only if its
-    dHash differs from every already-kept frame by >= min_distance bits (Hamming),
-    so a static run collapses to one frame while genuinely distinct scenes all
-    survive. Order + timestamps preserved. CPU-only (64-bit int XORs), so it runs
-    in the decode stage and spares the GPU the skipped frames entirely.
-
-    min_distance is the coarseness dial: higher keeps more frames (safer for brief
-    localized changes an 8×8 hash can miss), 0 disables. The first frame is always
-    kept (nothing to compare against)."""
-    if min_distance <= 0 or len(frames) <= 1:
-        return frames
-    kept: list[tuple[float, Image.Image]] = []
-    hashes: list[int] = []
-    for t, frame in frames:
-        h = _dhash(frame)
-        if all(bin(h ^ k).count("1") >= min_distance for k in hashes):
-            hashes.append(h)
-            kept.append((t, frame))
-    return kept
-
-
 def to_rgb(img: Image.Image) -> Image.Image:
     """RGB, flattening any transparency onto white first. A naive convert('RGB')
     on a palette-with-transparency image (common for character PNGs on a clear
@@ -92,162 +32,32 @@ def load_image(data: bytes) -> Image.Image:
     return to_rgb(Image.open(io.BytesIO(data)))
 
 
-# ffmpeg reconnect flags — resume a dropped HTTP transfer (a slow/contended media
+def sample_frames(
-# store can cut a long stream) instead of failing the whole job. Relies only on
+    data: bytes, interval_seconds: float, max_frames: int
-# HTTP + Range, which every FC deployment serves → environment-agnostic.
-_RECONNECT = [
-    "-reconnect", "1", "-reconnect_streamed", "1",
-    "-reconnect_on_network_error", "1", "-reconnect_delay_max", "5",
-]
-
-
-def _collect_frames(
-    tmp: str, interval: float, cap: int
 ) -> list[tuple[float, Image.Image]]:
-    out: list[tuple[float, Image.Image]] = []
+    """Extract up to max_frames frames at one-every-interval_seconds via ffmpeg.
-    names = sorted(n for n in os.listdir(tmp) if n.startswith("f_"))
+    Returns [(timestamp_seconds, frame)]. Empty on failure (caller falls back)."""
-    for i, name in enumerate(names[:cap]):
-        with Image.open(os.path.join(tmp, name)) as im:
-            out.append((round(i * interval, 2), to_rgb(im)))
-    return out
-
-
-def _terminate(proc: subprocess.Popen) -> None:
-    """Stop an ffmpeg cleanly, then hard-kill if it ignores SIGTERM."""
-    try:
-        # A bandwidth-paused (SIGSTOPped) process can't receive SIGTERM until it
-        # resumes — always CONT first so termination is prompt, not queued.
-        proc.send_signal(signal.SIGCONT)
-    except OSError:
-        pass
-    proc.terminate()
-    try:
-        proc.wait(timeout=2)
-    except subprocess.TimeoutExpired:
-        proc.kill()
-        try:
-            proc.wait(timeout=2)
-        except subprocess.TimeoutExpired:
-            pass
-
-
-def _pause(proc: subprocess.Popen, seconds: float, should_stop) -> bool:
-    """SIGSTOP ffmpeg for ~`seconds` of bandwidth debt, staying responsive to
-    Stop. While paused, the kernel socket buffer fills and TCP flow control
-    stalls curator's send side — that's the throttle. SIGCONT is ALWAYS sent
-    before returning. False = a Stop arrived mid-pause."""
-    try:
-        proc.send_signal(signal.SIGSTOP)
-    except OSError:
-        return True                     # already exited — nothing to pause
-    try:
-        end = time.monotonic() + seconds
-        while (left := end - time.monotonic()) > 0:
-            if should_stop and should_stop():
-                return False
-            time.sleep(min(0.5, left))
-        return True
-    finally:
-        try:
-            proc.send_signal(signal.SIGCONT)
-        except OSError:
-            pass
-
-
-def sample_frames_from_url(
-    url: str, interval_seconds: float, max_frames: int,
-    *, headers: str = "", timeout: float = 1200.0, should_stop=None,
-    governor=None,
-) -> tuple[list[tuple[float, Image.Image]], str | None]:
-    """Sample frames by pointing ffmpeg STRAIGHT at the media URL — it Range-reads
-    only the video index + up to max_frames worth of content, so the agent never
-    downloads the whole file (VR/4K originals run 800MB+ and would buffer ~1GB in
-    RAM and get cut off mid-download). Reconnect flags resume a dropped transfer;
-    the timeout is the per-video ceiling (a slow/reconnecting stream can otherwise
-    run for minutes). `should_stop` is polled while ffmpeg runs so a Stop KILLS the
-    subprocess at once — otherwise a downloader stuck in a long decode keeps the
-    agent "working" long after Stop. `governor` (the worker's shared TokenBucket)
-    meters ffmpeg's network reads from outside via /proc/<pid>/io and SIGSTOPs
-    the process into budget, so video streaming honors the same aggregate
-    bandwidth cap as still downloads.
-
-    Returns (frames, reason): frames is empty on failure/stop/timeout, and
-    `reason` then carries the SPECIFIC cause (ffmpeg's stderr tail / timeout) so
-    the caller can put it in the job's error — a bare "no frames" hid a filter
-    bug as "unprocessable" for weeks. None reason on success."""
     interval = max(0.5, float(interval_seconds or 4.0))
     cap = max(1, int(max_frames or 64))
-    hdr = ["-headers", headers] if headers else []
-    # select (NOT the fps filter): always keep the FIRST frame, then one per
-    # `interval` seconds of timestamp. fps=1/N emits round(duration/N) frames,
-    # which is ZERO for any clip shorter than ~N/2 seconds — a whole class of
-    # short animation loops failed as "unprocessable" that way (operator-flagged
-    # 2026-07-02: 0.5s/1.75s clips). scale=out_range=full converts limited-range
-    # yuv420p to full range so the mjpeg (jpg) encoder accepts it at default
-    # strictness instead of erroring on "non full-range YUV".
-    vf = (
-        f"select='isnan(prev_selected_t)+gte(t-prev_selected_t\\,{interval})',"
-        "scale=out_range=full"
-    )
     with tempfile.TemporaryDirectory() as tmp:
+        src = os.path.join(tmp, "in")
+        with open(src, "wb") as fh:
+            fh.write(data)
         pattern = os.path.join(tmp, "f_%05d.jpg")
-        cmd = ["ffmpeg", "-nostdin", "-loglevel", "error", *_RECONNECT, *hdr,
-               "-i", url, "-vf", vf, "-fps_mode", "vfr",
-               "-frames:v", str(cap), "-q:v", "3", pattern]
-        # ffmpeg's stderr goes to a file (not a PIPE, which could fill and
-        # deadlock; not DEVNULL, which is how a filter bug hid as "unprocessable"
-        # for weeks) — on failure its tail is logged so the operator can see WHY.
-        errpath = os.path.join(tmp, "stderr.txt")
         try:
-            with open(errpath, "wb") as errf:
+            subprocess.run(
-                proc = subprocess.Popen(
+                [
-                    cmd, stdin=subprocess.DEVNULL,
+                    "ffmpeg", "-nostdin", "-loglevel", "error", "-i", src,
-                    stdout=subprocess.DEVNULL, stderr=errf,
+                    "-vf", f"fps=1/{interval}", "-frames:v", str(cap),
-                )
+                    "-q:v", "3", pattern,
-                meter = PidReadMeter(proc.pid) if governor is not None else None
+                ],
-                # Poll rather than block, so a Stop (or the per-video timeout) can
+                check=True, timeout=600,
-                # kill a slow/wedged ffmpeg promptly instead of waiting it out.
+            )
-                start = time.monotonic()
+        except (subprocess.SubprocessError, FileNotFoundError):
-                while True:
+            return []
-                    try:
+        out: list[tuple[float, Image.Image]] = []
-                        proc.wait(timeout=0.5)
+        names = sorted(n for n in os.listdir(tmp) if n.startswith("f_"))
-                        break
+        for i, name in enumerate(names[:cap]):
-                    except subprocess.TimeoutExpired:
+            with Image.open(os.path.join(tmp, name)) as im:
-                        stopped = should_stop and should_stop()
+                out.append((round(i * interval, 2), to_rgb(im)))
-                        if stopped or (time.monotonic() - start > timeout):
+        return out
-                            _terminate(proc)
-                            if stopped:
-                                return [], "stopped"
-                            log.warning("ffmpeg timed out after %.0fs: %s",
-                                        timeout, url)
-                            return [], f"ffmpeg timed out after {timeout:.0f}s"
-                        if meter is not None:
-                            read = meter.delta()
-                            if read is None:      # /proc gone → stop governing
-                                meter = None
-                            elif (debt := governor.charge(read)) > 0:
-                                # Over budget: pause ffmpeg until the bucket
-                                # recovers. Pause time counts toward `timeout`
-                                # (it stays the wedge backstop either way).
-                                if not _pause(proc, debt, should_stop):
-                                    _terminate(proc)
-                                    return [], "stopped"
-        except (OSError, ValueError) as exc:
-            return [], f"ffmpeg not runnable: {exc}"
-        frames = _collect_frames(tmp, interval, cap)
-        if not frames:
-            reason = f"ffmpeg exit {proc.returncode}: {_tail(errpath)}"
-            log.warning("ffmpeg produced no frames for %s — %s", url, reason)
-            return [], reason
-        return frames, None
-
-
-def _tail(path: str, limit: int = 300) -> str:
-    """Last `limit` chars of a (stderr) file, flattened — for failure logs."""
-    try:
-        with open(path, "rb") as f:
-            f.seek(0, os.SEEK_END)
-            f.seek(max(0, f.tell() - limit))
-            return f.read().decode("utf-8", "replace").replace("\n", " ").strip()
-    except OSError:
-        return "?"

agent/fc_agent/throttle.py

@@ -1,111 +0,0 @@
-"""Global download-bandwidth governor (one token bucket for the whole agent).
-
-The agent lives on someone's desktop and shares that desktop's network —
-typically WiFi, where saturating the link doesn't just slow other apps: it
-bufferbloats the airtime (RTT 21→45ms) and collapses EVERY connection,
-the operator's browser included. Measured 2026-07-02: the idle link moved
-~38 MB/s single-stream, but under the 8-downloader sweep every stream on the
-machine crawled at ~1-1.5 MB/s. So the cap is on the AGGREGATE, not per
-stream: still downloads pump their chunks through take(), and ffmpeg video
-streams — whose sockets live in a subprocess we can't wrap — are metered from
-outside via /proc/<pid>/io and paused (SIGSTOP) into budget using charge()'s
-debt signal; TCP flow control then stalls the sender while ffmpeg sleeps.
-
-Accounting is post-paid (charge the bytes first, then wait out any debt): the
-bytes have already crossed the network by the time we count them, and it means
-a chunk larger than one second of budget can never deadlock the bucket.
-Stdlib-only on purpose — unit-tested in CI, where the agent's ML deps
-don't exist.
-"""
-import threading
-import time
-
-
-class TokenBucket:
-    """Thread-safe token bucket in bytes/second. rate 0 = unlimited.
-
-    `consumed` is the monotonic total of bytes charged (throttled or not) —
-    the worker's rate loop derives the UI's "net MB/s" readout from it.
-    """
-
-    def __init__(self, rate_bytes_per_s: float = 0.0):
-        self._cond = threading.Condition()
-        self._rate = max(0.0, float(rate_bytes_per_s))
-        # Burst = one second of budget: enough that chunked reads stay smooth,
-        # small enough that a burst can't meaningfully lift the average.
-        self._level = self._rate
-        self._stamp = time.monotonic()
-        self.consumed = 0
-
-    @property
-    def rate(self) -> float:
-        return self._rate
-
-    def set_rate(self, rate_bytes_per_s: float) -> None:
-        """Retune live (the UI dial). Waiters re-check immediately, so raising
-        the cap (or lifting it with 0) unblocks a mid-download wait at once."""
-        with self._cond:
-            self._refill_locked()          # settle elapsed time at the OLD rate
-            self._rate = max(0.0, float(rate_bytes_per_s))
-            self._level = min(self._level, self._rate)
-            self._cond.notify_all()
-
-    def _refill_locked(self) -> None:
-        now = time.monotonic()
-        self._level = min(self._rate, self._level + (now - self._stamp) * self._rate)
-        self._stamp = now
-
-    def take(self, n: int) -> None:
-        """Charge n bytes and block until the budget recovers (stills path)."""
-        with self._cond:
-            self.consumed += n
-            if self._rate <= 0:
-                return
-            self._refill_locked()
-            self._level -= n
-            while self._level < 0:
-                # Wake early on set_rate; cap the wait so a big debt is paid in
-                # re-checked slices rather than one long uninterruptible sleep.
-                self._cond.wait(min(-self._level / self._rate, 0.5))
-                if self._rate <= 0:
-                    return
-                self._refill_locked()
-
-    def charge(self, n: int) -> float:
-        """Charge n bytes WITHOUT blocking; return seconds of debt (0 = within
-        budget). The ffmpeg governor can't block the subprocess's own reads, so
-        it SIGSTOPs the process for (about) the returned debt instead."""
-        with self._cond:
-            self.consumed += n
-            if self._rate <= 0:
-                return 0.0
-            self._refill_locked()
-            self._level -= n
-            return max(0.0, -self._level / self._rate)
-
-
-class PidReadMeter:
-    """Cumulative read-bytes meter for a subprocess, via /proc/<pid>/io.
-
-    `rchar` counts every read() syscall's bytes — for a streaming ffmpeg the
-    network reads dominate, so the delta is a good-enough aggregate-bandwidth
-    signal (it's a governor, not a billing meter). Returns None when /proc is
-    unavailable (process exited, or a non-Linux host): the caller then simply
-    doesn't govern — degrade to unthrottled rather than break video sampling.
-    """
-
-    def __init__(self, pid: int):
-        self._path = f"/proc/{pid}/io"
-        self._last = 0
-
-    def delta(self) -> int | None:
-        try:
-            with open(self._path, "rb") as f:
-                for line in f:
-                    if line.startswith(b"rchar:"):
-                        total = int(line.split()[1])
-                        d, self._last = total - self._last, total
-                        return max(0, d)
-        except (OSError, ValueError):
-            return None
-        return None

agent/ruff.toml

@@ -1,7 +0,0 @@
-# The agent runs on the CUDA base image's Python 3.12 (Ubuntu 24.04) — NOT the
-# 3.14 that CI's ci-python image and the repo-root ruff.toml target. Pin the
-# agent to py312 so ruff enforces 3.12 compatibility and never auto-applies a
-# 3.14-only fix (e.g. unquoting a self-referential annotation, which PEP 649
-# makes safe on 3.14 but NameErrors on 3.12). Inherit the root lint rules.
-extend = "../ruff.toml"
-target-version = "py312"

alembic/versions/0070_gpu_job_lease_indexes.py

@@ -1,44 +0,0 @@
-"""partial indexes so GPU-job leasing stays O(batch), not O(completed)
-
-The lease claims the lowest-id pending (or expired-leased) jobs. With only a
-plain `status` index, `... ORDER BY id LIMIT n` walked the primary-key index from
-the start, skipping the entire prefix of already-done/error rows before reaching
-pending ones — so leasing slowed to a crawl as `done` piled up (the whole reason
-throughput fell off a cliff mid-run and /status stalled). Two partial indexes fix
-it: the pending one is id-ordered so the hot path reads just the first n entries,
-and the leased-expiry one keeps the crash-recovery reclaim + the orphan sweep
-cheap. They cover only the small live slice of the table, so they stay tiny even
-as the done/error history grows to millions.
-
-Revision ID: 0070
-Revises: 0069
-Create Date: 2026-06-30
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0070"
-down_revision: Union[str, None] = "0069"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # Hot path: lowest-id pending jobs. Index on id, restricted to pending, so
-    # `WHERE status='pending' ORDER BY id LIMIT n` is a short index-order scan.
-    op.create_index(
-        "ix_gpu_job_pending", "gpu_job", ["id"],
-        postgresql_where=sa.text("status = 'pending'"),
-    )
-    # Crash-recovery: expired leases, for the lease backstop + recover_orphaned.
-    op.create_index(
-        "ix_gpu_job_leased_expires", "gpu_job", ["lease_expires_at"],
-        postgresql_where=sa.text("status = 'leased'"),
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_gpu_job_leased_expires", table_name="gpu_job")
-    op.drop_index("ix_gpu_job_pending", table_name="gpu_job")

alembic/versions/0071_image_record_earliest_post_date.py

@@ -1,80 +0,0 @@
-"""image_record.earliest_post_date: original-publish gallery sort key + index
-
-Revision ID: 0071
-Revises: 0070
-Create Date: 2026-07-01
-
-effective_date (0035) keys off the PRIMARY post — which is often the repost /
-download the file actually came from — and falls back to created_at, so the
-gallery's default order surfaces download dates rather than when content was
-first posted (operator-flagged 2026-07-01). Materialize a second sort key,
-earliest_post_date = MIN(post_date) across ALL of an image's provenance posts
-(every post it appears in), falling back to created_at only when no linked post
-carries a date. Indexed (DESC, id DESC) so the "post date" gallery sort is an
-index range scan just like effective_date.
-
-Backfill mirrors 0035: created_at baseline, then override with the MIN over
-image_provenance ⋈ post. New rows get the created_at-equivalent server default;
-services/importer.py recomputes it whenever a dated post is linked.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0071"
-down_revision: Union[str, None] = "0070"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    # Add nullable first so the backfill can populate before NOT NULL.
-    op.add_column(
-        "image_record",
-        sa.Column("earliest_post_date", sa.DateTime(timezone=True), nullable=True),
-    )
-    # Baseline: download date. Set-based (no per-row binds) → immune to the
-    # 65535 bind-parameter ceiling regardless of library size.
-    op.execute(
-        """
-        UPDATE image_record
-        SET earliest_post_date = created_at
-        """
-    )
-    # Override with the earliest post_date across EVERY post the image appears
-    # in (image_provenance is the many-to-many edge; ignore posts with no date).
-    op.execute(
-        """
-        UPDATE image_record AS ir
-        SET earliest_post_date = sub.min_date
-        FROM (
-            SELECT ip.image_record_id AS iid, MIN(p.post_date) AS min_date
-            FROM image_provenance AS ip
-            JOIN post AS p ON p.id = ip.post_id
-            WHERE p.post_date IS NOT NULL
-            GROUP BY ip.image_record_id
-        ) AS sub
-        WHERE ir.id = sub.iid
-        """
-    )
-    op.alter_column(
-        "image_record",
-        "earliest_post_date",
-        nullable=False,
-        server_default=sa.text("now()"),
-    )
-    # DESC/DESC matches the gallery's ORDER BY earliest_post_date DESC, id DESC
-    # so the "post date" scroll is a forward index scan; raw SQL because
-    # alembic's column list doesn't express per-column DESC cleanly.
-    op.execute(
-        "CREATE INDEX ix_image_record_earliest_post_date "
-        "ON image_record (earliest_post_date DESC, id DESC)"
-    )
-
-
-def downgrade() -> None:
-    op.drop_index(
-        "ix_image_record_earliest_post_date", table_name="image_record"
-    )
-    op.drop_column("image_record", "earliest_post_date")

alembic/versions/0072_gpu_job_triage_status.py

@@ -1,32 +0,0 @@
-"""gpu_job.triage_status — the probe's verdict on an errored job's FILE
-
-Failure triage (#125): a periodic sweep probes each errored image's file
-(sha256 + decode, verify_integrity's machinery) exactly once and stores the
-verdict here — 'defect' (the file is bad: recovery material, excluded from
-/retry_errors) or 'file_ok' (failure was operational, safe to retry). NULL
-means not yet probed; selecting on NULL is what makes the sweep resumable.
-No index: the errored slice the sweep scans is tiny by design (tombstones).
-
-Revision ID: 0072
-Revises: 0071
-Create Date: 2026-07-02
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0072"
-down_revision: Union[str, None] = "0071"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "gpu_job", sa.Column("triage_status", sa.String(16), nullable=True)
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("gpu_job", "triage_status")

alembic/versions/0073_drop_tag_eval_run.py

@@ -1,46 +0,0 @@
-"""drop tag_eval_run — the head-vs-centroid eval harness is retired
-
-The eval (#1130) existed to prove the heads tagging spine on the operator's own
-data. It did; the operator accepted the system and retired the harness
-(2026-07-02) — card, API, task, model and this table all go. The eval's data
-loaders + metric helpers live on in services/ml/training_data.py, where the
-production heads trainer uses them nightly.
-
-Revision ID: 0073
-Revises: 0072
-Create Date: 2026-07-02
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-from sqlalchemy.dialects import postgresql
-
-revision: str = "0073"
-down_revision: Union[str, None] = "0072"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.drop_index("ix_tag_eval_run_status", table_name="tag_eval_run")
-    op.drop_table("tag_eval_run")
-
-
-def downgrade() -> None:
-    # Recreates the shape from 0056 (data is not restorable).
-    op.create_table(
-        "tag_eval_run",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column("params", postgresql.JSONB(), nullable=False),
-        sa.Column("status", sa.String(length=16), nullable=False,
-                  server_default="running"),
-        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False,
-                  server_default=sa.func.now()),
-        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("report", postgresql.JSONB(), nullable=True),
-        sa.Column("error", sa.Text(), nullable=True),
-        sa.Column("last_progress_at", sa.DateTime(timezone=True),
-                  nullable=True),
-    )
-    op.create_index("ix_tag_eval_run_status", "tag_eval_run", ["status"])

alembic/versions/0074_ml_settings_cpu_embed_enabled.py

@@ -1,35 +0,0 @@
-"""ml_settings.cpu_embed_enabled — the CPU embed fallback becomes a switch
-
-B3 (operator 2026-07-02): the ml-worker's only processing role is the CPU
-whole-image embed for stacks without a GPU agent. ON by default (a fresh
-install works agent-less); agent-equipped stacks that drop the ml-worker
-container turn it off so import hooks stop queueing embed work into a queue
-nothing consumes — the daily GPU 'embed' backfill covers those images.
-
-Revision ID: 0074
-Revises: 0073
-Create Date: 2026-07-02
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0074"
-down_revision: Union[str, None] = "0073"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "cpu_embed_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.true(),
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "cpu_embed_enabled")

alembic/versions/0075_tag_is_system.py

@@ -1,60 +0,0 @@
-"""tag.is_system + seed the three hygiene system tags
-
-Training hygiene (operator 2026-07-03, milestone #128): rough WIPs tagged as a
-character poison that character's head and CCIP references; banners/editor
-screenshots pollute whole-image similarity. The fix keys on SYSTEM tags the
-product ships — not operator configuration — so the seed lives here.
-
-Seeding ADOPTS an existing same-(name, kind=general) tag (case-insensitive,
-matching TagService.rename's collision stance) instead of inserting a
-duplicate, so an operator who already tagged `wip` keeps their applications.
-
-Revision ID: 0075
-Revises: 0074
-Create Date: 2026-07-03
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0075"
-down_revision: Union[str, None] = "0074"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot")
-
-
-def upgrade() -> None:
-    op.add_column(
-        "tag",
-        sa.Column(
-            "is_system", sa.Boolean(), nullable=False,
-            server_default=sa.false(),
-        ),
-    )
-    conn = op.get_bind()
-    for name in SYSTEM_TAG_NAMES:
-        adopted = conn.execute(
-            sa.text(
-                "UPDATE tag SET is_system = true "
-                "WHERE lower(name) = lower(:name) AND kind = 'general'"
-            ),
-            {"name": name},
-        )
-        if adopted.rowcount == 0:
-            conn.execute(
-                sa.text(
-                    "INSERT INTO tag (name, kind, is_system) "
-                    "VALUES (:name, 'general', true)"
-                ),
-                {"name": name},
-            )
-
-
-def downgrade() -> None:
-    # The seeded rows survive as ordinary general tags — dropping the flag is
-    # enough to disarm the mechanism, and deleting rows would orphan any
-    # operator applications made while the flag existed.
-    op.drop_column("tag", "is_system")

alembic/versions/0076_pixiv_ledgers.py

@@ -1,82 +0,0 @@
-"""pixiv_seen_media + pixiv_failed_media: per-source ledgers
-
-Revision ID: 0076
-Revises: 0075
-Create Date: 2026-07-03
-
-Pixiv native ingester (milestone #129, gallery-dl → native-core migration).
-Mirrors the Patreon (0037/0038) and SubscribeStar (0054) ledger tables: a
-seen-ledger so routine walks skip already-ingested media (recovery bypasses
-it) and a dead-letter ledger so persistently-failing media stops re-burning
-backfill chunks. Pixiv URLs carry no content hash, so `filehash` is always the
-synthesized ``<illust_id>:p<num>`` / ``<illust_id>:ugoira`` key — String(128)
-matches the siblings. UNIQUE (source_id, filehash) is the upsert key on each.
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0076"
-down_revision: Union[str, None] = "0075"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "pixiv_seen_media",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "source_id",
-            sa.Integer,
-            sa.ForeignKey("source.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column("filehash", sa.String(128), nullable=False),
-        sa.Column("post_id", sa.String(64), nullable=True),
-        sa.Column(
-            "seen_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-        sa.UniqueConstraint(
-            "source_id", "filehash", name="uq_pixiv_seen_media_source_id"
-        ),
-    )
-    op.create_table(
-        "pixiv_failed_media",
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column(
-            "source_id",
-            sa.Integer,
-            sa.ForeignKey("source.id", ondelete="CASCADE"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column("filehash", sa.String(128), nullable=False),
-        sa.Column("attempts", sa.Integer, nullable=False, server_default="1"),
-        sa.Column("last_error", sa.Text, nullable=True),
-        sa.Column(
-            "first_failed_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-        sa.Column(
-            "last_failed_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("NOW()"),
-        ),
-        sa.UniqueConstraint(
-            "source_id", "filehash", name="uq_pixiv_failed_media_source_id"
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_table("pixiv_failed_media")
-    op.drop_table("pixiv_seen_media")

alembic/versions/0077_artist_name_not_unique.py

@@ -1,32 +0,0 @@
-"""drop uq_artist_name — decouple display name from identity/storage
-
-Revision ID: 0077
-Revises: 0076
-Create Date: 2026-07-04
-
-Artist model fragility fix (milestone #130). One `slug` column was doing
-identity + storage-path + display, and BOTH `name` and `slug` were UNIQUE, so
-the display name couldn't be edited freely and two genuinely different creators
-collided. Decouple: `slug` stays the immutable, unique storage/identity key (the
-on-disk path component — untouched here); `name` becomes freely editable, NON-
-unique display text. This migration only drops the `uq_artist_name` constraint;
-no data moves and no path changes.
-"""
-from typing import Sequence, Union
-
-from alembic import op
-
-revision: str = "0077"
-down_revision: Union[str, None] = "0076"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.drop_constraint("uq_artist_name", "artist", type_="unique")
-
-
-def downgrade() -> None:
-    # Re-adding the UNIQUE would fail if duplicate names now exist; callers that
-    # need to reverse this must dedupe names first.
-    op.create_unique_constraint("uq_artist_name", "artist", ["name"])

alembic/versions/0078_ml_settings_detectors.py

@@ -1,83 +0,0 @@
-"""ml_settings crop-proposer / detector config (#134)
-
-Move the WHERE-to-crop detector config (per-proposer enable + weights + conf,
-plus caps + dedupe IoU) into the DB so it's UI-tunable and announced to the GPU
-agent in the lease (like the embedder model) — no restart, agent env is now
-bootstrap-only. All server_defaults are the working values so existing rows +
-fresh installs crop out-of-the-box with all three proposers ON.
-
-Revision ID: 0078
-Revises: 0077
-Create Date: 2026-07-05
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0078"
-down_revision: Union[str, None] = "0077"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-_ANATOMY_DEFAULT = (
-    "https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt"
-)
-_PANEL_DEFAULT = "mosesb/best-comic-panel-detection::best.pt"
-
-
-def upgrade() -> None:
-    op.add_column("ml_settings", sa.Column(
-        "detector_person_enabled", sa.Boolean(), nullable=False,
-        server_default=sa.true()))
-    op.add_column("ml_settings", sa.Column(
-        "detector_person_weights", sa.String(512), nullable=False,
-        server_default="yolo11n.pt"))
-    op.add_column("ml_settings", sa.Column(
-        "detector_person_conf", sa.Float(), nullable=False,
-        server_default=sa.text("0.35")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_anatomy_enabled", sa.Boolean(), nullable=False,
-        server_default=sa.true()))
-    op.add_column("ml_settings", sa.Column(
-        "detector_anatomy_weights", sa.String(512), nullable=False,
-        server_default=_ANATOMY_DEFAULT))
-    op.add_column("ml_settings", sa.Column(
-        "detector_anatomy_conf", sa.Float(), nullable=False,
-        server_default=sa.text("0.30")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_panel_enabled", sa.Boolean(), nullable=False,
-        server_default=sa.true()))
-    op.add_column("ml_settings", sa.Column(
-        "detector_panel_weights", sa.String(512), nullable=False,
-        server_default=_PANEL_DEFAULT))
-    op.add_column("ml_settings", sa.Column(
-        "detector_panel_conf", sa.Float(), nullable=False,
-        server_default=sa.text("0.30")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_max_figures", sa.Integer(), nullable=False,
-        server_default=sa.text("8")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_max_components", sa.Integer(), nullable=False,
-        server_default=sa.text("8")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_max_panels", sa.Integer(), nullable=False,
-        server_default=sa.text("8")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_max_regions", sa.Integer(), nullable=False,
-        server_default=sa.text("128")))
-    op.add_column("ml_settings", sa.Column(
-        "detector_dedupe_iou", sa.Float(), nullable=False,
-        server_default=sa.text("0.85")))
-
-
-def downgrade() -> None:
-    for col in (
-        "detector_person_enabled", "detector_person_weights", "detector_person_conf",
-        "detector_anatomy_enabled", "detector_anatomy_weights", "detector_anatomy_conf",
-        "detector_panel_enabled", "detector_panel_weights", "detector_panel_conf",
-        "detector_max_figures", "detector_max_components", "detector_max_panels",
-        "detector_max_regions", "detector_dedupe_iou",
-    ):
-        op.drop_column("ml_settings", col)

alembic/versions/0079_character_prototypes.py

@@ -1,77 +0,0 @@
-"""character prototype store (#1317) — precomputed, incremental CCIP references
-
-New tables character_prototype + ccip_prototype_state, plus MLSettings columns
-ccip_ref_signature (cheap global change gate) + ccip_prototype_cap (per-character
-reference cap). The reference set the CCIP matcher uses becomes a precomputed
-artifact refreshed incrementally off the request path. See milestone 138 /
-backend.app.services.ml.character_prototypes.
-
-Revision ID: 0079
-Revises: 0078
-Create Date: 2026-07-06
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-from pgvector.sqlalchemy import Vector
-
-revision: str = "0079"
-down_revision: Union[str, None] = "0078"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-# Matches models.image_region.CCIP_DIM (the CCIP figure-embedding width).
-_CCIP_DIM = 768
-
-
-def upgrade() -> None:
-    op.create_table(
-        "character_prototype",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column(
-            "tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="CASCADE"), nullable=False,
-        ),
-        sa.Column("ccip_embedding", Vector(_CCIP_DIM), nullable=False),
-        sa.Column(
-            "region_id", sa.Integer(),
-            sa.ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True,
-        ),
-    )
-    op.create_index(
-        "ix_character_prototype_tag_id", "character_prototype", ["tag_id"]
-    )
-    op.create_table(
-        "ccip_prototype_state",
-        sa.Column(
-            "tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True,
-        ),
-        sa.Column("fingerprint", sa.String(64), nullable=False),
-        sa.Column(
-            "updated_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column("ccip_ref_signature", sa.String(128), nullable=True),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "ccip_prototype_cap", sa.Integer(), nullable=False,
-            server_default=sa.text("64"),
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("ml_settings", "ccip_prototype_cap")
-    op.drop_column("ml_settings", "ccip_ref_signature")
-    op.drop_table("ccip_prototype_state")
-    op.drop_index(
-        "ix_character_prototype_tag_id", table_name="character_prototype"
-    )
-    op.drop_table("character_prototype")

alembic/versions/0080_tag_head_train_fingerprint.py

@@ -1,31 +0,0 @@
-"""tag_head.train_fingerprint (#1317 phase 2) — incremental head retraining
-
-A per-head training-data fingerprint (positive + rejection count/latest-timestamp)
-so a manual Retrain refits only the tags whose data changed; the nightly run
-ignores it (full reconcile). Nullable — a NULL fingerprint (existing heads) forces
-a refit on the first incremental run, then it's stamped.
-
-Revision ID: 0080
-Revises: 0079
-Create Date: 2026-07-06
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0080"
-down_revision: Union[str, None] = "0079"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "tag_head",
-        sa.Column("train_fingerprint", sa.String(128), nullable=True),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("tag_head", "train_fingerprint")

alembic/versions/0081_stricter_auto_apply_defaults.py

@@ -1,43 +0,0 @@
-"""stricter auto-apply defaults (milestone 139) — cut auto-apply misfires
-
-head_auto_apply_min_positives 30→50 and ccip_auto_apply_threshold 0.92→0.95
-(operator-asked 2026-07-06). The head graduation precision bar stays 0.97 — the
-operator confirmed the general-tag confidence was already well tuned; only the
-support floor + the CCIP match confidence are raised. The model defaults change
-for fresh installs; here we bump the existing singleton row IFF it is still at
-the previous default, so a deliberate operator change is NOT clobbered.
-
-Revision ID: 0081
-Revises: 0080
-Create Date: 2026-07-06
-"""
-from typing import Sequence, Union
-
-from alembic import op
-
-revision: str = "0081"
-down_revision: Union[str, None] = "0080"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.execute(
-        "UPDATE ml_settings SET head_auto_apply_min_positives = 50 "
-        "WHERE head_auto_apply_min_positives = 30"
-    )
-    op.execute(
-        "UPDATE ml_settings SET ccip_auto_apply_threshold = 0.95 "
-        "WHERE ccip_auto_apply_threshold = 0.92"
-    )
-
-
-def downgrade() -> None:
-    op.execute(
-        "UPDATE ml_settings SET head_auto_apply_min_positives = 30 "
-        "WHERE head_auto_apply_min_positives = 50"
-    )
-    op.execute(
-        "UPDATE ml_settings SET ccip_auto_apply_threshold = 0.92 "
-        "WHERE ccip_auto_apply_threshold = 0.95"
-    )

alembic/versions/0082_presentation_auto_hide.py

@@ -1,85 +0,0 @@
-"""presentation-chrome auto-hide (#141) — settings knobs + review table
-
-MLSettings gains presentation_auto_apply_enabled / _threshold and
-presentation_conflict_threshold: banner + editor-screenshot auto-hide on the
-sweep with a FLAT threshold (decoupled from content-head graduation), and a
-conflict threshold that flags an auto-hide that "also looks like content".
-
-New table presentation_review records an auto-hidden chrome image that also
-scored high on a content head, surfaced in the Hidden view for a keep-hidden /
-un-hide decision. Resolved rows are pruned by retention.
-
-Revision ID: 0082
-Revises: 0081
-Create Date: 2026-07-07
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0082"
-down_revision: Union[str, None] = "0081"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "presentation_auto_apply_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.text("true"),
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "presentation_auto_apply_threshold", sa.Float(), nullable=False,
-            server_default=sa.text("0.90"),
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "presentation_conflict_threshold", sa.Float(), nullable=False,
-            server_default=sa.text("0.50"),
-        ),
-    )
-    op.create_table(
-        "presentation_review",
-        sa.Column(
-            "image_record_id", sa.Integer(),
-            sa.ForeignKey("image_record.id", ondelete="CASCADE"),
-            primary_key=True,
-        ),
-        sa.Column(
-            "tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True,
-        ),
-        sa.Column(
-            "conflict_tag_id", sa.Integer(),
-            sa.ForeignKey("tag.id", ondelete="SET NULL"), nullable=True,
-        ),
-        sa.Column("conflict_score", sa.Float(), nullable=False),
-        sa.Column(
-            "created_at", sa.DateTime(timezone=True), nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
-    )
-    # The review list queries the unresolved flags (resolved_at IS NULL).
-    op.create_index(
-        "ix_presentation_review_resolved_at", "presentation_review",
-        ["resolved_at"],
-    )
-
-
-def downgrade() -> None:
-    op.drop_index(
-        "ix_presentation_review_resolved_at", table_name="presentation_review"
-    )
-    op.drop_table("presentation_review")
-    op.drop_column("ml_settings", "presentation_conflict_threshold")
-    op.drop_column("ml_settings", "presentation_auto_apply_threshold")
-    op.drop_column("ml_settings", "presentation_auto_apply_enabled")

alembic/versions/0083_post_translation.py

@@ -1,73 +0,0 @@
-"""post-text translation via Interpreter (milestone 143) — Post columns + settings
-
-Post gains the translated title/description + the detected source language,
-Interpreter engine_version (cache key), and translated_at — filled by the
-translate sweep. ImportSettings gains translation_enabled (OFF by default),
-interpreter_base_url (EMPTY — the operator sets their own, behind a reverse
-proxy), and translation_target_lang (en).
-
-Revision ID: 0083
-Revises: 0082
-Create Date: 2026-07-07
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0083"
-down_revision: Union[str, None] = "0082"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "post", sa.Column("post_title_translated", sa.Text(), nullable=True)
-    )
-    op.add_column(
-        "post", sa.Column("description_translated", sa.Text(), nullable=True)
-    )
-    op.add_column(
-        "post",
-        sa.Column("translated_source_lang", sa.String(8), nullable=True),
-    )
-    op.add_column(
-        "post",
-        sa.Column("translation_engine_version", sa.String(128), nullable=True),
-    )
-    op.add_column(
-        "post",
-        sa.Column("translated_at", sa.DateTime(timezone=True), nullable=True),
-    )
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "translation_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.text("false"),
-        ),
-    )
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "interpreter_base_url", sa.Text(), nullable=False, server_default="",
-        ),
-    )
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "translation_target_lang", sa.Text(), nullable=False,
-            server_default="en",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("import_settings", "translation_target_lang")
-    op.drop_column("import_settings", "interpreter_base_url")
-    op.drop_column("import_settings", "translation_enabled")
-    op.drop_column("post", "translated_at")
-    op.drop_column("post", "translation_engine_version")
-    op.drop_column("post", "translated_source_lang")
-    op.drop_column("post", "description_translated")
-    op.drop_column("post", "post_title_translated")

alembic/versions/0084_translation_strictness_override.py

@@ -1,51 +0,0 @@
-"""translation strictness setting + per-post translation override (milestone 155)
-
-ImportSettings gains ``translation_min_confidence`` (the latin-script acceptance
-floor, now operator-tunable in the UI; default 0.9 — stricter than the old
-hardcoded 0.8, since Interpreter confidently mis-detects short ASCII English at
-~0.86). Post gains ``translation_override`` — a sticky per-post choice of
-auto / force / original so the operator can force a skipped translation on, or
-knock a wrongly-translated one back to the original, and have it survive a
-Re-translate-all.
-
-Revision ID: 0084
-Revises: 0083
-Create Date: 2026-07-10
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0084"
-down_revision: Union[str, None] = "0083"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "translation_min_confidence", sa.Float(), nullable=False,
-            server_default=sa.text("0.9"),
-        ),
-    )
-    op.add_column(
-        "post",
-        sa.Column(
-            "translation_override", sa.String(16), nullable=False,
-            server_default="auto",
-        ),
-    )
-    op.create_check_constraint(
-        "ck_post_translation_override",
-        "post",
-        "translation_override IN ('auto', 'force', 'original')",
-    )
-
-
-def downgrade() -> None:
-    op.drop_constraint("ck_post_translation_override", "post", type_="check")
-    op.drop_column("post", "translation_override")
-    op.drop_column("import_settings", "translation_min_confidence")

alembic/versions/0085_wip_title_tagging.py

@@ -1,35 +0,0 @@
-"""title-based WIP auto-tagging (task #1458) — ImportSettings toggle
-
-ImportSettings gains wip_title_tagging_enabled (ON by default): when a freshly
-imported post's title explicitly declares work-in-progress ("WIP" / "work in
-progress"), the importer applies the `wip` system tag to its images. No new
-table — the tag itself is the seeded `wip` system tag (migration 0075) and the
-application reuses image_tag with source='wip_title'.
-
-Revision ID: 0085
-Revises: 0084
-Create Date: 2026-07-12
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0085"
-down_revision: Union[str, None] = "0084"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "import_settings",
-        sa.Column(
-            "wip_title_tagging_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.text("true"),
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("import_settings", "wip_title_tagging_enabled")

alembic/versions/0086_process_auto_apply_settings.py

@@ -1,61 +0,0 @@
-"""process auto-apply settings + review mode (#1464) — system-tag refactor
-
-The system-tag behavior refactor gives `wip` / `editor screenshot` (the PROCESS
-group) their own provisional auto-apply, parallel to the presentation (chrome)
-sweep. MLSettings gains three knobs: enabled (OFF by default — a new whole-library
-auto-tagger is opt-in), the flat apply threshold, and the ring-loud conflict
-threshold. presentation_review gains a `mode` column so one review surface serves
-both chrome and process flags (existing rows backfill 'chrome'). server_defaults
-so the existing rows fill cleanly.
-
-Revision ID: 0086
-Revises: 0085
-Create Date: 2026-07-13
-"""
-from typing import Sequence, Union
-
-import sqlalchemy as sa
-from alembic import op
-
-revision: str = "0086"
-down_revision: Union[str, None] = "0085"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "process_auto_apply_enabled", sa.Boolean(), nullable=False,
-            server_default=sa.text("false"),
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "process_auto_apply_threshold", sa.Float(), nullable=False,
-            server_default="0.90",
-        ),
-    )
-    op.add_column(
-        "ml_settings",
-        sa.Column(
-            "process_conflict_threshold", sa.Float(), nullable=False,
-            server_default="0.50",
-        ),
-    )
-    op.add_column(
-        "presentation_review",
-        sa.Column(
-            "mode", sa.String(16), nullable=False,
-            server_default="chrome",
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("presentation_review", "mode")
-    op.drop_column("ml_settings", "process_conflict_threshold")
-    op.drop_column("ml_settings", "process_auto_apply_threshold")
-    op.drop_column("ml_settings", "process_auto_apply_enabled")

backend/app/__init__.py

@@ -34,23 +34,6 @@ def create_app() -> Quart:
     app = Quart(__name__)
     app.secret_key = cfg.secret_key
 
-    # Stream files in 4 MiB chunks instead of Quart's 8 KiB default. The image
-    # library lives on a CIFS/SMB share (mounted rsize=4 MiB), so 8 KiB reads
-    # meant ~19k network round-trips for one large original — 30–58s downloads
-    # that starved both the GPU agent and the browser (operator-flagged
-    # 2026-07-01). 4 MiB matches the mount's read size → one round-trip per read,
-    # ~500× fewer. buffer_size is the MAX read, so small thumbnails still read in
-    # a single gulp, and Range/mime/ETag/conditional handling lives on Response,
-    # so this keeps all of it. Guarded so a future Quart-internal change can't
-    # break boot — worst case we fall back to the slow default.
-    try:
-        from quart.wrappers.response import FileBody
-        FileBody.buffer_size = 4 * 1024 * 1024
-    except Exception:
-        logging.getLogger(__name__).warning(
-            "could not raise FileBody.buffer_size — file serving stays on 8 KiB chunks"
-        )
-
     for bp in all_blueprints():
         app.register_blueprint(bp)
     # Registered last so /api/* routes win over the SPA catch-all.

backend/app/api/__init__.py

@@ -38,6 +38,7 @@ def all_blueprints() -> list[Blueprint]:
     from .suggestions import suggestions_bp
     from .system_activity import system_activity_bp
     from .system_backup import system_backup_bp
+    from .tag_eval import tag_eval_bp
     from .tags import tags_bp
     from .thumbnails import thumbnails_bp
     return [
@@ -57,6 +58,7 @@ def all_blueprints() -> list[Blueprint]:
         import_admin_bp,
         suggestions_bp,
         aliases_bp,
+        tag_eval_bp,
         heads_bp,
         gpu_bp,
         ccip_bp,

backend/app/api/admin.py

@@ -1,13 +1,13 @@
 """FC-3k: /api/admin — destructive admin actions.
 
-Action surfaces:
+Five action surfaces:
   POST   /api/admin/artists/<slug>/cascade-delete  (Tier C)
   POST   /api/admin/images/bulk-delete             (Tier C)
   DELETE /api/admin/tags/<int:tag_id>              (Tier B)
   POST   /api/admin/tags/<int:dest_id>/merge       (Tier B)
   POST   /api/admin/tags/prune-unused              (Tier A)
   POST   /api/admin/posts/prune-bare               (Tier A)
-  POST   /api/admin/posts/refetch-external         (Tier A)
+  POST   /api/admin/tags/purge-legacy              (Tier A)
   GET    /api/admin/tags/<int:tag_id>/usage-count  (helper)
 
 Tier-C ops take a dry_run body flag (returns projection inline,
@@ -23,7 +23,7 @@ from quart import Blueprint, jsonify, request
 from sqlalchemy import select, text
 
 from ..extensions import get_session
-from ..models import Artist, Post
+from ..models import Artist
 from ..services.cleanup_service import project_artist_cascade, project_bulk_image_delete
 from ._responses import error_response as _bad
 
@@ -156,10 +156,6 @@ async def tag_delete(tag_id: int):
             )
         except LookupError:
             return _bad("not_found", status=404)
-        except ValueError as exc:
-            # System tags (#128) — the training-hygiene machinery keys on
-            # these rows.
-            return _bad("system_tag", detail=str(exc))
     return jsonify(result)
 
 
@@ -281,86 +277,31 @@ async def posts_reconcile_duplicates():
     return await _run_dry_run_op(reconcile_duplicate_posts, source_id=source_id)
 
 
-@admin_bp.route("/posts/refetch-external", methods=["POST"])
+@admin_bp.route("/tags/purge-legacy", methods=["POST"])
-async def posts_refetch_external():
+async def tags_purge_legacy():
-    """Surgical re-fetch of a post's external file-host links (operator
+    """Tier-A: delete legacy IR-migration tags — archive/post/artist
-    2026-07-03): the normal cadence never re-walks deep back-catalogue posts,
+    kinds (e.g. `BlenderKnight:Hannah_BJ_Loops`) PLUS general tags with
-    so a deleted external file only comes back by resetting its ExternalLink
+    a legacy name prefix (`source:*`, from IR's source kind that fell
-    row(s) — this endpoint does that per post and dispatches the fetches.
+    back to general). dry-run preview returns per-kind + per-prefix
-    Sha-dedupe discards payload files that still exist, so only what's
+    counts + a sample so the UI shows exactly what'll go before the
-    missing lands. Body: {external_post_id: str, source_id?: int (to
+    operator confirms with dry_run=false."""
-    disambiguate the same external id across sources)}."""
+    from ..services.cleanup_service import purge_legacy_tags
-    from ..services.external_links import refetch_links_for_post
 
-    body = await request.get_json(silent=True) or {}
+    return await _run_dry_run_op(purge_legacy_tags)
-    ext_id = str(body.get("external_post_id") or "").strip()
-    if not ext_id:
-        return _bad("missing_external_post_id",
-                    detail="external_post_id is required")
-    raw_source = body.get("source_id")
-    try:
-        source_id = int(raw_source) if raw_source is not None else None
-    except (TypeError, ValueError):
-        return _bad("invalid_source_id", detail="source_id must be an integer")
-
-    async with get_session() as session:
-        stmt = select(Post.id).where(Post.external_post_id == ext_id)
-        if source_id is not None:
-            stmt = stmt.where(Post.source_id == source_id)
-        post_ids = (await session.execute(stmt)).scalars().all()
-        if not post_ids:
-            return _bad("post_not_found", status=404,
-                        detail=f"no post with external_post_id {ext_id!r}")
-        results = {}
-        for pid in post_ids:
-            results[str(pid)] = await session.run_sync(
-                lambda s, p=pid: refetch_links_for_post(s, p)
-            )
-    return jsonify({"posts": results})
-
-
-def _reset_content_confirm_token(projection: dict) -> str:
-    """Stable 8-hex token derived from the live counts (mirrors the Tier-C
-    bulk-delete token): it changes whenever the data changes, so the apply can
-    only ever run against numbers the operator just previewed."""
-    canon = f"reset-content:{projection.get('count')}:{projection.get('applications')}"
-    return hashlib.sha256(canon.encode("utf-8")).hexdigest()[:8]
 
 
 @admin_bp.route("/tags/reset-content", methods=["POST"])
 async def tags_reset_content():
-    """Full-instance reset of the CONTENT vocabulary: deletes ALL general +
+    """Tier-A: delete ALL general + character tags (the Camie-suggestable
-    character tags and their image applications — INCLUDING the examples the
+    content vocabulary) so the operator can re-tag from scratch via
-    tagging heads learned from. Suggestions do NOT repopulate on their own
+    auto-suggest. fandom + series tags + series_page ordering are preserved,
-    (the Camie predictions that once did are long retired): the operator
+    and image_prediction rows are untouched so suggestions repopulate.
-    re-tags from scratch and the heads retrain from the new signal. fandom +
+    dry-run preview returns per-kind counts + applications + a sample so the
-    series tags + series_page ordering are preserved.
+    UI shows exactly what'll go before the operator confirms (dry_run=false).
-
+    Irreversible except via DB backup restore."""
-    Deliberately Tier-C-gated despite the Tier-A shape (operator 2026-07-02:
-    the full reset stays, but behind extra steps): dry_run returns the
-    projection + a `confirm` token derived from the live counts; the apply
-    must echo that token back or it is rejected."""
     from ..services.cleanup_service import reset_content_tagging
 
-    body = await request.get_json(silent=True) or {}
+    return await _run_dry_run_op(reset_content_tagging)
-    dry_run = bool(body.get("dry_run", False))
-    async with get_session() as session:
-        projection = await session.run_sync(
-            lambda s: reset_content_tagging(s, dry_run=True)
-        )
-        token = _reset_content_confirm_token(projection)
-        if dry_run:
-            projection["confirm"] = token
-            return jsonify(projection)
-        if str(body.get("confirm", "")) != token:
-            return _bad(
-                "confirm_mismatch",
-                detail="run a fresh preview and echo its confirm token",
-            )
-        result = await session.run_sync(
-            lambda s: reset_content_tagging(s, dry_run=False)
-        )
-    return jsonify(result)
 
 
 @admin_bp.route("/tags/normalize", methods=["POST"])

backend/app/api/artists.py

@@ -31,24 +31,6 @@ async def create_or_get():
     }), 201
 
 
-@artists_bp.route("/<int:artist_id>", methods=["PATCH"])
-async def rename(artist_id: int):
-    """Rename an artist's DISPLAY NAME (#130). Name only — the slug and every
-    on-disk path stay put, so this is instant and safe. Name is non-unique."""
-    body = await request.get_json()
-    if not isinstance(body, dict) or not isinstance(body.get("name"), str):
-        return jsonify({"error": "invalid_body"}), 400
-    async with get_session() as session:
-        svc = ArtistService(session)
-        try:
-            artist = await svc.rename(artist_id, body["name"])
-        except ValueError as exc:
-            return jsonify({"error": "empty_name", "detail": str(exc)}), 400
-        if artist is None:
-            return jsonify({"error": "not_found"}), 404
-        return jsonify({"id": artist.id, "name": artist.name, "slug": artist.slug})
-
-
 @artists_bp.route("/autocomplete", methods=["GET"])
 async def autocomplete():
     q = request.args.get("q") or ""

backend/app/api/extension.py

@@ -84,15 +84,11 @@ async def quick_add_source():
     if not isinstance(url, str) or not url.strip():
         return _bad("invalid_body", detail="url is required")
 
-    from .credentials import _get_crypto
-
     async with get_session() as session:
         if not await _ext_key_required(session):
             return _bad("unauthorized", status=401)
         try:
-            # crypto lets a pixiv add resolve the artist's display name via the
+            result = await ExtensionService(session).quick_add_source(url)
-            # stored OAuth token (else it falls back to the numeric id). #130.
-            result = await ExtensionService(session, _get_crypto()).quick_add_source(url)
         except UnknownPlatformError as exc:
             return _bad(
                 "unknown_platform",

backend/app/api/gallery.py

@@ -3,18 +3,9 @@
 from datetime import UTC, datetime, timedelta
 
 from quart import Blueprint, jsonify, request
-from sqlalchemy import delete, select, update
-from sqlalchemy.orm import aliased
 
 from ..extensions import get_session
-from ..models import (
+from ..services.gallery_service import GalleryService
-    ImageRecord,
-    PresentationReview,
-    Tag,
-    TagSuggestionRejection,
-)
-from ..models.tag import image_tag
-from ..services.gallery_service import GalleryService, image_url, thumbnail_url
 
 gallery_bp = Blueprint("gallery", __name__, url_prefix="/api/gallery")
 
@@ -53,8 +44,7 @@ def _parse_filters():
       the image must match at least one tag from EACH group (groups ANDed).
     - `tag_not` is a comma-separated exclude list (image must carry none).
 
-    `media` is image|video; `sort` is newest|oldest|posted_new|posted_old
+    `media` is image|video; `sort` is newest|oldest; `platform` selects one
-    (default posted_new); `platform` selects one
     platform (or the UNSOURCED_PLATFORM sentinel); `untagged`/`no_artist` are
     boolean flags; `date_from`/`date_to` are inclusive calendar-day bounds
     (date_to is widened by a day so the whole day is covered by the service's
@@ -77,18 +67,11 @@ def _parse_filters():
     artist_id = int(artist_id_raw) if artist_id_raw else None
     media = request.args.get("media")
     media_type = media if media in ("image", "video") else None
-    # newest/oldest key off effective_date (primary post / download); posted_new/
-    # posted_old off earliest_post_date (original publish across all posts). The
-    # default is posted_new so the grid leads with original publish date, not the
-    # download/repost the primary post points at (operator-flagged 2026-07-01).
     sort = request.args.get("sort")
-    sort = sort if sort in ("newest", "oldest", "posted_new", "posted_old") else "posted_new"
+    sort = sort if sort in ("newest", "oldest") else "newest"
     platform = request.args.get("platform") or None
     untagged = request.args.get("untagged") in ("1", "true", "yes")
     no_artist = request.args.get("no_artist") in ("1", "true", "yes")
-    # Show the presentation chrome (banner / editor screenshot) that the default
-    # gallery hides — the Hidden view sets this (milestone 141).
-    include_hidden = request.args.get("include_hidden") in ("1", "true", "yes")
     date_from = _parse_date(request.args.get("date_from"))
     date_to = _parse_date(request.args.get("date_to"))
     if date_to is not None:
@@ -100,7 +83,6 @@ def _parse_filters():
         "platform": platform,
         "untagged": untagged, "no_artist": no_artist,
         "date_from": date_from, "date_to": date_to,
-        "include_hidden": include_hidden,
     }
     return filters, sort
 
@@ -145,20 +127,12 @@ async def similar():
         filters, _sort = _parse_filters()
     except (KeyError, ValueError):
         return jsonify({"error": "similar_to query param required"}), 400
-    # Explore passes exclude_wip=1 to also drop work-in-progress from the
-    # rabbit-hole; the gallery's own "similar" button omits it (keeps wip, #1274).
-    exclude_wip = request.args.get("exclude_wip") in ("1", "true", "True")
     # post_id is the exclusive post-detail view — not a similarity scope.
-    # include_hidden is a gallery-browse flag; similar() has its OWN presentation
+    scope = {k: v for k, v in filters.items() if k != "post_id"}
-    # exclusion (a similarity-quality concern, #1274), so drop it here (#141).
-    scope = {
-        k: v for k, v in filters.items() if k not in ("post_id", "include_hidden")
-    }
     async with get_session() as session:
         svc = GalleryService(session)
         try:
-            images = await svc.similar(
+            images = await svc.similar(image_id=similar_to, limit=limit, **scope)
-                image_id=similar_to, limit=limit, exclude_wip=exclude_wip, **scope)
         except ValueError as exc:
             return jsonify({"error": str(exc)}), 400
     if images is None:
@@ -232,101 +206,6 @@ async def jump():
     return jsonify({"cursor": cursor})
 
 
-# -- Hidden-view review (#141): auto-hidden chrome flagged "also looks like
-#    content", surfaced in the gallery's Show-hidden review strip. -----------
-@gallery_bp.route("/hidden-review", methods=["GET"])
-async def hidden_review():
-    """Unresolved system-tag auto-apply review flags (chrome + process, #1464),
-    most-concerning first (highest content score) — for the review strip. `mode`
-    tells the client whether the flagged tag hid the image ('chrome') or left it
-    visible ('process'), which decides the resolve labels (un-hide vs remove-tag)."""
-    ptag = aliased(Tag)
-    ctag = aliased(Tag)
-    async with get_session() as session:
-        rows = (await session.execute(
-            select(
-                PresentationReview.image_record_id,
-                PresentationReview.tag_id,
-                PresentationReview.conflict_tag_id,
-                PresentationReview.conflict_score,
-                PresentationReview.mode,
-                ImageRecord.path, ImageRecord.thumbnail_path,
-                ImageRecord.sha256, ImageRecord.mime,
-                ptag.name.label("tag_name"),
-                ctag.name.label("conflict_name"),
-            )
-            .join(ImageRecord, ImageRecord.id == PresentationReview.image_record_id)
-            .join(ptag, ptag.id == PresentationReview.tag_id)
-            .outerjoin(ctag, ctag.id == PresentationReview.conflict_tag_id)
-            .where(PresentationReview.resolved_at.is_(None))
-            .order_by(PresentationReview.conflict_score.desc())
-        )).all()
-    return jsonify({"items": [
-        {
-            "image_id": r.image_record_id,
-            "tag_id": r.tag_id,
-            "tag_name": r.tag_name,
-            "conflict_tag_id": r.conflict_tag_id,
-            "conflict_name": r.conflict_name,
-            "conflict_score": r.conflict_score,
-            "mode": r.mode,
-            "thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime),
-            "image_url": image_url(r.path),
-        }
-        for r in rows
-    ]})
-
-
-@gallery_bp.route(
-    "/hidden-review/<int:image_id>/<int:tag_id>/keep", methods=["POST"]
-)
-async def hidden_review_keep(image_id, tag_id):
-    """Keep the auto-hide: resolve the flag; the tag stays applied (#141)."""
-    async with get_session() as session:
-        await session.execute(
-            update(PresentationReview)
-            .where(
-                PresentationReview.image_record_id == image_id,
-                PresentationReview.tag_id == tag_id,
-            )
-            .values(resolved_at=datetime.now(UTC))
-        )
-        await session.commit()
-    return "", 204
-
-
-@gallery_bp.route(
-    "/hidden-review/<int:image_id>/<int:tag_id>/unhide", methods=["POST"]
-)
-async def hidden_review_unhide(image_id, tag_id):
-    """Un-hide: remove the presentation tag (image returns to the gallery), record
-    a rejection so the head LEARNS it misfired, and resolve the flag (#141)."""
-    from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-    async with get_session() as session:
-        await session.execute(
-            delete(image_tag).where(
-                image_tag.c.image_record_id == image_id,
-                image_tag.c.tag_id == tag_id,
-            )
-        )
-        await session.execute(
-            pg_insert(TagSuggestionRejection)
-            .values(image_record_id=image_id, tag_id=tag_id)
-            .on_conflict_do_nothing()
-        )
-        await session.execute(
-            update(PresentationReview)
-            .where(
-                PresentationReview.image_record_id == image_id,
-                PresentationReview.tag_id == tag_id,
-            )
-            .values(resolved_at=datetime.now(UTC))
-        )
-        await session.commit()
-    return "", 204
-
-
 @gallery_bp.route("/image/<int:image_id>", methods=["GET"])
 async def image_detail(image_id: int):
     async with get_session() as session:

backend/app/api/gpu.py

@@ -9,25 +9,19 @@ homelab admin.
 """
 
 import secrets
-from pathlib import Path
 
 from quart import Blueprint, jsonify, request
-from sqlalchemy import func, or_, select, update
+from sqlalchemy import func, select
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 
 from ..extensions import get_session
 from ..models import AppSetting, GpuJob, ImageRecord, MLSettings
 from ..services.gallery_service import image_url
-from ..services.ml.gpu_jobs import GpuJobService, error_dedupe_statements
+from ..services.ml.gpu_jobs import GpuJobService
-from ..services.ml.gpu_triage import classify_reason, recover_defective_image
 from ..services.ml.regions import RegionService
 
 gpu_bp = Blueprint("gpu", __name__, url_prefix="/api/gpu")
 
-# Same container mount the maintenance tasks use (tasks/admin.py) — recovery
-# deletes the defective original + thumbnail under it.
-_IMAGES_ROOT = Path("/images")
-
 _TOKEN_KEY = "gpu_agent_token"
 
 
@@ -96,7 +90,7 @@ async def backfill():
     """Enqueue a job for every image that doesn't already have one for `task`."""
     body = await request.get_json(silent=True) or {}
     task = str(body.get("task") or "ccip")
-    from ..tasks.gpu_queue import enqueue_gpu_backfill
+    from ..tasks.ml import enqueue_gpu_backfill
 
     r = enqueue_gpu_backfill.delay(task)
     return jsonify({"celery_task_id": r.id, "task": task}), 202
@@ -109,139 +103,12 @@ async def reprocess():
     detectors). Heavy — the back-catalogue is otherwise skipped by the backfills."""
     body = await request.get_json(silent=True) or {}
     task = str(body.get("task") or "ccip")
-    from ..tasks.gpu_queue import reprocess_gpu_jobs
+    from ..tasks.ml import reprocess_gpu_jobs
 
     r = reprocess_gpu_jobs.delay(task)
     return jsonify({"celery_task_id": r.id, "task": task}), 202
 
 
-@gpu_bp.route("/retry_errors", methods=["POST"])
-async def retry_errors():
-    """Requeue every ERRORED job (all task types) back to pending — the scoped
-    recovery after an agent-side fix (e.g. the short-video sampler), where
-    /reprocess would needlessly re-run the whole done library too. Attempts and
-    the stored error reset so each job gets its full retry budget under the
-    fixed pipeline. Stale tombstones are pruned FIRST (loop-era duplicates and
-    rows a later success made moot — the same statements the backfills run), so
-    one failing file requeues as ONE job, never a fan-out of duplicates. Small
-    row count (errors only) → inline statements; the response carries the
-    counts for the UI toast. Triage-confirmed defects are NOT requeued (see
-    the WHERE below) — they stay on the recovery surface."""
-    async with get_session() as session:
-        pruned = 0
-        for stmt in error_dedupe_statements():
-            pruned += (await session.execute(stmt)).rowcount or 0
-        res = await session.execute(
-            update(GpuJob)
-            .where(
-                GpuJob.status == "error",
-                # Triage-confirmed DEFECTS stay errored: the integrity probe
-                # already proved the FILE is bad, so re-running the job just
-                # burns agent time re-minting the same tombstone — those go
-                # through /errors/<id>/recover instead.
-                or_(GpuJob.triage_status.is_(None),
-                    GpuJob.triage_status != "defect"),
-            )
-            .values(
-                status="pending", attempts=0, error=None, lease_token=None,
-                leased_at=None, lease_expires_at=None, triage_status=None,
-                updated_at=func.now(),
-            )
-        )
-        kept = (
-            await session.execute(
-                select(func.count()).select_from(GpuJob)
-                .where(GpuJob.status == "error")
-            )
-        ).scalar_one()
-        await session.commit()
-    return jsonify({
-        "requeued": res.rowcount or 0, "pruned": pruned, "defects_kept": kept,
-    })
-
-
-# --- Failure triage + recovery (#125) ------------------------------------
-
-@gpu_bp.route("/errors", methods=["GET"])
-async def errors():
-    """The triage view of the error tombstones: every errored job joined with
-    its image's integrity verdict, bucketed by reason for the overview. The
-    probe sweep (triage_gpu_errors, 15-min beat) fills triage_status; 'defect'
-    rows are the recovery surface's list."""
-    async with get_session() as session:
-        rows = (
-            await session.execute(
-                select(
-                    GpuJob.id, GpuJob.image_record_id, GpuJob.task,
-                    GpuJob.error, GpuJob.triage_status, GpuJob.updated_at,
-                    ImageRecord.integrity_status, ImageRecord.mime,
-                    ImageRecord.path, ImageRecord.thumbnail_path,
-                )
-                .join(ImageRecord, ImageRecord.id == GpuJob.image_record_id)
-                .where(GpuJob.status == "error")
-                .order_by(GpuJob.updated_at.desc())
-                .limit(500)
-            )
-        ).all()
-        total = (
-            await session.execute(
-                select(func.count()).select_from(GpuJob)
-                .where(GpuJob.status == "error")
-            )
-        ).scalar_one()
-    by_class: dict[str, int] = {}
-    triage = {"defect": 0, "file_ok": 0, "unclassified": 0}
-    items = []
-    for r in rows:
-        cls = classify_reason(r.error)
-        by_class[cls] = by_class.get(cls, 0) + 1
-        bucket = r.triage_status or "unclassified"
-        triage[bucket] = triage.get(bucket, 0) + 1
-        items.append({
-            "job_id": r.id,
-            "image_id": r.image_record_id,
-            "task": r.task,
-            "error": r.error,
-            "reason_class": cls,
-            "triage_status": r.triage_status,
-            "integrity_status": r.integrity_status,
-            "mime": r.mime,
-            "image_url": image_url(r.path),
-            "thumbnail_url": (
-                image_url(r.thumbnail_path) if r.thumbnail_path else None
-            ),
-            "updated_at": r.updated_at.isoformat() if r.updated_at else None,
-        })
-    return jsonify({
-        "total": total, "by_class": by_class, "triage": triage, "items": items,
-    })
-
-
-@gpu_bp.route("/errors/triage", methods=["POST"])
-async def errors_triage():
-    """Run the probe sweep NOW (the card's button) instead of waiting out the
-    15-minute beat cadence."""
-    from ..tasks.maintenance import triage_gpu_errors
-
-    r = triage_gpu_errors.delay()
-    return jsonify({"celery_task_id": r.id}), 202
-
-
-@gpu_bp.route("/errors/<int:image_id>/recover", methods=["POST"])
-async def errors_recover(image_id: int):
-    """Recover a defect-triaged original: delete the bad copy + record and
-    re-poll its subscription Source (a fresh fetch re-imports the file, which
-    re-enters the GPU pipeline). Returns status 'no_source' when nothing
-    pollable resolves — the file needs manual replacement there."""
-    async with get_session() as session:
-        result = await session.run_sync(
-            lambda s: recover_defective_image(
-                s, image_id, images_root=_IMAGES_ROOT,
-            )
-        )
-    return jsonify(result)
-
-
 # --- Agent (bearer token): lease / submit / heartbeat / fail ------------
 
 @gpu_bp.route("/jobs/lease", methods=["POST"])
@@ -269,33 +136,6 @@ async def lease():
             ).scalars()
         } if ids else {}
         await session.commit()
-        # Crop-proposer config, announced FROM THE SETTING like embed_model_name
-        # (#134): the agent builds its detectors from this, rebuilding live when
-        # it changes — so tuning is a DB/UI edit, never an agent restart. Same
-        # block for every job in the batch (it's global), built once. An enabled
-        # toggle off is carried through so the agent skips that proposer.
-        detectors = {
-            "person": {
-                "enabled": ml.detector_person_enabled,
-                "weights": ml.detector_person_weights,
-                "conf": ml.detector_person_conf,
-            },
-            "anatomy": {
-                "enabled": ml.detector_anatomy_enabled,
-                "weights": ml.detector_anatomy_weights,
-                "conf": ml.detector_anatomy_conf,
-            },
-            "panel": {
-                "enabled": ml.detector_panel_enabled,
-                "weights": ml.detector_panel_weights,
-                "conf": ml.detector_panel_conf,
-            },
-            "max_figures": ml.detector_max_figures,
-            "max_components": ml.detector_max_components,
-            "max_panels": ml.detector_max_panels,
-            "max_regions": ml.detector_max_regions,
-            "dedupe_iou": ml.detector_dedupe_iou,
-        }
         out = []
         for j in jobs:
             img = imgs.get(j.image_record_id)
@@ -317,7 +157,6 @@ async def lease():
                 # re-embed, never an agent change.
                 "embed_model_name": ml.embedder_model_name,
                 "embed_version": ml.embedder_model_version,
-                "detectors": detectors,
             })
     return jsonify({"jobs": out})
 

backend/app/api/ml_admin.py

@@ -8,28 +8,7 @@ from ..models import MLSettings
 ml_admin_bp = Blueprint("ml_admin", __name__, url_prefix="/api/ml")
 
 
-# Crop-proposer / detector config (#134). Announced to the GPU agent in the lease
-# → tunable here with no restart. weights = ultralytics name | URL | hf_repo::file
-# (empty, or enabled off, skips that proposer).
-_DETECTOR_FIELDS = (
-    "detector_person_enabled",
-    "detector_person_weights",
-    "detector_person_conf",
-    "detector_anatomy_enabled",
-    "detector_anatomy_weights",
-    "detector_anatomy_conf",
-    "detector_panel_enabled",
-    "detector_panel_weights",
-    "detector_panel_conf",
-    "detector_max_figures",
-    "detector_max_components",
-    "detector_max_panels",
-    "detector_max_regions",
-    "detector_dedupe_iou",
-)
-
 _EDITABLE = (
-    "cpu_embed_enabled",
     "video_frame_interval_seconds",
     "video_max_frames",
     "head_min_positives",
@@ -39,15 +18,8 @@ _EDITABLE = (
     "ccip_match_threshold",
     "ccip_auto_apply_enabled",
     "ccip_auto_apply_threshold",
-    "presentation_auto_apply_enabled",
-    "presentation_auto_apply_threshold",
-    "presentation_conflict_threshold",
-    "process_auto_apply_enabled",
-    "process_auto_apply_threshold",
-    "process_conflict_threshold",
     "embedder_model_name",
     "embedder_model_version",
-    *_DETECTOR_FIELDS,
 )
 
 
@@ -91,7 +63,6 @@ async def get_settings():
         ).scalar_one()
     return jsonify(
         {
-            "cpu_embed_enabled": s.cpu_embed_enabled,
             "video_frame_interval_seconds": s.video_frame_interval_seconds,
             "video_max_frames": s.video_max_frames,
             "embedder_model_version": s.embedder_model_version,
@@ -102,14 +73,7 @@ async def get_settings():
             "ccip_match_threshold": s.ccip_match_threshold,
             "ccip_auto_apply_enabled": s.ccip_auto_apply_enabled,
             "ccip_auto_apply_threshold": s.ccip_auto_apply_threshold,
-            "presentation_auto_apply_enabled": s.presentation_auto_apply_enabled,
-            "presentation_auto_apply_threshold": s.presentation_auto_apply_threshold,
-            "presentation_conflict_threshold": s.presentation_conflict_threshold,
-            "process_auto_apply_enabled": s.process_auto_apply_enabled,
-            "process_auto_apply_threshold": s.process_auto_apply_threshold,
-            "process_conflict_threshold": s.process_conflict_threshold,
             "embedder_model_name": s.embedder_model_name,
-            **{f: getattr(s, f) for f in _DETECTOR_FIELDS},
         }
     )
 
@@ -162,36 +126,11 @@ def _validate(p: dict) -> str | None:
         return "ccip_match_threshold must be between 0.5 and 0.999"
     if not (0.5 <= float(p["ccip_auto_apply_threshold"]) <= 0.999):
         return "ccip_auto_apply_threshold must be between 0.5 and 0.999"
-    # Presentation chrome auto-hide (#141). Auto-apply runs high (hiding is
-    # consequential); the conflict cut is a plain probability [0,1].
-    if not (0.5 <= float(p["presentation_auto_apply_threshold"]) <= 0.999):
-        return "presentation_auto_apply_threshold must be between 0.5 and 0.999"
-    if not (0.0 <= float(p["presentation_conflict_threshold"]) <= 1.0):
-        return "presentation_conflict_threshold must be between 0 and 1"
-    # Process auto-apply (#1464). wip/editor stay VISIBLE so a false apply is
-    # low-harm (excludes-from-training + a review flag), but keep the same bar.
-    if not (0.5 <= float(p["process_auto_apply_threshold"]) <= 0.999):
-        return "process_auto_apply_threshold must be between 0.5 and 0.999"
-    if not (0.0 <= float(p["process_conflict_threshold"]) <= 1.0):
-        return "process_conflict_threshold must be between 0 and 1"
     # Embedder model swap (#1190): both must be non-empty. Changing them means a
     # different embedding space — the operator must re-embed + retrain after.
     for key in ("embedder_model_name", "embedder_model_version"):
         if not str(p[key]).strip():
             return f"{key} must not be empty"
-    # Crop proposers (#134). Weights may be empty (that proposer is just off);
-    # confidences are probabilities; caps are positive counts; IoU is [0,1].
-    for key in ("detector_person_conf", "detector_anatomy_conf", "detector_panel_conf"):
-        if not (0.0 <= float(p[key]) <= 1.0):
-            return f"{key} must be between 0 and 1"
-    for key in (
-        "detector_max_figures", "detector_max_components",
-        "detector_max_panels", "detector_max_regions",
-    ):
-        if int(p[key]) < 1:
-            return f"{key} must be >= 1"
-    if not (0.0 <= float(p["detector_dedupe_iou"]) <= 1.0):
-        return "detector_dedupe_iou must be between 0 and 1"
     return None
 
 

backend/app/api/posts.py

@@ -3,28 +3,12 @@
 from quart import Blueprint, jsonify, request
 
 from ..extensions import get_session
-from ..models import ImportSettings, Post
-from ..services import interpreter_client as ic
 from ..services.post_feed_service import PostFeedService
 from ..services.source_service import KNOWN_PLATFORMS
-from ..utils.text import html_to_plain
 from ._responses import error_response as _bad
 
 posts_bp = Blueprint("posts", __name__, url_prefix="/api/posts")
 
-_TRANSLATION_OVERRIDES = ("auto", "force", "original")
-
-
-def _queue_for_sweep(post: Post) -> None:
-    """Mark a post untranslated (all translation columns NULL) so the periodic
-    sweep re-runs it under its new override — used when Interpreter is down and we
-    can't translate inline."""
-    post.post_title_translated = None
-    post.description_translated = None
-    post.translated_source_lang = None
-    post.translation_engine_version = None
-    post.translated_at = None
-
 
 @posts_bp.route("", methods=["GET"])
 async def list_posts():
@@ -98,70 +82,3 @@ async def get_post(post_id: int):
     if item is None:
         return _bad("not_found", status=404, detail=f"post id={post_id}")
     return jsonify(item)
-
-
-@posts_bp.route("/<int:post_id>/translation-override", methods=["POST"])
-async def set_translation_override(post_id: int):
-    """Sticky per-post translation override (milestone 155). Body:
-    ``{"override": "auto" | "force" | "original"}``.
-
-    'original' keeps the original (clears any stored translation now — no
-    Interpreter needed). 'force'/'auto' translate the post immediately if the
-    service is up (force bypasses the acceptance floor; auto re-runs the gate);
-    if it's down we save the flag and mark the post untranslated so the next sweep
-    applies it. The override persists, so the sweep + Re-translate-all keep
-    honoring it. Returns the updated translation fields + an ``applied`` status."""
-    body = await request.get_json(silent=True) or {}
-    override = body.get("override")
-    if override not in _TRANSLATION_OVERRIDES:
-        return _bad(
-            "invalid_override",
-            detail=f"override must be one of {list(_TRANSLATION_OVERRIDES)}",
-        )
-
-    # Lazy import (mirrors settings.py) so the API module doesn't pull the celery
-    # task graph at import time.
-    from ..tasks.translation import _store_translation, _translate_field
-
-    async with get_session() as session:
-        post = await session.get(Post, post_id)
-        if post is None:
-            return _bad("not_found", status=404, detail=f"post id={post_id}")
-        post.translation_override = override
-        cfg = await ImportSettings.load(session)
-        target = (cfg.translation_target_lang or "en").strip() or "en"
-
-        if override == "original":
-            _store_translation(post, (None, None, None), (None, None, None), target)
-            applied = "cleared"
-        else:
-            base_url = (cfg.interpreter_base_url or "").strip()
-            if cfg.translation_enabled and base_url and ic.health(base_url):
-                force = override == "force"
-                title = (post.post_title or "").strip()
-                desc = (html_to_plain(post.description) if post.description else "") or ""
-                desc = desc.strip()
-                mc = cfg.translation_min_confidence
-                try:
-                    title_res = _translate_field(title, base_url, target, mc, force=force)
-                    desc_res = _translate_field(desc, base_url, target, mc, force=force)
-                except ic.InterpreterUnavailable:
-                    _queue_for_sweep(post)
-                    applied = "queued"
-                else:
-                    _store_translation(post, title_res, desc_res, target)
-                    applied = "translated"
-            else:
-                # Disabled / no URL / unhealthy → let the sweep apply it later.
-                _queue_for_sweep(post)
-                applied = "queued"
-
-        await session.commit()
-        return jsonify({
-            "id": post.id,
-            "translation_override": post.translation_override,
-            "post_title_translated": post.post_title_translated,
-            "description_translated": post.description_translated,
-            "translated_source_lang": post.translated_source_lang,
-            "applied": applied,
-        })

backend/app/api/settings.py

@@ -1,24 +1,12 @@
 """Settings API: import filters, system stats."""
 
-import asyncio
 import secrets
 
 from quart import Blueprint, jsonify, request
-from sqlalchemy import func, or_, select
+from sqlalchemy import func, select
 
 from ..extensions import get_session
-from ..models import (
+from ..models import AppSetting, Artist, ImageRecord, ImportBatch, ImportSettings, ImportTask, Tag
-    AppSetting,
-    Artist,
-    ImageRecord,
-    ImportBatch,
-    ImportSettings,
-    ImportTask,
-    Post,
-    Tag,
-    TaskRun,
-)
-from ..services import interpreter_client as ic
 
 settings_bp = Blueprint("settings", __name__, url_prefix="/api")
 
@@ -44,11 +32,6 @@ _EDITABLE_FIELDS = (
     "extdl_mediafire_enabled",
     "extdl_dropbox_enabled",
     "extdl_pixeldrain_enabled",
-    "translation_enabled",
-    "interpreter_base_url",
-    "translation_target_lang",
-    "translation_min_confidence",
-    "wip_title_tagging_enabled",
 )
 
 # Per-host external-download toggles — all plain booleans, validated uniformly.
@@ -86,11 +69,6 @@ async def get_import_settings():
         "extdl_mediafire_enabled": row.extdl_mediafire_enabled,
         "extdl_dropbox_enabled": row.extdl_dropbox_enabled,
         "extdl_pixeldrain_enabled": row.extdl_pixeldrain_enabled,
-        "translation_enabled": row.translation_enabled,
-        "interpreter_base_url": row.interpreter_base_url,
-        "translation_target_lang": row.translation_target_lang,
-        "translation_min_confidence": row.translation_min_confidence,
-        "wip_title_tagging_enabled": row.wip_title_tagging_enabled,
     })
 
 
@@ -150,35 +128,12 @@ async def update_import_settings():
     for tog in _EXTDL_TOGGLE_FIELDS:
         if tog in body and not isinstance(body[tog], bool):
             return jsonify({"error": f"{tog} must be a boolean"}), 400
-    # Translation (#143): base URL may be empty (feature off until set — no
-    # default host; the operator points it at their own Interpreter proxy).
-    if "translation_enabled" in body and not isinstance(
-        body["translation_enabled"], bool
-    ):
-        return jsonify({"error": "translation_enabled must be a boolean"}), 400
-    for key in ("interpreter_base_url", "translation_target_lang"):
-        if key in body and not isinstance(body[key], str):
-            return jsonify({"error": f"{key} must be a string"}), 400
-    # Acceptance floor (milestone 155): latin-script translations below this
-    # Interpreter confidence are kept as the original.
-    if "translation_min_confidence" in body:
-        v = body["translation_min_confidence"]
-        if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
-            return jsonify(
-                {"error": "translation_min_confidence must be a number in [0, 1]"}
-            ), 400
     if "series_suggest_threshold" in body:
         v = body["series_suggest_threshold"]
         if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
             return jsonify(
                 {"error": "series_suggest_threshold must be a number in [0, 1]"}
             ), 400
-    if "wip_title_tagging_enabled" in body and not isinstance(
-        body["wip_title_tagging_enabled"], bool
-    ):
-        return jsonify(
-            {"error": "wip_title_tagging_enabled must be a boolean"}
-        ), 400
 
     async with get_session() as session:
         row = await ImportSettings.load(session)
@@ -190,18 +145,6 @@ async def update_import_settings():
     return await get_import_settings()
 
 
-@settings_bp.route("/settings/wip-title/scan", methods=["POST"])
-async def wip_title_scan():
-    """Enqueue the back-catalogue WIP-title scan (task #1458 Settings button):
-    apply the `wip` system tag to EXISTING posts whose title declares
-    work-in-progress. New imports are tagged live by the importer; this catches
-    the existing library. Returns the Celery task id (202)."""
-    from ..tasks.maintenance import backfill_wip_title_tags
-
-    r = backfill_wip_title_tags.delay()
-    return jsonify({"celery_task_id": r.id}), 202
-
-
 @settings_bp.route("/system/stats", methods=["GET"])
 async def system_stats():
     async with get_session() as session:
@@ -327,159 +270,3 @@ async def rotate_extension_api_key():
             row.value = new_value
         await session.commit()
     return jsonify({"key": new_value})
-
-
-# --- Translation (#143): live status + manual "Translate now" --------------
-
-
-@settings_bp.route("/settings/translation/status", methods=["GET"])
-async def translation_status():
-    """For the Settings card: is it on, is a URL set, is the service reachable,
-    and how many posts still await translation. Health runs the sync client in a
-    thread so the event loop isn't blocked."""
-    translation_tasks = (
-        "backend.app.tasks.translation.translate_posts",
-        "backend.app.tasks.translation.retranslate_posts",
-    )
-    async with get_session() as session:
-        cfg = await ImportSettings.load(session)
-        untranslated = (await session.execute(
-            select(func.count(Post.id))
-            .where(Post.translated_source_lang.is_(None))
-            .where(or_(
-                Post.post_title.is_not(None), Post.description.is_not(None),
-            ))
-        )).scalar_one()
-        # Live progress: is a sweep running now, and what did the last one do?
-        # (run-until-done re-enqueues itself, so `active` stays true across a
-        # bulk re-translate; `last_run` surfaces a completed run's outcome.)
-        active = (await session.execute(
-            select(func.count(TaskRun.id))
-            .where(TaskRun.task_name.in_(translation_tasks))
-            .where(TaskRun.status == "running")
-        )).scalar_one()
-        last = (await session.execute(
-            select(TaskRun.task_name, TaskRun.status, TaskRun.finished_at)
-            .where(TaskRun.task_name.in_(translation_tasks))
-            .where(TaskRun.finished_at.is_not(None))
-            .order_by(TaskRun.finished_at.desc())
-            .limit(1)
-        )).first()
-    base_url = (cfg.interpreter_base_url or "").strip()
-    healthy = await asyncio.to_thread(ic.health, base_url) if base_url else False
-    return jsonify({
-        "enabled": cfg.translation_enabled,
-        "base_url_set": bool(base_url),
-        "healthy": healthy,
-        "untranslated_count": int(untranslated),
-        "active": int(active) > 0,
-        "last_run": {
-            "task": last[0].rsplit(".", 1)[-1],
-            "status": last[1],
-            "finished_at": last[2].isoformat() if last[2] else None,
-        } if last else None,
-    })
-
-
-@settings_bp.route("/settings/translation/test", methods=["POST"])
-async def translation_test():
-    """On-demand reachability check for a GIVEN Interpreter base URL (the Settings
-    'Test connection' button) — pings /v1/health without saving, so the operator
-    can verify a URL before enabling. Health runs in a thread (sync client)."""
-    body = await request.get_json()
-    base_url = ""
-    if isinstance(body, dict):
-        base_url = (body.get("base_url") or "").strip()
-    healthy = await asyncio.to_thread(ic.health, base_url) if base_url else False
-    return jsonify({"healthy": healthy})
-
-
-@settings_bp.route("/settings/translation/probe", methods=["POST"])
-async def translation_probe():
-    """Diagnostic for the Settings 'Test translation' box: translate a pasted
-    snippet WITHOUT saving anything, returning what Interpreter *detected*
-    (language + confidence) alongside the result. Lets the operator see why a
-    given string was (mis-)detected — e.g. a short English title flagged as
-    another language — so a detection guard can be tuned from real numbers.
-    Read-only: no post is touched. Uses the currently-saved base URL + target."""
-    body = await request.get_json(silent=True)
-    body = body if isinstance(body, dict) else {}
-    text = (body.get("text") or "").strip()
-    if not text:
-        return jsonify({"error": "provide text to translate"}), 400
-    async with get_session() as session:
-        cfg = await ImportSettings.load(session)
-    base_url = (cfg.interpreter_base_url or "").strip()
-    if not base_url:
-        return jsonify({"error": "no Interpreter base URL is set"}), 400
-    target = (cfg.translation_target_lang or "en").strip() or "en"
-    try:
-        res = await asyncio.to_thread(
-            ic.translate, [text], base_url=base_url, target=target,
-        )
-    except ic.InterpreterUnavailable as e:
-        return jsonify({"error": f"Interpreter unavailable: {e}"}), 503
-    except ic.InterpreterBadRequest as e:
-        return jsonify({"error": f"Interpreter rejected the request: {e}"}), 400
-    translations = res.get("translations") or []
-    return jsonify({
-        "target": target,
-        "detected_lang": res.get("detected_lang"),
-        "detected_confidence": res.get("detected_confidence"),
-        "engine": res.get("engine"),
-        "engine_version": res.get("engine_version"),
-        "translated": translations[0] if translations else None,
-    })
-
-
-@settings_bp.route("/settings/translation/run", methods=["POST"])
-async def translation_run():
-    """Enqueue the translate sweep now (the Settings 'Translate now' button).
-    Runs in drain mode — run-until-done — so one press chases the whole
-    untranslated backlog to zero rather than a single 300-post chunk."""
-    async with get_session() as session:
-        cfg = await ImportSettings.load(session)
-    if not cfg.translation_enabled or not (cfg.interpreter_base_url or "").strip():
-        return jsonify(
-            {"error": "translation is disabled or no base URL is set"}
-        ), 400
-    from ..tasks.translation import translate_posts
-
-    r = translate_posts.delay(drain=True)
-    return jsonify({"celery_task_id": r.id}), 202
-
-
-@settings_bp.route("/settings/translation/retranslate", methods=["POST"])
-async def translation_retranslate():
-    """Re-translate stored translations after a model change (m146). Body:
-    ``{"artist_id": <int>}`` aims at one artist; ``{"all": true}`` re-runs every
-    artist. ``all`` must be explicit so an empty/typo body can't wipe everything.
-    Clears the scoped translations and enqueues the run-until-done retranslate
-    sweep (the Interpreter cache re-translates on a changed model, is cache-fast
-    otherwise). Same enabled + base-URL guard as 'Translate now'."""
-    body = await request.get_json(silent=True)
-    body = body if isinstance(body, dict) else {}
-    artist_id = body.get("artist_id")
-    do_all = bool(body.get("all"))
-    if artist_id is None and not do_all:
-        return jsonify(
-            {"error": "provide artist_id, or all=true to re-translate everything"}
-        ), 400
-    if artist_id is not None:
-        try:
-            artist_id = int(artist_id)
-        except (TypeError, ValueError):
-            return jsonify({"error": "artist_id must be an integer"}), 400
-
-    async with get_session() as session:
-        cfg = await ImportSettings.load(session)
-    if not cfg.translation_enabled or not (cfg.interpreter_base_url or "").strip():
-        return jsonify(
-            {"error": "translation is disabled or no base URL is set"}
-        ), 400
-    from ..tasks.translation import retranslate_posts
-
-    # artist_id wins when both are sent; otherwise all=true → None (every artist).
-    artist_ids = [artist_id] if artist_id is not None else None
-    r = retranslate_posts.delay(artist_ids=artist_ids)
-    return jsonify({"celery_task_id": r.id}), 202

backend/app/api/sources.py

@@ -136,25 +136,6 @@ async def delete_source(source_id: int):
     return "", 204
 
 
-@sources_bp.route("/<int:source_id>/reassign", methods=["POST"])
-async def reassign_source(source_id: int):
-    """Move this source (and the content it brought in) to another artist
-    (#130). Files don't move — the slug is immutable — so this just re-attributes
-    the source, its posts, and its images. Body: {target_artist_id}."""
-    body = await request.get_json(silent=True) or {}
-    target = body.get("target_artist_id")
-    if not isinstance(target, int):
-        return _bad("invalid_body", detail="target_artist_id (int) required")
-    async with get_session() as session:
-        try:
-            record = await SourceService(session).reassign(source_id, target)
-        except LookupError:
-            return _bad("not_found", status=404)
-        except ArtistNotFoundError:
-            return _bad("artist_not_found", detail="target artist not found", status=404)
-    return jsonify(record.to_dict())
-
-
 @sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
 async def set_backfill(source_id: int):
     """Plan #693/#697 + #830: start/stop a backfill, or start a recovery /
@@ -230,6 +211,50 @@ async def set_backfill(source_id: int):
     return jsonify(record.to_dict())
 
 
+@sources_bp.route("/<int:source_id>/preview", methods=["POST"])
+async def preview_source_endpoint(source_id: int):
+    """Plan #708 B4: dry-run — count what a backfill WOULD download for a native
+    platform (Patreon today), without downloading. Walks the first few feed pages
+    and counts media not already in the seen/dead ledgers. Returns
+    {total_new, posts_scanned, pages_scanned, has_more, sample[]} or 409 + reason
+    (unresolvable campaign id / auth / drift). 400 for gallery-dl platforms (no
+    cheap dry-run — their verify is a slow --simulate)."""
+    from pathlib import Path
+
+    from ..services.credential_service import CredentialService
+    from ..services.download_backends import preview_source, uses_native_ingester
+    from ..tasks._sync_engine import sync_session_factory
+    from .credentials import _get_crypto
+
+    async with get_session() as session:
+        rec = await SourceService(session).get(source_id)
+        if rec is None:
+            return _bad("not_found", status=404)
+        if not uses_native_ingester(rec.platform):
+            return _bad(
+                "unsupported",
+                detail="Preview is only available for native-ingester platforms.",
+                status=400,
+            )
+        cred = CredentialService(session, _get_crypto())
+        cookies_path = await cred.get_cookies_path(rec.platform)
+
+    # The walk + ledger reads are sync (run off the request loop); the process
+    # sync engine is the same one the download task uses.
+    result = await preview_source(
+        platform=rec.platform,
+        url=rec.url,
+        source_id=source_id,
+        config_overrides=rec.config_overrides or {},
+        cookies_path=str(cookies_path) if cookies_path else None,
+        images_root=Path("/images"),
+        sync_session_factory=sync_session_factory(),
+    )
+    if "error" in result:
+        return _bad("preview_failed", detail=result["error"], status=409)
+    return jsonify(result)
+
+
 @sources_bp.route("/<int:source_id>/check", methods=["POST"])
 async def check_source(source_id: int):
     """FC-3c: enqueue a download for this source.

backend/app/api/suggestions.py

@@ -11,11 +11,10 @@ suggestions_bp = Blueprint("suggestions", __name__, url_prefix="/api")
 
 @suggestions_bp.route("/images/<int:image_id>/suggestions", methods=["GET"])
 async def get_suggestions(image_id: int):
-    # ?min=<float> overrides the per-head suggest thresholds for INCLUSION. The
+    # ?min=<float> overrides the configured per-category thresholds so the typed
-    # rail sends min=0 in its single per-image fetch to get EVERY head (each row
+    # tag-input dropdown can surface EVERY stored prediction (min=0), including
-    # still carries above_threshold vs its natural cut), then derives the panel
+    # low-confidence actions/features, in canonical formatting. Omitted → the
-    # (above_threshold) and the typed dropdown (all, filtered by text) client-side
+    # curated above-threshold list the Suggestions panel uses.
-    # — no second request. Omitted → only above-threshold rows.
     override = None
     raw_min = request.args.get("min")
     if raw_min is not None:
@@ -37,19 +36,16 @@ async def get_suggestions(image_id: int):
                         "category": s.category,
                         "score": round(s.score, 4),
                         "source": s.source,
-                        # whether the score cleared the head's own suggest cut.
+                        "creates_new_tag": s.creates_new_tag,
-                        # The single min=0 fetch returns every head; the panel
+                        # raw model key (alias is stored under this) + whether an
-                        # shows above_threshold, the typed dropdown shows all and
+                        # operator alias produced this suggestion — drive the
-                        # annotates each match with its score.
+                        # modal's "Treat as alias"/"Remove alias" affordances.
-                        "above_threshold": s.above_threshold,
+                        "raw_name": s.raw_name,
+                        "via_alias": s.via_alias,
                         # operator dismissed this tag for this image — surfaced
                         # (not dropped) so the rail can show it rejected + offer
                         # one-click un-reject.
                         "rejected": s.rejected,
-                        # the crop region that produced this tag (#1206) —
-                        # {bbox,kind,detector} or null (whole-image won). Drives
-                        # the hover→overlay highlight.
-                        "grounding": s.grounding,
                     }
                     for s in items
                 ]
@@ -73,6 +69,26 @@ async def accept_suggestion(image_id: int):
     return jsonify({"accepted": True, "tag_id": tag_id})
 
 
+@suggestions_bp.route(
+    "/images/<int:image_id>/suggestions/alias", methods=["POST"]
+)
+async def alias_suggestion(image_id: int):
+    body = await request.get_json()
+    required = {"alias_string", "alias_category", "canonical_tag_id"}
+    if not body or not required.issubset(body):
+        return jsonify({"error": f"required: {sorted(required)}"}), 400
+    canonical_tag_id = body["canonical_tag_id"]
+    async with get_session() as session:
+        await AllowlistService(session).add_alias_and_accept(
+            image_id,
+            body["alias_string"],
+            body["alias_category"],
+            canonical_tag_id,
+        )
+        await session.commit()
+    return jsonify({"accepted": True, "tag_id": canonical_tag_id})
+
+
 @suggestions_bp.route(
     "/images/<int:image_id>/suggestions/dismiss", methods=["POST"]
 )

backend/app/api/tag_eval.py

@@ -0,0 +1,70 @@
+"""Tag-eval API (#1130): trigger + revisit the head-vs-centroid eval.
+
+The run + full report live in the tag_eval_run row, so the admin card rehydrates
+from GET (history / detail) on mount — the report survives navigation rather than
+living in transient frontend state.
+"""
+
+from quart import Blueprint, jsonify, request
+from sqlalchemy import select
+
+from ..extensions import get_session
+from ..models import TagEvalRun
+from ..services.ml.tag_eval import EvalAlreadyRunning, start_tag_eval_run
+
+tag_eval_bp = Blueprint("tag_eval", __name__, url_prefix="/api/tag-eval")
+
+
+def _serialize(run: TagEvalRun, *, include_report: bool) -> dict:
+    out = {
+        "id": run.id,
+        "params": run.params,
+        "status": run.status,
+        "started_at": run.started_at.isoformat() if run.started_at else None,
+        "finished_at": run.finished_at.isoformat() if run.finished_at else None,
+        "error": run.error,
+    }
+    if include_report:
+        out["report"] = run.report
+    return out
+
+
+@tag_eval_bp.route("", methods=["POST"])
+async def create():
+    body = await request.get_json(silent=True) or {}
+    params = body.get("params") or body or {}
+    async with get_session() as session:
+        try:
+            run_id = await session.run_sync(
+                lambda s: start_tag_eval_run(s, params)
+            )
+        except EvalAlreadyRunning as running:
+            return jsonify({
+                "error": "eval_already_running",
+                "running_id": int(running.args[0]),
+            }), 409
+        await session.commit()
+    return jsonify({"run_id": run_id, "status": "running"}), 202
+
+
+@tag_eval_bp.route("", methods=["GET"])
+async def history():
+    try:
+        limit = min(int(request.args.get("limit", "20")), 100)
+    except ValueError:
+        return jsonify({"error": "invalid_limit"}), 400
+    async with get_session() as session:
+        rows = (await session.execute(
+            select(TagEvalRun).order_by(TagEvalRun.id.desc()).limit(limit)
+        )).scalars().all()
+        # List is light — no full report (the detail endpoint carries it).
+        return jsonify({"runs": [_serialize(r, include_report=False) for r in rows]})
+
+
+@tag_eval_bp.route("/<int:run_id>", methods=["GET"])
+async def detail(run_id: int):
+    async with get_session() as session:
+        run = await session.get(TagEvalRun, run_id)
+        if run is None:
+            return jsonify({"error": "not_found"}), 404
+        return jsonify(_serialize(run, include_report=True))

backend/app/api/tags.py

@@ -11,7 +11,6 @@ from ..models.tag import image_tag
 from ..models.tag_suggestion_rejection import TagSuggestionRejection
 from ..services.bulk_tag_service import BulkTagService
 from ..services.ml.aliases import AliasService
-from ..services.ml.heads import ground_applied_tag
 from ..services.series_match_service import SeriesMatchService
 from ..services.series_service import SeriesError, SeriesService
 from ..services.tag_directory_service import TagDirectoryService
@@ -311,21 +310,6 @@ async def confirm_tag_on_image(image_id: int, tag_id: int):
     return "", 204
 
 
-@tags_bp.route(
-    "/images/<int:image_id>/tags/<int:tag_id>/grounding", methods=["GET"]
-)
-async def tag_grounding(image_id: int, tag_id: int):
-    """Which crop region best explains an ALREADY-APPLIED tag on this image
-    (#1206 Step 4). Powers the hover→overlay highlight on applied tag chips,
-    mirroring the suggestion rail's live grounding. Computed on demand (applied
-    tags aren't scored live). → {grounding: {bbox,kind,detector}|null,
-    has_head: bool}; has_head False means the tag has no head to localize with,
-    so the chip shows no overlay."""
-    async with get_session() as session:
-        grounding, has_head = await ground_applied_tag(session, image_id, tag_id)
-    return jsonify({"grounding": grounding, "has_head": has_head})
-
-
 @tags_bp.route("/tags/<int:tag_id>", methods=["GET"])
 async def get_tag(tag_id: int):
     """Resolve a single tag (used by the gallery to label its active
@@ -340,7 +324,6 @@ async def get_tag(tag_id: int):
                 "name": tag.name,
                 "kind": tag.kind.value,
                 "fandom_id": tag.fandom_id,
-                "is_system": tag.is_system,
             }
         )
 
@@ -407,7 +390,6 @@ async def update_tag(tag_id: int):
                 "name": tag.name,
                 "kind": tag.kind.value,
                 "fandom_id": tag.fandom_id,
-                "is_system": tag.is_system,
             }
         )
 

backend/app/celery_app.py

@@ -7,7 +7,7 @@ Queues:
   download    — gallery-dl tasks (FC-3)
   scan        — periodic source checks (FC-3) — kept separate so long imports
                 don't starve the scheduler
-  maintenance — recovery sweeps, pHash backfill, GPU-queue coordination, etc.
+  maintenance — pHash recomputation, centroid rebuild, etc. (FC-2/FC-3)
   default     — anything not explicitly routed
 """
 
@@ -29,13 +29,11 @@ def make_celery() -> Celery:
             "backend.app.tasks.thumbnail",
             "backend.app.tasks.maintenance",
             "backend.app.tasks.ml",
-            "backend.app.tasks.gpu_queue",
             "backend.app.tasks.download",
             "backend.app.tasks.external",
             "backend.app.tasks.backup",
             "backend.app.tasks.admin",
             "backend.app.tasks.library_audit",
-            "backend.app.tasks.translation",
         ],
     )
     app.conf.update(
@@ -43,11 +41,6 @@ def make_celery() -> Celery:
         task_routes={
             "backend.app.tasks.import_file.*": {"queue": "import"},
             "backend.app.tasks.ml.*": {"queue": "ml"},
-            # GPU-queue coordination (backfill enqueues, orphan recovery,
-            # reprocess) is pure DB work — it rides the maintenance quick lane
-            # so the GPU agent pipeline works even on stacks that drop the
-            # (now-optional, B3) ml-worker container entirely.
-            "backend.app.tasks.gpu_queue.*": {"queue": "maintenance"},
             "backend.app.tasks.thumbnail.*": {"queue": "thumbnail"},
             "backend.app.tasks.download.*": {"queue": "download"},
             # External file-host fetches are downloads — same lane (they can run
@@ -64,20 +57,9 @@ def make_celery() -> Celery:
             "backend.app.tasks.backup.*": {"queue": "maintenance_long"},
             "backend.app.tasks.admin.*": {"queue": "maintenance_long"},
             "backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
-            # Translation backfill hits the LLM (~1–6s/item) → the long lane so it
-            # never starves the quick self-healing sweeps (#143).
-            "backend.app.tasks.translation.*": {"queue": "maintenance_long"},
         },
         # Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
         task_acks_late=True,
-        # Deploy graceful-shutdown safety: with acks_late, a task killed because
-        # it outran the container's stop-grace window (SIGKILL) is re-queued
-        # rather than silently lost. Safe because our long tasks are idempotent +
-        # chunked (translation per-post commit, downloads terminal-status, audits
-        # chunk) and the 5-min recovery sweeps re-drive anything left non-terminal
-        # — a re-run resumes cleanly and never corrupts. No redeliver-loop risk:
-        # heavy GPU work is tombstoned via gpu_queue, not run inline in a worker.
-        task_reject_on_worker_lost=True,
         worker_prefetch_multiplier=1,
         # Broker resilience (2026-06-24): a swarm overlay-network blip after a
         # redeploy left Redis healthy but transiently unreachable, and a worker
@@ -115,48 +97,34 @@ def make_celery() -> Celery:
                 "task": "backend.app.tasks.maintenance.cleanup_old_tasks",
                 "schedule": 86400.0,  # daily
             },
-            "cleanup-orphaned-temp-files": {
+            "ml-backfill-daily": {
-                "task": "backend.app.tasks.maintenance.cleanup_orphaned_temp_files",
+                "task": "backend.app.tasks.ml.backfill",
-                "schedule": 86400.0,  # daily — sweep .part/.partial left by a
+                "schedule": 86400.0,
-                # download/import killed mid-write (graceful-shutdown fallout)
             },
             "train-heads-nightly": {
                 "task": "backend.app.tasks.ml.scheduled_train_heads",
                 "schedule": 86400.0,  # passive cadence; manual retrain stays available
             },
-            "refresh-character-prototypes": {
-                "task": "backend.app.tasks.ml.refresh_character_prototypes",
-                "schedule": 900.0,  # ~15 min; cheap global-gate no-op when idle (#1317)
-            },
-            "reconcile-character-prototypes-nightly": {
-                "task": "backend.app.tasks.ml.refresh_character_prototypes",
-                "schedule": 86400.0,  # nightly FULL reconcile (belt-and-suspenders)
-                "args": (True,),      # full=True
-            },
             "apply-head-tags-daily": {
                 "task": "backend.app.tasks.ml.scheduled_apply_head_tags",
                 "schedule": 86400.0,  # no-op unless head_auto_apply_enabled
             },
             "recover-orphaned-gpu-jobs": {
-                "task": "backend.app.tasks.gpu_queue.recover_orphaned_gpu_jobs",
+                "task": "backend.app.tasks.ml.recover_orphaned_gpu_jobs",
                 "schedule": 60.0,  # quick pickup of work a dead agent orphaned
             },
-            "triage-gpu-errors": {
-                "task": "backend.app.tasks.maintenance.triage_gpu_errors",
-                "schedule": 900.0,  # probe errored jobs' files → defect/file_ok
-            },
             "enqueue-ccip-backfill-hourly": {
-                "task": "backend.app.tasks.gpu_queue.enqueue_gpu_backfill",
+                "task": "backend.app.tasks.ml.enqueue_gpu_backfill",
-                "schedule": 3600.0,  # auto-feed NEW images; errored are
+                "schedule": 3600.0,  # auto-feed new images (+ retry errored) so
-                "args": ("ccip",),   # tombstoned — retry is the button only
+                "args": ("ccip",),   # the queue keeps moving without the button
             },
             "enqueue-siglip-backfill-daily": {
-                "task": "backend.app.tasks.gpu_queue.enqueue_gpu_backfill",
+                "task": "backend.app.tasks.ml.enqueue_gpu_backfill",
-                "schedule": 86400.0,  # drain the concept-crop back-catalogue
+                "schedule": 86400.0,  # drain the concept-crop back-catalogue +
-                "args": ("siglip",),  # (errored are tombstoned, not retried)
+                "args": ("siglip",),  # retry failed embeds, no button needed
             },
             "enqueue-embed-backfill-daily": {
-                "task": "backend.app.tasks.gpu_queue.enqueue_gpu_backfill",
+                "task": "backend.app.tasks.ml.enqueue_gpu_backfill",
                 "schedule": 86400.0,  # whole-image re-embed under the current
                 "args": ("embed",),   # model (an operator swap) drains via agent
             },
@@ -164,33 +132,6 @@ def make_celery() -> Celery:
                 "task": "backend.app.tasks.ml.scheduled_ccip_auto_apply",
                 "schedule": 86400.0,  # no-op unless ccip_auto_apply_enabled
             },
-            "retract-auto-tags-daily": {
-                "task": "backend.app.tasks.ml.scheduled_retract_auto_tags",
-                "schedule": 86400.0,  # soft auto-apply: drop auto-tags now below
-                # their threshold (m139); no-op unless the auto-apply switch is on
-            },
-            "presentation-auto-apply-daily": {
-                "task": "backend.app.tasks.ml.scheduled_presentation_auto_apply",
-                "schedule": 86400.0,  # auto-hide banner chrome (#141);
-                # no-op unless presentation_auto_apply_enabled
-            },
-            "process-auto-apply-daily": {
-                "task": "backend.app.tasks.ml.scheduled_process_auto_apply",
-                "schedule": 86400.0,  # auto-tag wip/editor process art (#1464);
-                # no-op unless process_auto_apply_enabled (opt-in)
-            },
-            "prune-presentation-reviews-daily": {
-                "task": "backend.app.tasks.ml.prune_presentation_reviews",
-                "schedule": 86400.0,  # retention: drop resolved review flags >30d
-            },
-            "translate-posts-8h": {
-                "task": "backend.app.tasks.translation.translate_posts",
-                "schedule": 28800.0,  # every 8h: steady-state cadence for the
-                # trickle of newly-imported posts (no-op unless translation
-                # configured + healthy). One bounded 300-chunk per fire — the
-                # one-time backlog drains via the "Translate now" button
-                # (drain=True, run-until-done), not this sweep.
-            },
             "snapshot-head-metrics-daily": {
                 "task": "backend.app.tasks.maintenance.snapshot_head_metrics",
                 "schedule": 86400.0,
@@ -242,6 +183,10 @@ def make_celery() -> Celery:
                 "task": "backend.app.tasks.maintenance.recover_stalled_library_audit_runs",
                 "schedule": 300.0,
             },
+            "recover-stalled-tag-eval-runs": {
+                "task": "backend.app.tasks.maintenance.recover_stalled_tag_eval_runs",
+                "schedule": 300.0,
+            },
             "recover-stalled-head-training-runs": {
                 "task": "backend.app.tasks.maintenance.recover_stalled_head_training_runs",
                 "schedule": 300.0,

backend/app/models/__init__.py

@@ -5,7 +5,6 @@ from .artist import Artist
 from .artist_visit import ArtistVisit
 from .backup_run import BackupRun
 from .base import Base
-from .character_prototype import CcipPrototypeState, CharacterPrototype
 from .credential import Credential
 from .download_event import DownloadEvent
 from .external_link import ExternalLink
@@ -24,11 +23,8 @@ from .library_audit_run import LibraryAuditRun
 from .ml_settings import MLSettings
 from .patreon_failed_media import PatreonFailedMedia
 from .patreon_seen_media import PatreonSeenMedia
-from .pixiv_failed_media import PixivFailedMedia
-from .pixiv_seen_media import PixivSeenMedia
 from .post import Post
 from .post_attachment import PostAttachment
-from .presentation_review import PresentationReview
 from .series_chapter import SeriesChapter
 from .series_page import SeriesPage
 from .series_suggestion import SeriesSuggestion
@@ -37,6 +33,7 @@ from .subscribestar_failed_media import SubscribeStarFailedMedia
 from .subscribestar_seen_media import SubscribeStarSeenMedia
 from .tag import Tag, TagKind, image_tag
 from .tag_alias import TagAlias
+from .tag_eval_run import TagEvalRun
 from .tag_head import TagHead
 from .tag_positive_confirmation import TagPositiveConfirmation
 from .tag_suggestion_rejection import TagSuggestionRejection
@@ -52,13 +49,10 @@ __all__ = [
     "Credential",
     "PatreonFailedMedia",
     "PatreonSeenMedia",
-    "PixivFailedMedia",
-    "PixivSeenMedia",
     "SubscribeStarFailedMedia",
     "SubscribeStarSeenMedia",
     "Post",
     "PostAttachment",
-    "PresentationReview",
     "SeriesChapter",
     "SeriesPage",
     "SeriesSuggestion",
@@ -81,9 +75,8 @@ __all__ = [
     "HeadMetricsSnapshot",
     "HeadTrainingRun",
     "TagAlias",
+    "TagEvalRun",
     "TagHead",
-    "CharacterPrototype",
-    "CcipPrototypeState",
     "TagPositiveConfirmation",
     "TagSuggestionRejection",
     "TaskRun",

backend/app/models/artist.py

@@ -15,14 +15,7 @@ class Artist(Base):
     __tablename__ = "artist"
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    # Display name: freely editable, NON-unique (two real creators can share a
+    name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
-    # name). Decoupled from identity/storage in migration 0077 (#130) — renaming
-    # touches ONLY this. Was unique until then.
-    name: Mapped[str] = mapped_column(String(255), nullable=False)
-    # Storage/identity key: IMMUTABLE + unique. This is the on-disk path
-    # component (download_service artist_slug = artist.slug → images_root/<slug>/
-    # <platform>/…), so it is set once at creation (collision-suffixed) and NEVER
-    # changes — a rename must not move files. Existing artists keep their slug.
     slug: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
     notes: Mapped[str | None] = mapped_column(Text, nullable=True)
 

backend/app/models/character_prototype.py

@@ -1,62 +0,0 @@
-"""Precomputed CCIP character prototypes (#1317, milestone 138).
-
-The live matcher (ccip.match_image) needs each character's reference figure
-vectors. Building that on the request path reloaded EVERY figure CCIP vector in
-the library on any change (~4s, invalidated by every character accept). These
-tables make the references a PRECOMPUTED, INCREMENTAL artifact refreshed off the
-request path (services.ml.character_prototypes):
-
-- CharacterPrototype: a character's reference vectors, capped to
-  MLSettings.ccip_prototype_cap so MATCH cost doesn't grow with a character's
-  popularity. The async matcher only READS these.
-- CcipPrototypeState: a per-character fingerprint (reference count + max region
-  id) so a refresh rebuilds ONLY the characters whose references changed, and
-  its updated_at lets the matcher's cache reload just the advanced characters.
-"""
-
-from datetime import datetime
-
-from pgvector.sqlalchemy import Vector
-from sqlalchemy import DateTime, ForeignKey, Integer, String, func
-from sqlalchemy.orm import Mapped, mapped_column
-
-from .base import Base
-from .image_region import CCIP_DIM
-
-
-class CharacterPrototype(Base):
-    __tablename__ = "character_prototype"
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    # The character tag these vectors identify. CASCADE: deleting the tag drops
-    # its prototypes.
-    tag_id: Mapped[int] = mapped_column(
-        ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
-    )
-    # A reference figure/face CCIP vector (same space as
-    # ImageRegion.ccip_embedding).
-    ccip_embedding: Mapped[list[float]] = mapped_column(
-        Vector(CCIP_DIM), nullable=False
-    )
-    # Provenance: the region this vector was copied from. SET NULL so pruning a
-    # region doesn't delete the prototype mid-cycle (the next refresh reconciles).
-    region_id: Mapped[int | None] = mapped_column(
-        ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True
-    )
-
-
-class CcipPrototypeState(Base):
-    __tablename__ = "ccip_prototype_state"
-
-    # One row per character that currently has prototypes.
-    tag_id: Mapped[int] = mapped_column(
-        ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
-    )
-    # count(reference regions) + max(region id) at last build — the cheap
-    # per-character change detector that drives incremental rebuilds.
-    fingerprint: Mapped[str] = mapped_column(String(64), nullable=False)
-    # Bumped when this character's prototypes are rebuilt; the matcher cache
-    # reloads only characters whose updated_at advanced.
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )

backend/app/models/gpu_job.py

@@ -14,16 +14,7 @@ pending for another agent).
 
 from datetime import datetime
 
-from sqlalchemy import (
+from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, func
-    DateTime,
-    ForeignKey,
-    Index,
-    Integer,
-    String,
-    Text,
-    func,
-    text,
-)
 from sqlalchemy.orm import Mapped, mapped_column
 
 from .base import Base
@@ -32,17 +23,6 @@ from .base import Base
 class GpuJob(Base):
     __tablename__ = "gpu_job"
 
-    # Partial indexes over just the live slice (see migration 0070): the lease
-    # reads the lowest-id pending jobs on the hot path, and reclaims expired
-    # leases as a backstop — both stay O(batch) as done/error history grows.
-    __table_args__ = (
-        Index("ix_gpu_job_pending", "id", postgresql_where=text("status = 'pending'")),
-        Index(
-            "ix_gpu_job_leased_expires", "lease_expires_at",
-            postgresql_where=text("status = 'leased'"),
-        ),
-    )
-
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
     image_record_id: Mapped[int] = mapped_column(
         ForeignKey("image_record.id", ondelete="CASCADE"), index=True
@@ -62,11 +42,6 @@ class GpuJob(Base):
     )
     attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     error: Mapped[str | None] = mapped_column(Text, nullable=True)
-    # Triage verdict for an ERRORED job (#125): NULL = not yet probed;
-    # 'defect' = the integrity probe says the FILE itself is bad (surfaced for
-    # recovery, excluded from /retry_errors); 'file_ok' = the file passes —
-    # the failure was operational (timeout/transient), safe to retry.
-    triage_status: Mapped[str | None] = mapped_column(String(16), nullable=True)
     created_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )

backend/app/models/head_training_run.py

@@ -1,7 +1,7 @@
 """HeadTrainingRun — persisted lifecycle of a head-training batch (#114).
 
-A persisted run row (not transient frontend state) so the run SURVIVES
+Mirrors TagEvalRun so the run SURVIVES navigation and the admin card can show
-navigation and the admin card can show live + historical status.
+live + historical status instead of holding it in transient frontend state.
 Training is idempotent (it upserts tag_head rows), so a SIGKILL'd run is harmless
 — a maintenance recovery sweep flips a stalled `running` row to `error`, and the
 next run re-trains. State machine: running → ready / error.
@@ -37,8 +37,8 @@ class HeadTrainingRun(Base):
     n_trained: Mapped[int | None] = mapped_column(Integer, nullable=True)
     n_skipped: Mapped[int | None] = mapped_column(Integer, nullable=True)
     error: Mapped[str | None] = mapped_column(Text, nullable=True)
-    # Last time the task made progress — the recovery sweep tells a live run
+    # Last time the task made progress — the recovery sweep tells a live run from
-    # from a SIGKILL'd one by this.
+    # a SIGKILL'd one by this (mirrors TagEvalRun).
     last_progress_at: Mapped[datetime | None] = mapped_column(
         DateTime(timezone=True), nullable=True
     )

backend/app/models/image_record.py

@@ -97,16 +97,6 @@ class ImageRecord(Base):
     effective_date: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )
-    # Denormalized ORIGINAL-publish sort key (alembic 0071) = MIN(post_date)
-    # across ALL of the image's provenance posts, else created_at. effective_date
-    # above keys off the PRIMARY post (often the repost/download the file came
-    # from); this keys off the earliest publish across EVERY post the image
-    # appears in, so the gallery can sort by when content was first posted rather
-    # than when it was downloaded (operator-flagged 2026-07-01). Maintained by
-    # services/importer.py, recomputed whenever a dated post is linked.
-    earliest_post_date: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
     updated_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True),
         nullable=False,

backend/app/models/import_settings.py

@@ -92,41 +92,6 @@ class ImportSettings(Base):
         Boolean, nullable=False, default=True, server_default="true",
     )
 
-    # -- Post-text translation via the Interpreter LAN service (milestone 143).
-    # Off by default with NO default host — it needs a reachable Interpreter
-    # service (the operator's, behind a reverse proxy), which not every install
-    # has; the operator sets the URL and flips it on. Empty base_url OR disabled
-    # → the translate sweep no-ops.
-    translation_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=False, server_default="false",
-    )
-    interpreter_base_url: Mapped[str] = mapped_column(
-        Text, nullable=False, default="", server_default="",
-    )
-    translation_target_lang: Mapped[str] = mapped_column(
-        Text, nullable=False, default="en", server_default="en",
-    )
-    # The latin-script acceptance floor for the translation gate: a translation
-    # whose Interpreter-reported confidence is below this is kept as the original
-    # (operator-tunable, milestone 155). Default 0.9 — stricter than the old
-    # hardcoded 0.8, because Interpreter confidently mis-detects short ASCII
-    # English (e.g. "… WIP Part 1") as a European language at ~0.86. CJK stays
-    # trusted regardless (script-detected). Per-post overrides handle the misses.
-    translation_min_confidence: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.9, server_default="0.9",
-    )
-
-    # Title-based WIP auto-tagging (task #1458). When a freshly-imported post's
-    # TITLE explicitly declares work-in-progress ("WIP" / "work in progress"),
-    # the importer applies the `wip` system tag to its images — the artist's own
-    # label, used to keep unfinished pieces out of the Explore/gallery browse. ON
-    # by default (rule 26 — the feature works out of the box). Gates only the
-    # LIVE import hook; the existing catalogue is caught by the operator-triggered
-    # "Scan existing posts" backfill (which runs regardless of this flag).
-    wip_title_tagging_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=True, server_default="true",
-    )
-
     @classmethod
     async def load(cls, session) -> ImportSettings:
         """The singleton settings row (id=1), via an async session."""

backend/app/models/ml_settings.py

@@ -23,15 +23,6 @@ class MLSettings(Base):
     __table_args__ = (CheckConstraint("id = 1", name="singleton"),)
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    # CPU whole-image embedding (B3, operator 2026-07-02). The ml-worker's ONLY
-    # processing role is the embed fallback for stacks WITHOUT a GPU agent — ON
-    # by default so a fresh install works with no agent. Stacks that run the
-    # agent and drop the ml-worker container turn this OFF so import hooks stop
-    # queueing embed work nothing will consume (the daily GPU 'embed' backfill
-    # covers those images instead).
-    cpu_embed_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=True
-    )
     # Video embedding (#747). Sample one frame every N seconds (fixed CADENCE, not
     # a fixed count) so coverage reflects real screen time regardless of length;
     # cap the total so a long video can't explode into hundreds of embeds. The
@@ -63,9 +54,7 @@ class MLSettings(Base):
         Boolean, nullable=False, default=True
     )
     head_auto_apply_min_positives: Mapped[int] = mapped_column(
-        # Support floor raised 30→50 (operator-asked 2026-07-06): a head needs
+        Integer, nullable=False, default=30
-        # more human labels before it may fire without a human.
-        Integer, nullable=False, default=50
     )
     # CCIP character-match cosine cut (#114). 0.85 default — the v1 flat 0.75
     # over-fired (high-reference characters matched a scatter of images); 0.85
@@ -80,47 +69,7 @@ class MLSettings(Base):
         Boolean, nullable=False, default=True
     )
     ccip_auto_apply_threshold: Mapped[float] = mapped_column(
-        # Raised 0.92→0.95 (operator-asked 2026-07-06) so only very confident
+        Float, nullable=False, default=0.92
-        # character matches auto-tag.
-        Float, nullable=False, default=0.95
-    )
-    # -- Presentation chrome auto-hide (#141) -------------------------------
-    # `banner` (chrome — clusters on UI, not content) auto-applies on the sweep
-    # with its OWN flat threshold (decoupled from content-head graduation) and is
-    # HIDDEN from the gallery. Hiding is consequential so it runs HIGH. When an
-    # image would be auto-hidden but ALSO scores >= presentation_conflict_threshold
-    # on a content head, it's still hidden but flagged for review
-    # (PresentationReview, mode='chrome') instead of buried silently. ON by default
-    # (opt-out); every auto-tag is reversible. NOTE (#1464): `wip` + `editor
-    # screenshot` are no longer chrome — they went to the PROCESS path below.
-    presentation_auto_apply_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=True
-    )
-    presentation_auto_apply_threshold: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.90
-    )
-    presentation_conflict_threshold: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.50
-    )
-    # -- Process auto-apply (#1464) ----------------------------------------
-    # `wip` / `editor screenshot` are PROCESS art — unfinished pieces + program
-    # screenshots that must stay OUT of head/CCIP training but, unlike chrome,
-    # remain VISIBLE in the gallery (operator 2026-07-12). They auto-apply on the
-    # sweep with their OWN flat threshold and a PROVISIONAL source (`process_auto`,
-    # in training_data._AUTO_SOURCES) so the head NEVER trains on its own output —
-    # it learns only from title (`wip_title`) + manual labels, which breaks the
-    # runaway loop. When a process tag would be applied but the image ALSO scores
-    # >= process_conflict_threshold on a content head, it's flagged for review
-    # (PresentationReview, mode='process') rather than silently marked. OFF by
-    # default — a new whole-library auto-tagger is opt-in; every auto-tag reversible.
-    process_auto_apply_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=False
-    )
-    process_auto_apply_threshold: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.90
-    )
-    process_conflict_threshold: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.50
     )
     # Default = SigLIP 2 (so400m, 512px) for new installs (migration 0069);
     # existing libraries keep their stored value until the operator re-embeds.
@@ -133,82 +82,6 @@ class MLSettings(Base):
     embedder_model_name: Mapped[str] = mapped_column(
         String(128), nullable=False, default="google/siglip2-so400m-patch16-512"
     )
-    # -- Crop proposers / detectors (#1202, #134) --------------------------
-    # WHERE-to-crop YOLO detectors feeding the crop→SigLIP bag + CCIP. Config
-    # lives HERE (DB) and is announced to the GPU agent in the lease — same as
-    # the embedder model — so it is UI-tunable with NO restart, and the agent's
-    # env is bootstrap-only. Each weights spec is an ultralytics builtin name,
-    # an http(s) URL, or "hf_repo::file" (agent's _resolve). enabled off (or an
-    # empty weights) skips that proposer. All ON by default (operator 2026-07-05)
-    # so a fresh install crops out-of-the-box.
-    # person: general COCO figure detector for Western/realistic art the anime
-    # person-detector misses → NMS-merged with imgutils → CCIP + concept.
-    detector_person_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=True
-    )
-    detector_person_weights: Mapped[str] = mapped_column(
-        String(512), nullable=False, default="yolo11n.pt"
-    )
-    detector_person_conf: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.35
-    )
-    # anatomy: booru_yolo anime/furry/NSFW torso components → concept crops.
-    # Default = yolov11m_aa22 (26 classes, best mAP50-95 0.96), committed in the
-    # upstream repo so the URL resolves. License UNSTATED — fine for a private
-    # homelab (operator accepted #1202).
-    detector_anatomy_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=True
-    )
-    detector_anatomy_weights: Mapped[str] = mapped_column(
-        String(512), nullable=False,
-        default=(
-            "https://github.com/aperveyev/booru_yolo/raw/main/models/"
-            "yolov11m_aa22.pt"
-        ),
-    )
-    detector_anatomy_conf: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.30
-    )
-    # panel: comic page → panel regions → concept crops (Apache-2.0, YOLOv12x).
-    detector_panel_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=True
-    )
-    detector_panel_weights: Mapped[str] = mapped_column(
-        String(512), nullable=False,
-        default="mosesb/best-comic-panel-detection::best.pt",
-    )
-    detector_panel_conf: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.30
-    )
-    # Per-frame caps bound the crop→embed explosion; max_regions is the hard
-    # per-job backstop; dedupe_iou drops near-duplicate crops before the embed.
-    detector_max_figures: Mapped[int] = mapped_column(
-        Integer, nullable=False, default=8
-    )
-    detector_max_components: Mapped[int] = mapped_column(
-        Integer, nullable=False, default=8
-    )
-    detector_max_panels: Mapped[int] = mapped_column(
-        Integer, nullable=False, default=8
-    )
-    detector_max_regions: Mapped[int] = mapped_column(
-        Integer, nullable=False, default=128
-    )
-    detector_dedupe_iou: Mapped[float] = mapped_column(
-        Float, nullable=False, default=0.85
-    )
-    # -- CCIP character prototypes (#1317) ---------------------------------
-    # The per-character reference set is precomputed + refreshed INCREMENTALLY
-    # (services.ml.character_prototypes) instead of rebuilt on the request path.
-    # ccip_ref_signature is the cheap GLOBAL gate — when it's unchanged the
-    # refresh no-ops; ccip_prototype_cap bounds the reference vectors kept per
-    # character so MATCH cost doesn't grow with a character's popularity.
-    ccip_ref_signature: Mapped[str | None] = mapped_column(
-        String(128), nullable=True
-    )
-    ccip_prototype_cap: Mapped[int] = mapped_column(
-        Integer, nullable=False, default=64
-    )
     updated_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )

backend/app/models/pixiv_failed_media.py

@@ -1,45 +0,0 @@
-"""PixivFailedMedia — per-source dead-letter ledger of Pixiv media that keeps
-failing to download/validate.
-
-Mirror of PatreonFailedMedia/SubscribeStarFailedMedia. Media that fails every
-walk (404'd pximg URL, deleted work, persistently-corrupt bytes) would
-otherwise re-error forever and re-burn backfill chunks. After ``attempts``
-reaches the dead-letter threshold the ingester skips it on routine
-tick/backfill walks (recovery still re-attempts). A later clean download
-clears the row.
-
-`filehash` is the same synthesized ``<illust_id>:p<num>`` /
-``<illust_id>:ugoira`` key the seen-ledger uses. UNIQUE (source_id, filehash)
-is the upsert key.
-"""
-
-from datetime import datetime
-
-from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func
-from sqlalchemy.orm import Mapped, mapped_column
-from sqlalchemy.types import DateTime
-
-from .base import Base
-
-
-class PixivFailedMedia(Base):
-    __tablename__ = "pixiv_failed_media"
-    __table_args__ = (
-        UniqueConstraint(
-            "source_id", "filehash", name="uq_pixiv_failed_media_source_id"
-        ),
-    )
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    source_id: Mapped[int] = mapped_column(
-        ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
-    )
-    filehash: Mapped[str] = mapped_column(String(128), nullable=False)
-    attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
-    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
-    first_failed_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    last_failed_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )

backend/app/models/pixiv_seen_media.py

@@ -1,42 +0,0 @@
-"""PixivSeenMedia — per-source ledger of Pixiv media already
-downloaded+processed.
-
-Mirror of PatreonSeenMedia/SubscribeStarSeenMedia for the Pixiv native
-ingester (replacing gallery-dl). One queryable row per (source, media) so
-routine walks skip media we've already ingested; recovery mode bypasses the
-ledger to re-walk.
-
-Pixiv original URLs carry no content hash, so `filehash` is always the
-synthesized ``<illust_id>:p<num>`` (page) / ``<illust_id>:ugoira`` (frame
-zip) key — stable across any URL-shape drift. String(128) matches the sibling
-ledgers.
-"""
-
-from datetime import datetime
-
-from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
-from sqlalchemy.orm import Mapped, mapped_column
-from sqlalchemy.types import DateTime
-
-from .base import Base
-
-
-class PixivSeenMedia(Base):
-    __tablename__ = "pixiv_seen_media"
-    __table_args__ = (
-        # Dedup key the downloader upserts against: one ledger row per
-        # (source, media). A second sighting of the same media is a no-op.
-        UniqueConstraint(
-            "source_id", "filehash", name="uq_pixiv_seen_media_source_id"
-        ),
-    )
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    source_id: Mapped[int] = mapped_column(
-        ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
-    )
-    filehash: Mapped[str] = mapped_column(String(128), nullable=False)
-    post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
-    seen_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )

backend/app/models/post.py

@@ -8,17 +8,7 @@ artist-filter queries don't depend on the Source detour).
 
 from datetime import datetime
 
-from sqlalchemy import (
+from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
-    JSON,
-    CheckConstraint,
-    DateTime,
-    ForeignKey,
-    Integer,
-    String,
-    Text,
-    UniqueConstraint,
-    func,
-)
 from sqlalchemy.orm import Mapped, mapped_column
 
 from .base import Base
@@ -33,10 +23,6 @@ class Post(Base):
         # (created in alembic 0030) covers that case via
         # (artist_id, external_post_id).
         UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
-        CheckConstraint(
-            "translation_override IN ('auto', 'force', 'original')",
-            name="ck_post_translation_override",
-        ),
     )
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
@@ -61,34 +47,6 @@ class Post(Base):
     description: Mapped[str | None] = mapped_column(Text, nullable=True)
     attachment_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
 
-    # -- Post-text translation (milestone 143). Filled by the translate_posts
-    # sweep via the Interpreter LAN service so viewing is instant.
-    # translated_source_lang is the DETECTED original language; "en" (or a
-    # passthrough) means nothing to translate and the *_translated columns stay
-    # NULL. engine_version keys the Interpreter cache — re-runs are ~1ms and a
-    # model upgrade re-translates instead of serving stale.
-    post_title_translated: Mapped[str | None] = mapped_column(Text, nullable=True)
-    description_translated: Mapped[str | None] = mapped_column(Text, nullable=True)
-    translated_source_lang: Mapped[str | None] = mapped_column(
-        String(8), nullable=True
-    )
-    translation_engine_version: Mapped[str | None] = mapped_column(
-        String(128), nullable=True
-    )
-    translated_at: Mapped[datetime | None] = mapped_column(
-        DateTime(timezone=True), nullable=True
-    )
-    # Sticky per-post override of the translation decision (milestone 155):
-    # 'auto' = the acceptance gate decides; 'force' = always store Interpreter's
-    # translation even below the confidence floor (rescue a skipped legit-foreign
-    # title); 'original' = never translate, keep the original (kill a confidently
-    # mis-flagged one the floor can't catch). The sweep reads this on every run,
-    # and re-translate leaves 'original' posts alone, so the choice survives a
-    # Re-translate-all.
-    translation_override: Mapped[str] = mapped_column(
-        String(16), nullable=False, default="auto", server_default="auto",
-    )
-
     downloaded_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )

backend/app/models/presentation_review.py

@@ -1,48 +0,0 @@
-"""PresentationReview — a system-tag the auto-apply sweep applied that ALSO looked
-like real content, flagged for operator review (milestone 141 + #1464).
-
-When a sweep applies a system tag but the image ALSO scores highly on a content
-head, it still applies the tag but records this row so a review strip can surface
-it ("⚠ also looks like <conflict tag>"). Two modes (#1464): 'chrome' (banner —
-image is HIDDEN, review is keep-hidden / un-hide) and 'process' (wip / editor
-screenshot — image stays VISIBLE, review is confirm / remove-tag). Resolved rows
-are pruned by retention.
-"""
-
-from datetime import datetime
-
-from sqlalchemy import DateTime, Float, ForeignKey, String, func
-from sqlalchemy.orm import Mapped, mapped_column
-
-from .base import Base
-
-
-class PresentationReview(Base):
-    __tablename__ = "presentation_review"
-
-    image_record_id: Mapped[int] = mapped_column(
-        ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True
-    )
-    # The presentation tag that was auto-applied (banner / editor screenshot).
-    tag_id: Mapped[int] = mapped_column(
-        ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
-    )
-    # The content tag the image ALSO scored high on — the "concerning" signal.
-    # SET NULL (not CASCADE): losing the conflict tag shouldn't erase the flag.
-    conflict_tag_id: Mapped[int | None] = mapped_column(
-        ForeignKey("tag.id", ondelete="SET NULL"), nullable=True
-    )
-    conflict_score: Mapped[float] = mapped_column(Float, nullable=False)
-    # Which sweep flagged this (#1464): 'chrome' (banner, hidden) or 'process'
-    # (wip / editor screenshot, shown). Drives which review strip surfaces it and
-    # what "resolve" means (un-hide vs remove-tag). Existing rows backfill 'chrome'.
-    mode: Mapped[str] = mapped_column(
-        String(16), nullable=False, default="chrome", server_default="chrome"
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    # Set when the operator keeps-hidden or un-hides; retention prunes resolved.
-    resolved_at: Mapped[datetime | None] = mapped_column(
-        DateTime(timezone=True), nullable=True
-    )

backend/app/models/tag.py

@@ -10,7 +10,6 @@ from datetime import datetime
 from enum import StrEnum
 
 from sqlalchemy import (
-    Boolean,
     CheckConstraint,
     Column,
     DateTime,
@@ -18,7 +17,6 @@ from sqlalchemy import (
     Integer,
     String,
     Table,
-    false,
     func,
 )
 from sqlalchemy import (
@@ -43,21 +41,6 @@ class TagKind(StrEnum):
     # to keep historic tag rows queryable.
 
 
-# The seeded system tags (migration 0075). Two behavior groups (#1464):
-#   CHROME (banner): clusters on UI chrome, not content → HIDDEN from the default
-#     gallery + from similarity; auto-applied via the sweep's chrome mode.
-#   PROCESS (wip, editor screenshot): real-but-unfinished art / program screenshots
-#     → SHOWN in the gallery (operator 2026-07-12), but excluded from the Explore
-#     rabbit-hole; auto-applied via the sweep's process mode (provisional source,
-#     ring-loud review guard).
-# ALL three are excluded from OTHER concepts' head/CCIP training (training-hygiene,
-# keyed on is_system); a system tag's OWN head trains on them — that's what makes
-# auto-flagging work.
-SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot")
-CHROME_SYSTEM_TAGS = ("banner",)
-PROCESS_SYSTEM_TAGS = ("wip", "editor screenshot")
-WIP_SYSTEM_TAG = "wip"
-
 image_tag = Table(
     "image_tag",
     Base.metadata,
@@ -91,14 +74,6 @@ class Tag(Base):
     fandom_id: Mapped[int | None] = mapped_column(
         ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, index=True
     )
-    # System tags ship with FC (wip / banner / editor screenshot, seeded in
-    # migration 0075) and drive the training-hygiene exclusions: images
-    # carrying one are excluded from OTHER concepts' head training and from
-    # CCIP identity references. The mechanism keys on these exact rows, so
-    # they're protected from rename/merge-away/re-fandom in TagService.
-    is_system: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, default=False, server_default=false()
-    )
 
     created_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()

backend/app/models/tag_eval_run.py

@@ -0,0 +1,45 @@
+"""TagEvalRun — persisted lifecycle of a head-vs-centroid tagging eval (#1130).
+
+Mirrors LibraryAuditRun so the result SURVIVES navigation: the run + its full
+report live in this row, and the admin card rehydrates from it on mount instead
+of holding the report in transient frontend state. State machine:
+running → ready / error. The async ml-queue task writes `report` (JSONB) when
+done; a maintenance recovery sweep flips a stalled `running` row to `error`.
+"""
+
+from datetime import datetime
+from typing import Any
+
+from sqlalchemy import DateTime, Integer, String, Text, func
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column
+
+from .base import Base
+
+
+class TagEvalRun(Base):
+    __tablename__ = "tag_eval_run"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    # The eval parameters: {concepts: [...], curve_points: [...], neg_ratio,
+    # cv_folds, ...} — echoed back so the report is self-describing.
+    params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
+    status: Mapped[str] = mapped_column(
+        String(16), nullable=False, default="running", index=True,
+    )
+    # running | ready | error
+    started_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now(),
+    )
+    finished_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True,
+    )
+    # The full result: per-concept metrics (head vs centroid), learning-curve
+    # points, and example image ids. Null until the task finishes.
+    report: Mapped[dict[str, Any] | None] = mapped_column(JSONB, nullable=True)
+    error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    # Last time the task made progress — the recovery sweep tells a live run
+    # from a SIGKILL'd one by this (mirrors LibraryAuditRun).
+    last_progress_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True,
+    )

backend/app/models/tag_head.py

@@ -73,12 +73,5 @@ class TagHead(Base):
     trained_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )
-    # Training-data fingerprint (positives + rejections) at last fit — the
-    # incremental-retrain change detector (#1317 p2). A manual Retrain refits only
-    # heads whose fingerprint moved; the nightly run ignores it (full reconcile).
-    # NULL forces a refit (pre-fingerprint heads).
-    train_fingerprint: Mapped[str | None] = mapped_column(
-        String(128), nullable=True
-    )
     # Extra detail (auto-apply operating point, F1, etc.) — non-load-bearing.
     metrics: Mapped[dict[str, Any] | None] = mapped_column(JSONB, nullable=True)

backend/app/services/artist_service.py

@@ -283,23 +283,6 @@ class ArtistService:
             await self.session.commit()
         return artist, created
 
-    async def rename(self, artist_id: int, name: str) -> Artist | None:
-        """Change the display NAME only (#130). The slug — and every on-disk path
-        keyed off it — is IMMUTABLE, so a rename never moves files or risks a path
-        collision. Name is free text and NON-unique (migration 0077). Returns the
-        updated Artist, None if not found; raises ValueError on empty name."""
-        cleaned = (name or "").strip()
-        if not cleaned:
-            raise ValueError("artist name must not be empty")
-        artist = (await self.session.execute(
-            select(Artist).where(Artist.id == artist_id)
-        )).scalar_one_or_none()
-        if artist is None:
-            return None
-        artist.name = cleaned
-        await self.session.commit()
-        return artist
-
     async def autocomplete(self, prefix: str, limit: int = 20) -> list[Artist]:
         cleaned = (prefix or "").strip()
         if not cleaned:

backend/app/services/audits/single_color.py

@@ -1,9 +1,8 @@
 """Single-color audit: matches images where one color dominates beyond
-the threshold (within the given Euclidean RGB tolerance). The canonical
+the threshold (within the given Euclidean RGB tolerance). The first
-predicate for BOTH surfaces: FC-Cleanup's retroactive audit and — since
+canonical implementation — the import-side filter (SkipReason.single_color)
-2026-07-02 — the import-side filter (Importer._single_color_hit /
+was never wired; FC-Cleanup's audit module is the source of truth and a
-SkipReason.single_color), so what the audit flags and what the import
+future spec can adopt it on the import path too.
-skips can never disagree.
 """
 
 from PIL import Image

backend/app/services/cleanup_service.py

@@ -17,7 +17,7 @@ from datetime import UTC, datetime, timedelta
 from pathlib import Path
 from typing import Any
 
-from sqlalchemy import and_, delete, func, or_, select, update
+from sqlalchemy import delete, func, or_, select, update
 from sqlalchemy.orm import Session, aliased
 
 from ..models import (
@@ -203,9 +203,6 @@ def _unused_tag_conditions() -> list:
         Tag.id.not_in(used_via_series),
         Tag.id.not_in(used_via_chapter),
         Tag.id.not_in(used_via_fandom),
-        # System tags (#128) ship with zero applications and must survive a
-        # prune — the training-hygiene machinery keys on the rows.
-        Tag.is_system.is_(False),
     ]
 
 
@@ -405,8 +402,6 @@ def delete_tag(session: Session, *, tag_id: int) -> dict:
     tag = session.get(Tag, tag_id)
     if tag is None:
         raise LookupError(f"tag id not found: {tag_id}")
-    if tag.is_system:
-        raise ValueError(f"'{tag.name}' is a system tag and cannot be deleted")
     associations_count = count_tag_associations(session, tag_id=tag_id)
     info = {"id": tag.id, "name": tag.name, "kind": tag.kind.value}
     session.delete(tag)
@@ -723,6 +718,71 @@ def reconcile_duplicate_posts(
     return {"groups": len(groups), "merged": losers_total, "sample": sample}
 
 
+# Legacy tags FC no longer uses, in two shapes:
+#   (1) kinds the tag input never produces — archive/post/artist.
+#       provenance (post grouping) + archive membership are their own
+#       systems now, and artists are first-class Artist/Source rows.
+#       meta/rating were already hard-deleted by alembic 0023.
+#   (2) name prefixes from IR kinds FC never adopted — `source:*`.
+#       ImageRepo had a `source` kind; FC's enum doesn't, so ir_ingest
+#       fell those back to `general` (kind=general, name="source:patreon"
+#       etc.). They can't be caught by kind, so we match the name prefix.
+PURGEABLE_TAG_KINDS = ("archive", "post", "artist")
+LEGACY_NAME_PREFIXES = ("source:",)
+
+
+def _legacy_tag_predicate():
+    name_clauses = [Tag.name.like(f"{p}%") for p in LEGACY_NAME_PREFIXES]
+    return or_(Tag.kind.in_(PURGEABLE_TAG_KINDS), *name_clauses)
+
+
+def purge_legacy_tags(session: Session, *, dry_run: bool = False) -> dict:
+    """Count (dry_run) or delete legacy IR-migration tags: archive/post/
+    artist-kind tags PLUS general tags whose name matches a legacy
+    prefix (source:*).
+
+    CASCADE on image_tag / tag_alias / tag_suggestion_rejection / series_page
+    clears the related rows on the parent DELETE.
+
+    Returns:
+      {"by_kind": {kind: count, ...},        # kind-matched rows
+       "by_prefix": {"source:*": count},     # name-prefix-matched rows
+       "count": total, "sample_names": [first 50],
+       and on live runs "deleted": total}
+    """
+    predicate = _legacy_tag_predicate()
+    rows = session.execute(
+        select(Tag.id, Tag.name, Tag.kind).where(predicate)
+    ).all()
+    by_kind: dict[str, int] = {}
+    by_prefix: dict[str, int] = {}
+    for _id, name, kind in rows:
+        # Classify by name-prefix first so a source:* row counts once,
+        # under the prefix bucket, regardless of its (general) kind.
+        matched_prefix = next(
+            (p for p in LEGACY_NAME_PREFIXES if name.startswith(p)), None,
+        )
+        if matched_prefix is not None:
+            label = f"{matched_prefix}*"
+            by_prefix[label] = by_prefix.get(label, 0) + 1
+        else:
+            key = kind.value if hasattr(kind, "value") else str(kind)
+            by_kind[key] = by_kind.get(key, 0) + 1
+    sample = [name for _id, name, _kind in rows[:50]]
+    total = len(rows)
+    result = {
+        "by_kind": by_kind, "by_prefix": by_prefix,
+        "count": total, "sample_names": sample,
+    }
+    if dry_run:
+        return result
+    if total:
+        session.execute(Tag.__table__.delete().where(predicate))
+        session.commit()
+    result["deleted"] = total
+    return result
+
+
 # The CONTENT vocabulary. "Reset content tagging" wipes these so the operator
 # can re-tag from scratch. fandom + series (and series_page ordering) are
 # deliberately NOT here — they're kept.
@@ -731,15 +791,9 @@ RESETTABLE_TAG_KINDS = ("general", "character")
 
 def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict:
     """Count (dry_run) or DELETE every general + character tag so the operator
-    can re-tag from scratch. NB: the deleted applications include the tagging
+    can re-tag from scratch (heads/CCIP repopulate suggestions).
-    heads' training positives — suggestions do NOT repopulate on their own; the
-    heads retrain from whatever the operator re-tags. (The API route gates the
-    live run behind a preview-derived confirm token for exactly this reason.)
 
-    PRESERVED: fandom + series tags and their series_page ordering, AND the
+    PRESERVED: fandom + series tags and their series_page ordering. CASCADE on
-    system hygiene tags (#128) WITH their applications — the reset re-tags
-    CONTENT concepts, while wip/banner flags describe the file itself and
-    re-flagging hundreds of banners by hand would be pure loss. CASCADE on
     image_tag / tag_alias / tag_suggestion_rejection clears each deleted tag's
     applications + metadata. Tag.fandom_id is SET NULL, so deleting character
     tags never touches the fandom rows. Irreversible except via DB backup
@@ -752,9 +806,7 @@ def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict:
        "sample_names": [first 50],
        and on live runs "deleted": total}
     """
-    predicate = and_(
+    predicate = Tag.kind.in_(RESETTABLE_TAG_KINDS)
-        Tag.kind.in_(RESETTABLE_TAG_KINDS), Tag.is_system.is_(False)
-    )
     rows = session.execute(
         select(Tag.id, Tag.name, Tag.kind).where(predicate)
     ).all()
@@ -1018,7 +1070,7 @@ def reextract_archive_attachments(
     still an archive on disk, so the cursor is what guarantees forward progress.
     """
     from ..models import ImportSettings, Post, PostAttachment, Source
-    from ..tasks.ml import cpu_embed_enabled, embed_image
+    from ..tasks.ml import tag_and_embed
     from ..tasks.thumbnail import generate_thumbnail
     from .archive_extractor import is_archive
     from .importer import Importer
@@ -1099,12 +1151,10 @@ def reextract_archive_attachments(
 
     # Thumbnails + ML for the newly-imported members (best-effort; off the
     # critical path — a Redis hiccup must not fail the whole re-extract).
-    do_embed = cpu_embed_enabled()
     for img_id in enqueue_ids:
         try:
             generate_thumbnail.delay(img_id)
-            if do_embed:
+            tag_and_embed.delay(img_id)
-                embed_image.delay(img_id)
         except Exception as exc:
             log.warning("re-extract enqueue failed for image %s: %s", img_id, exc)
     return summary

backend/app/services/download_backends.py

@@ -24,17 +24,15 @@ import asyncio
 from pathlib import Path
 
 from .gallery_dl import DownloadResult, ErrorType
+from .native_ingest_common import NativeIngestError
 from .patreon_ingester import PatreonIngester
 from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
-from .pixiv_client import user_id_from_url
-from .pixiv_ingester import PixivIngester
 from .subscribestar_ingester import SubscribeStarIngester
 
 # Platforms whose download + verify go through the native ingester rather than
-# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord,
+# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord, pixiv,
-# deviantart — the latter slated for retirement, not migration) until they
+# deviantart) until they migrate too.
-# migrate too.
+NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})
-NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "pixiv"})
 
 # Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure
 # messages so the operator sees the exact lookup endpoint that was hit.
@@ -48,7 +46,6 @@ def _native_ingester_cls(platform: str):
     dispatch pick up the replacement."""
     return {
         "patreon": PatreonIngester,
-        "pixiv": PixivIngester,
         "subscribestar": SubscribeStarIngester,
     }[platform]
 
@@ -101,34 +98,14 @@ async def _resolve_native_campaign_id(
     platform: str, url: str, cookies_path: str | None, overrides: dict,
 ) -> tuple[str | None, str | None]:
     """`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
-    feed id IS the creator URL; Pixiv's is the numeric user id parsed straight
+    feed id IS the creator URL (no lookup → resolved None). Patreon resolves the
-    from it (no lookup → resolved None either way). Patreon resolves the
     campaign id from the vanity URL (resolved non-None when a lookup actually ran,
     so phase 3 caches it)."""
     if platform == "subscribestar":
         return url, None
-    if platform == "pixiv":
-        return user_id_from_url(url), None
     return await resolve_campaign_id_for_source(url, cookies_path, overrides)
 
 
-def _campaign_resolution_error(platform: str, url: str) -> str:
-    """Operator-facing message for a native source whose campaign id could not
-    be resolved — names the platform's own lookup mechanism."""
-    if platform == "pixiv":
-        return (
-            f"Could not extract a pixiv user id. source_url={url!r} — expected "
-            "a URL like https://www.pixiv.net/users/<id>."
-        )
-    vanity = extract_vanity(url)
-    return (
-        f"Could not resolve Patreon campaign id. source_url={url!r}; "
-        f"vanity={vanity!r}; "
-        f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
-        "(vanity lookup failed — cookies expired or creator moved?)"
-    )
-
-
 async def _run_native_ingester(
     ctx: dict, source_config, mode: str | None, gdl, sync_session_factory,
 ) -> tuple[DownloadResult, str | None]:
@@ -146,9 +123,9 @@ async def _run_native_ingester(
         platform, ctx["url"], ctx["cookies_path"], overrides
     )
     if not campaign_id:
-        # Patreon: vanity lookup failed. Pixiv: no numeric user id in the URL.
+        # Only reachable for Patreon (SubscribeStar's campaign id is the URL).
-        # (SubscribeStar's campaign id is the URL itself — never lands here.)
         url = ctx["url"]
+        vanity = extract_vanity(url)
         return (
             DownloadResult(
                 success=False,
@@ -156,7 +133,12 @@ async def _run_native_ingester(
                 artist_slug=ctx["artist_slug"],
                 platform=platform,
                 error_type=ErrorType.NOT_FOUND,
-                error_message=_campaign_resolution_error(platform, url),
+                error_message=(
+                    f"Could not resolve Patreon campaign id. source_url={url!r}; "
+                    f"vanity={vanity!r}; "
+                    f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
+                    "(vanity lookup failed — cookies expired or creator moved?)"
+                ),
             ),
             None,
         )
@@ -179,10 +161,6 @@ async def _run_native_ingester(
         validate=gdl._validate_files,
         rate_limit=rate_limit,
         request_sleep=request_sleep,
-        # Uniform across adapters: token platforms (pixiv) authenticate with
-        # it, cookie platforms accept-and-ignore — so this construction stays
-        # platform-agnostic.
-        auth_token=ctx["auth_token"],
     )
     loop = asyncio.get_running_loop()
     dl_result = await loop.run_in_executor(
@@ -203,6 +181,54 @@ async def _run_native_ingester(
     return dl_result, resolved_campaign_id
 
 
+async def preview_source(
+    *,
+    platform: str,
+    url: str,
+    source_id: int,
+    config_overrides: dict | None,
+    cookies_path: str | None,
+    images_root: Path,
+    sync_session_factory,
+    page_limit: int = 3,
+) -> dict:
+    """Dry-run preview for a native platform (plan #708 B4): resolve the campaign
+    id, then walk a few pages counting media not already seen/dead — no download.
+
+    Returns the preview dict (total_new / posts_scanned / pages_scanned /
+    has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure.
+    Native-only — the caller gates on `uses_native_ingester`.
+    """
+    import asyncio
+
+    campaign_id, _ = await _resolve_native_campaign_id(
+        platform, url, cookies_path, config_overrides or {}
+    )
+    if not campaign_id:
+        vanity = extract_vanity(url)
+        return {
+            "error": (
+                f"Couldn't resolve the campaign id. source_url={url!r}; "
+                f"vanity={vanity!r}; lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
+                "(cookies expired, or the creator moved/renamed?)."
+            )
+        }
+    ingester = _native_ingester_cls(platform)(
+        images_root=images_root,
+        cookies_path=cookies_path,
+        session_factory=sync_session_factory,
+    )
+    loop = asyncio.get_running_loop()
+    try:
+        result = await loop.run_in_executor(
+            None,
+            lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit),
+        )
+    except NativeIngestError as exc:
+        return {"error": f"Couldn't preview: {exc}"}
+    return result
+
+
 async def verify_source_credential(
     *,
     platform: str,
@@ -219,18 +245,13 @@ async def verify_source_credential(
     this and render the result.
     """
     if uses_native_ingester(platform):
-        # Native ingester platforms verify via their own lightweight auth probe.
+        # Native ingester platforms verify via their own lightweight auth probe
-        # SubscribeStar's probe takes the creator URL directly; Patreon's
+        # (one authenticated feed fetch). SubscribeStar's probe takes the creator
-        # resolves the campaign id first; Pixiv's is one OAuth refresh (the
+        # URL directly; Patreon's resolves the campaign id first.
-        # exact call that fails when the token is bad — no feed walk).
         if platform == "subscribestar":
             from .subscribestar_ingester import verify_subscribestar_credential
 
             return await verify_subscribestar_credential(url, cookies_path, config_overrides)
-        if platform == "pixiv":
-            from .pixiv_ingester import verify_pixiv_credential
-
-            return await verify_pixiv_credential(auth_token)
         from .patreon_ingester import verify_patreon_credential
 
         return await verify_patreon_credential(url, cookies_path, config_overrides)

backend/app/services/download_service.py

@@ -326,16 +326,14 @@ class DownloadService:
                 # for hours after a download landed. Lazy import to avoid
                 # circular-import risk between this service and the
                 # tasks/* modules that import it.
-                from ..tasks.ml import cpu_embed_enabled, embed_image
+                from ..tasks.ml import tag_and_embed
                 from ..tasks.thumbnail import generate_thumbnail
-                do_embed = cpu_embed_enabled()
                 ids = list(result.member_image_ids)
                 if result.image_id is not None and result.image_id not in ids:
                     ids.append(result.image_id)
                 for img_id in ids:
                     generate_thumbnail.delay(img_id)
-                    if do_embed:
+                    tag_and_embed.delay(img_id)
-                        embed_image.delay(img_id)
             elif result.status == "attached":
                 # Non-media or extracted archive captured as PostAttachment
                 # (FC-2d-iii). The canonical copy lives in the attachments
@@ -418,8 +416,7 @@ class DownloadService:
         # the duplicate file). Empty outside recapture mode.
         relink_pairs = getattr(dl_result, "relink_source_paths", None) or []
         relinked = 0
-        post_linked = 0
+        for rel_str, rel_url in relink_pairs:
-        for rel_str, rel_url, rel_post_id in relink_pairs:
             rel_path = Path(rel_str)
             if not rel_path.exists():  # noqa: ASYNC240
                 continue
@@ -429,27 +426,13 @@ class DownloadService:
 
             if await loop.run_in_executor(None, _relink):
                 relinked += 1
-
-            # #1288: link the on-disk image to its Post. Recapture disk-skips the
-            # media (never re-imported), so a pre-existing image (e.g. one pulled
-            # under the old gallery-dl path) otherwise stays orphaned even after
-            # the post record is written. Idempotent for already-linked images.
-            def _link(p=rel_path, pid=rel_post_id):
-                return self.importer.link_existing_image_to_post(
-                    p, pid, source=source_row, artist=artist,
-                )
-
-            if await loop.run_in_executor(None, _link):
-                post_linked += 1
         if relink_pairs:
             # recapture diagnostic: how many on-disk images got their
-            # source_filehash backfilled (inline-image localization) and how many
+            # source_filehash backfilled (inline-image localization). < total is
-            # got (re)linked to their Post. < total for source_filehash is normal
+            # normal — files already carrying a filehash are skipped (NULL-only).
-            # (files already carrying a filehash are skipped, NULL-only).
             log.info(
-                "recap: relinked source_filehash on %d and linked %d/%d on-disk "
+                "recap: relinked source_filehash on %d/%d on-disk image(s)",
-                "image(s) to their post",
+                relinked, len(relink_pairs),
-                relinked, post_linked, len(relink_pairs),
             )
 
         # Kick the off-platform file-host downloader for any links this run

backend/app/services/extension_service.py

@@ -8,7 +8,6 @@ and returns a JSON-shaped dict for the API layer.
 
 from __future__ import annotations
 
-import logging
 import re
 
 from sqlalchemy import select
@@ -19,8 +18,6 @@ from ..utils.slug import slugify
 from .db_helpers import get_or_create
 from .source_service import BACKFILL_MAX_CHUNKS
 
-log = logging.getLogger(__name__)
-
 
 class UnknownPlatformError(Exception):
     """URL didn't match any platform pattern."""
@@ -64,38 +61,15 @@ _PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
 
 
 class ExtensionService:
-    def __init__(self, session: AsyncSession, crypto=None) -> None:
+    def __init__(self, session: AsyncSession) -> None:
         self.session = session
-        # Optional decryptor for resolving a token-auth platform's display name
-        # (pixiv) at add-time. None → skip resolution, fall back to the handle.
-        self._crypto = crypto
 
     async def quick_add_source(self, url: str) -> dict:
         platform, raw_slug = self._derive(url)
-        # Identity by SOURCE handle (#130): an existing (platform, url) source
+        artist, created_artist = await self._find_or_create_artist(raw_slug)
-        # keeps its artist on re-add — even if that artist was since renamed (its
-        # frozen slug no longer matches the current name). Only a genuinely new
-        # source resolves/creates an artist.
-        existing = (await self.session.execute(
-            select(Source).where(Source.platform == platform, Source.url == url)
-        )).scalar_one_or_none()
-        if existing is not None:
-            artist = (await self.session.execute(
-                select(Artist).where(Artist.id == existing.artist_id)
-            )).scalar_one()
-            return self._shape(existing, artist, created_source=False, created_artist=False)
-
-        # New source → name the artist properly by resolving the real display
-        # name from the platform (falls back to the URL handle).
-        name = await self._resolve_artist_name(platform, raw_slug, url)
-        artist, created_artist = await self._find_or_create_artist(name)
         source, created_source = await self._find_or_create_source(
             artist_id=artist.id, platform=platform, url=url,
         )
-        return self._shape(source, artist, created_source, created_artist)
-
-    @staticmethod
-    def _shape(source, artist, created_source: bool, created_artist: bool) -> dict:
         return {
             "source": {
                 "id": source.id,
@@ -113,50 +87,6 @@ class ExtensionService:
             "created_artist": created_artist,
         }
 
-    async def _resolve_artist_name(
-        self, platform: str, raw_slug: str, url: str
-    ) -> str:
-        """The real display name for a new artist, resolved from the platform at
-        add-time (#130). Our native platforms each have a name source — pixiv the
-        app API (token), patreon the campaigns API, subscribestar the profile
-        page (both cookies). Other platforms (and any failure — no credential,
-        network error) fall back to the URL handle, which is already readable.
-        The resolvers are sync, so they run in an executor."""
-        if self._crypto is None or platform not in ("pixiv", "patreon", "subscribestar"):
-            return raw_slug
-        import asyncio
-
-        from .credential_service import CredentialService
-        cred = CredentialService(self.session, self._crypto)
-        loop = asyncio.get_running_loop()
-        try:
-            if platform == "pixiv":
-                token = await cred.get_token("pixiv")
-                if not token:
-                    return raw_slug
-                from .pixiv_client import PixivClient
-                name = await loop.run_in_executor(
-                    None, PixivClient(token).resolve_display_name, raw_slug
-                )
-            elif platform == "patreon":
-                cookies = await cred.get_cookies_path("patreon")
-                from .patreon_resolver import resolve_display_name
-                name = await loop.run_in_executor(
-                    None, resolve_display_name, raw_slug,
-                    str(cookies) if cookies else None,
-                )
-            else:  # subscribestar
-                cookies = await cred.get_cookies_path("subscribestar")
-                from .subscribestar_client import SubscribeStarClient
-                client = SubscribeStarClient(str(cookies) if cookies else None)
-                name = await loop.run_in_executor(
-                    None, client.resolve_display_name, url
-                )
-        except Exception as exc:      # resolution is best-effort — never block the add
-            log.warning("artist display-name resolution failed (%s): %s", platform, exc)
-            return raw_slug
-        return name or raw_slug
-
     async def probe(self, url: str) -> dict:
         """Read-only resolution of a creator-page URL against the FC DB.
         Returns one of:

backend/app/services/external_links.py

@@ -1,52 +0,0 @@
-"""Surgical re-fetch of a post's external file-host links.
-
-The normal download cadence never re-walks deep back-catalogue posts (the
-seen-gates exist precisely to keep old items from resurfacing), so when a
-file that CAME from an ExternalLink is deleted — e.g. the failure-triage
-recovery flow removing a corrupt original — a plain source re-check will
-never bring it back. The link ROW is the durable, per-post handle: resetting
-it to pending and dispatching the fetch re-downloads exactly that link's
-payload, and sha-dedupe at import discards anything that still exists — so
-only the missing file actually lands. (Operator 2026-07-03: recovery must
-not require artist-wide deep scans.)
-"""
-import logging
-
-from sqlalchemy import select
-from sqlalchemy.orm import Session
-
-from ..models import ExternalLink
-
-log = logging.getLogger(__name__)
-
-
-def refetch_links_for_post(session: Session, post_id: int) -> dict:
-    """Reset every settled ExternalLink on a post to pending (fresh attempt
-    budget) and dispatch its fetch. In-flight ('downloading') links are left
-    alone. Commits. Returns {links_total, links_reset}."""
-    links = session.execute(
-        select(ExternalLink).where(ExternalLink.post_id == post_id)
-    ).scalars().all()
-    reset_ids = []
-    for link in links:
-        if link.status == "downloading":
-            continue
-        link.status = "pending"
-        link.attempts = 0
-        link.last_error = None
-        link.completed_at = None
-        reset_ids.append(link.id)
-    session.commit()
-    if reset_ids:
-        # Lazy import (services -> tasks would cycle at module load). The
-        # 10-min extdl sweep would pick pending rows up anyway — dispatching
-        # directly just skips the wait.
-        from ..tasks.external import fetch_external_link
-
-        for lid in reset_ids:
-            fetch_external_link.delay(lid)
-        log.info(
-            "external refetch: post %s — %d/%d link(s) reset + dispatched",
-            post_id, len(reset_ids), len(links),
-        )
-    return {"links_total": len(links), "links_reset": len(reset_ids)}

backend/app/services/gallery_dl.py

@@ -157,7 +157,7 @@ class DownloadResult:
     # pairs for already-present media whose ImageRecord.source_filehash should be
     # backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
     # on the gallery-dl path and outside recapture.
-    relink_source_paths: list[tuple[str, str, str]] = field(default_factory=list)
+    relink_source_paths: list[tuple[str, str]] = field(default_factory=list)
     stdout: str = ""
     stderr: str = ""
     return_code: int = 0
@@ -298,9 +298,8 @@ class GalleryDLService:
     # removed at the plan-#697 cutover — it now uses the native ingester
     # (services/patreon_ingester.py), not gallery-dl.
     PLATFORM_DEFAULTS = {
-        # subscribestar removed — native-ingester platform now (#71); pixiv
+        # subscribestar removed — it's a native-ingester platform now (#71); the
-        # removed likewise (#129). The remaining entries are the gallery-dl
+        # remaining entries are the gallery-dl platforms not yet migrated.
-        # platforms not yet migrated.
         "hentaifoundry": {
             "content_types": ["all"],
             "directory": [],
@@ -316,6 +315,12 @@ class GalleryDLService:
             "reactions": False,
             "threads": True,
         },
+        "pixiv": {
+            "content_types": ["all"],
+            "directory": ["{category}"],
+            "filename": "{id}_{title[:50]}_{num:>02}.{extension}",
+            "ugoira": True,
+        },
         "deviantart": {
             "content_types": ["all"],
             "directory": [],
@@ -689,6 +694,9 @@ class GalleryDLService:
         if auth_token and platform == "discord":
             config["extractor"].setdefault("discord", {})
             config["extractor"]["discord"]["token"] = auth_token
+        if auth_token and platform == "pixiv":
+            config["extractor"].setdefault("pixiv", {})
+            config["extractor"]["pixiv"]["refresh-token"] = auth_token
 
         with tempfile.NamedTemporaryFile(
             mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
@@ -874,6 +882,8 @@ class GalleryDLService:
             config["extractor"]["cookies"] = cookies_path
         if auth_token and platform == "discord":
             config["extractor"].setdefault("discord", {})["token"] = auth_token
+        if auth_token and platform == "pixiv":
+            config["extractor"].setdefault("pixiv", {})["refresh-token"] = auth_token
 
         with tempfile.NamedTemporaryFile(
             mode="w", suffix=".json", delete=False, dir=str(self._config_dir),

backend/app/services/gallery_service.py

@@ -22,16 +22,8 @@ from sqlalchemy import Select, and_, distinct, exists, func, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import aliased
 
-from ..models import (
+from ..models import Artist, ImageProvenance, ImageRecord, Post, Source, Tag
-    Artist,
+from ..models.tag import image_tag
-    ImageProvenance,
-    ImageRecord,
-    Post,
-    Source,
-    Tag,
-    TagPositiveConfirmation,
-)
-from ..models.tag import CHROME_SYSTEM_TAGS, PROCESS_SYSTEM_TAGS, image_tag
 from .pagination import decode_cursor, encode_cursor
 from .tag_query import (
     fandom_join_alias,
@@ -63,25 +55,6 @@ def _effective_date_col():
     return ImageRecord.effective_date
 
 
-# Sort key -> the materialized column the gallery orders + cursors on. Both are
-# indexed (DESC, id DESC), so every sort is a forward index range scan.
-#   newest/oldest    → effective_date    (primary post's date, else download)
-#   posted_new/_old  → earliest_post_date (earliest publish across ALL posts)
-_SORT_COLUMNS = {
-    "newest": ImageRecord.effective_date,
-    "oldest": ImageRecord.effective_date,
-    "posted_new": ImageRecord.earliest_post_date,
-    "posted_old": ImageRecord.earliest_post_date,
-}
-_ASCENDING_SORTS = {"oldest", "posted_old"}
-
-
-def _sort_column(sort: str):
-    """The materialized date column a gallery sort orders/cursors on (falls back
-    to effective_date for any unknown sort)."""
-    return _SORT_COLUMNS.get(sort, ImageRecord.effective_date)
-
-
 def _outer_join_primary_post(stmt: Select) -> Select:
     """LEFT JOIN Post on ImageRecord.primary_post_id so the COALESCE
     above sees Post.post_date when available. Images without a post
@@ -187,7 +160,7 @@ def _apply_scope(
     stmt, *, tag_ids, post_id, artist_id, media_type,
     tag_or_groups=None, tag_exclude=None,
     platform=None, untagged=False, no_artist=False,
-    date_from=None, date_to=None, hidden_tag_ids=None,
+    date_from=None, date_to=None,
 ):
     """Apply the composable gallery filters to a statement.
 
@@ -224,12 +197,6 @@ def _apply_scope(
         stmt = stmt.where(image_in_any_tag_scope(group))
     if tag_exclude:
         stmt = stmt.where(~image_in_any_tag_scope(tag_exclude))
-    # Presentation chrome (banner / editor screenshot) is hidden from the default
-    # gallery — an implicit exclude the caller supplies unless the operator asked
-    # to include hidden or is explicitly filtering for a presentation tag
-    # (milestone 141). `wip` is NOT hidden. Resolved to ids by _hidden_tag_ids.
-    if hidden_tag_ids:
-        stmt = stmt.where(~image_in_any_tag_scope(hidden_tag_ids))
     prov = _provenance_clause(post_id, artist_id)
     if prov is not None:
         stmt = stmt.where(prov)
@@ -322,7 +289,7 @@ def _gallery_images(rows, artists: dict[int, dict]) -> list[GalleryImage]:
     ]
 
 
-def _diversify_similar(src, rows, limit, *, dup_threshold=8, lam=0.40):
+def _diversify_similar(src, rows, limit, *, dup_threshold=6, lam=0.55):
     """Trim a nearest-cosine candidate pool down to `limit` diverse picks.
 
     1. pHash collapse: drop any candidate whose perceptual hash is within
@@ -333,11 +300,6 @@ def _diversify_similar(src, rows, limit, *, dup_threshold=8, lam=0.40):
        the most relevant up top but pushes the selection to SPAN clusters
        instead of returning 40 variations of one image.
 
-    `lam` is the variance dial: lower = weight the diversity penalty harder, so
-    the rail reaches further across clusters (operator wanted MORE variance,
-    2026-07-01 — dropped 0.55→0.40, dup 6→8, paired with a wider pool in
-    `similar()`).
-
     Falls back to nearest-order (`rows[:limit]`) on any failure or a small pool.
     """
     if len(rows) <= 1:
@@ -416,32 +378,6 @@ class GalleryService:
     def __init__(self, session: AsyncSession):
         self.session = session
 
-    async def _hidden_tag_ids(
-        self, include_hidden, tag_ids, tag_or_groups,
-    ) -> list[int] | None:
-        """Chrome (banner) tag ids to implicitly exclude from a gallery query, or
-        None. None when the caller asked to include hidden, when the operator is
-        explicitly filtering FOR a chrome tag (they clearly want to see it), or when
-        no chrome tags exist. (milestone 141; #1464: editor screenshot is now PROCESS
-        — shown — so only `banner` hides here.)"""
-        if include_hidden:
-            return None
-        rows = await self.session.execute(
-            select(Tag.id).where(
-                Tag.is_system.is_(True),
-                Tag.name.in_(CHROME_SYSTEM_TAGS),
-            )
-        )
-        pres = [r[0] for r in rows]
-        if not pres:
-            return None
-        explicit = set(tag_ids or [])
-        for group in tag_or_groups or []:
-            explicit.update(group)
-        if explicit & set(pres):
-            return None
-        return pres
-
     async def scroll(
         self,
         cursor: str | None,
@@ -458,21 +394,14 @@ class GalleryService:
         no_artist: bool = False,
         date_from: datetime | None = None,
         date_to: datetime | None = None,
-        include_hidden: bool = False,
     ) -> GalleryPage:
         if limit < 1 or limit > 200:
             raise ValueError("limit must be between 1 and 200")
         _require_single_filter(
             tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
         )
-        hidden = await self._hidden_tag_ids(
-            include_hidden, tag_ids, tag_or_groups,
-        )
 
-        # eff is the ACTIVE sort column (effective_date or earliest_post_date);
+        eff = _effective_date_col()
-        # the cursor, ordering and year/month grouping all key off it, so the
-        # 'post date' sort paginates + buckets by original publish transparently.
-        eff = _sort_column(sort)
         stmt = select(ImageRecord, Post.post_date, eff.label("eff"))
         stmt = _outer_join_primary_post(stmt)
         stmt = _apply_scope(
@@ -480,10 +409,10 @@ class GalleryService:
             artist_id=artist_id, media_type=media_type,
             tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
             platform=platform, untagged=untagged, no_artist=no_artist,
-            date_from=date_from, date_to=date_to, hidden_tag_ids=hidden,
+            date_from=date_from, date_to=date_to,
         )
 
-        descending = sort not in _ASCENDING_SORTS
+        descending = sort != "oldest"
         if cursor:
             cur_ts, cur_id = decode_cursor(cursor)
             # The cursor is just (last eff, last id); the request's sort
@@ -533,7 +462,6 @@ class GalleryService:
         no_artist: bool = False,
         date_from: datetime | None = None,
         date_to: datetime | None = None,
-        include_hidden: bool = False,
     ) -> list[TimelineBucket]:
         eff = _effective_date_col()
         year_col = func.date_part("year", eff).label("yr")
@@ -545,15 +473,12 @@ class GalleryService:
         _require_single_filter(
             tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
         )
-        hidden = await self._hidden_tag_ids(
-            include_hidden, tag_ids, tag_or_groups,
-        )
         stmt = _apply_scope(
             stmt, tag_ids=tag_ids, post_id=post_id,
             artist_id=artist_id, media_type=media_type,
             tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
             platform=platform, untagged=untagged, no_artist=no_artist,
-            date_from=date_from, date_to=date_to, hidden_tag_ids=hidden,
+            date_from=date_from, date_to=date_to,
         )
         stmt = stmt.group_by(year_col, month_col).order_by(year_col.desc(), month_col.desc())
         rows = (await self.session.execute(stmt)).all()
@@ -567,7 +492,7 @@ class GalleryService:
         tag_exclude: list[int] | None = None,
         platform: str | None = None, untagged: bool = False,
         no_artist: bool = False, date_from: datetime | None = None,
-        date_to: datetime | None = None, include_hidden: bool = False,
+        date_to: datetime | None = None,
     ) -> str | None:
         """Returns a cursor that, when passed to scroll() with the same sort,
         positions at the first image of the given year-month. None if the
@@ -584,15 +509,12 @@ class GalleryService:
         _require_single_filter(
             tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
         )
-        hidden = await self._hidden_tag_ids(
-            include_hidden, tag_ids, tag_or_groups,
-        )
         stmt = _apply_scope(
             stmt, tag_ids=tag_ids, post_id=post_id,
             artist_id=artist_id, media_type=media_type,
             tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
             platform=platform, untagged=untagged, no_artist=no_artist,
-            date_from=date_from, date_to=date_to, hidden_tag_ids=hidden,
+            date_from=date_from, date_to=date_to,
         )
         descending = sort != "oldest"
         if descending:
@@ -617,7 +539,6 @@ class GalleryService:
         platform: str | None = None,
         untagged: bool = False, no_artist: bool = False,
         date_from: datetime | None = None, date_to: datetime | None = None,
-        include_hidden: bool = False,
     ) -> GalleryFacets:
         """Live facet counts scoped to the current filter. Each facet GROUP is
         computed with all OTHER active filters applied but its OWN selection
@@ -628,14 +549,10 @@ class GalleryService:
         _require_single_filter(
             tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
         )
-        hidden = await self._hidden_tag_ids(
-            include_hidden, tag_ids, tag_or_groups,
-        )
         common = {
             "tag_ids": tag_ids, "post_id": post_id,
             "artist_id": artist_id, "media_type": media_type,
             "tag_or_groups": tag_or_groups, "tag_exclude": tag_exclude,
-            "hidden_tag_ids": hidden,
         }
 
         # total — the full active filter (the headline result count).
@@ -716,7 +633,6 @@ class GalleryService:
         platform: str | None = None,
         untagged: bool = False, no_artist: bool = False,
         date_from: datetime | None = None, date_to: datetime | None = None,
-        exclude_wip: bool = False,
     ) -> list[GalleryImage] | None:
         """Visual "more like this": images near `image_id`'s SigLIP embedding
         (pgvector, HNSW-indexed — alembic 0036), then DIVERSIFIED so the result
@@ -742,36 +658,15 @@ class GalleryService:
             return []
 
         # Over-fetch so diversification has clusters to spread across — without a
-        # wide pool there's nothing but the near-dupes to choose from. Widened
+        # wide pool there's nothing but the near-dupes to choose from.
-        # (5×→8×, cap 200→400) so the stronger MMR has genuinely distinct
+        pool_n = min(200, max(limit * 5, 60))
-        # neighbourhoods to reach into for more variance (operator, 2026-07-01).
-        pool_n = min(400, max(limit * 8, 100))
         distance = ImageRecord.siglip_embedding.cosine_distance(src.siglip_embedding)
         eff = _effective_date_col()
         stmt = select(ImageRecord, Post.post_date, eff.label("eff"))
         stmt = _outer_join_primary_post(stmt)
-        # Chrome (banner, #128) clusters on UI rather than content, so near any one
-        # of them they'd fill the grid → excluded from CANDIDATES always (the anchor
-        # itself may be a banner). PROCESS art (wip / editor screenshot) stays
-        # surfaced here by default (real content; only the training pipelines exclude
-        # it), but the Explore rabbit-hole passes exclude_wip to also drop the whole
-        # process group so a browse doesn't keep surfacing work-in-progress
-        # (operator, 2026-07-08; #1464 — editor now rides with wip here).
-        excluded_system_tags = CHROME_SYSTEM_TAGS
-        if exclude_wip:
-            excluded_system_tags = (*CHROME_SYSTEM_TAGS, *PROCESS_SYSTEM_TAGS)
-        presentation = (
-            select(image_tag.c.image_record_id)
-            .join(Tag, Tag.id == image_tag.c.tag_id)
-            .where(
-                Tag.is_system.is_(True),
-                Tag.name.in_(excluded_system_tags),
-            )
-        )
         stmt = stmt.where(
             ImageRecord.siglip_embedding.is_not(None),
             ImageRecord.id != image_id,
-            ImageRecord.id.not_in(presentation),
         )
         stmt = _apply_scope(
             stmt, tag_ids=tag_ids, post_id=None,
@@ -793,14 +688,8 @@ class GalleryService:
         # Self-join Tag to resolve a character's fandom NAME (not just id) so the
         # modal chip can label it without an N+1 (shared tag_query helpers).
         fandom_alias = fandom_join_alias()
-        # source drives the auto-applied badge; confirmed = operator affirmed the
-        # tag (positive + retraction-shielded, milestone 139).
-        confirmed = exists().where(
-            TagPositiveConfirmation.image_record_id == image_id,
-            TagPositiveConfirmation.tag_id == Tag.id,
-        ).label("confirmed")
         tag_stmt = (
-            select(*tag_columns(fandom_alias), image_tag.c.source, confirmed)
+            select(*tag_columns(fandom_alias))
             .select_from(
                 Tag.__table__
                 .join(image_tag, image_tag.c.tag_id == Tag.id)

backend/app/services/importer.py

@@ -17,7 +17,7 @@ from enum import StrEnum
 from pathlib import Path
 
 from PIL import Image
-from sqlalchemy import func, select, update
+from sqlalchemy import select, update
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
@@ -44,17 +44,11 @@ from ..utils.sidecar import find_sidecar, parse_sidecar
 from ..utils.slug import slugify
 from .archive_extractor import extract_archive, is_archive
 from .attachment_store import AttachmentStore
-from .audits import single_color
 from .link_extract import extract_external_links
 from .thumbnailer import Thumbnailer
-from .wip_title import apply_wip_image_tags, matches_wip_title, resolve_wip_tag_id
 
 log = logging.getLogger(__name__)
 
-# Sentinel for the lazily-resolved wip tag id (distinguishes "not resolved yet"
-# from a genuine None = tag absent, so absence is cached and not re-queried).
-_UNSET = object()
-
 
 class SkipReason(StrEnum):
     too_small = "too_small"
@@ -188,10 +182,6 @@ class Importer:
         # invalidated mid-Importer (Importer instances are per-task /
         # per-archive-import so cross-instance staleness is harmless).
         self._phash_candidates: list[tuple] | None = None
-        # Lazily-resolved `wip` system tag id for title-based WIP auto-tagging
-        # (task #1458). Sentinel _UNSET so a genuine None (tag absent) is cached
-        # and not re-queried per media. Importer is per-task, so this can't stale.
-        self._wip_tag_id: int | None = _UNSET
 
     def _phash_candidates_cache(self) -> list[tuple]:
         """Cached `(phash, width, height, id)` rows from image_record.
@@ -800,13 +790,6 @@ class Importer:
                         error=f"{pct:.2%} transparent",
                     )
 
-            if self.settings.skip_single_color and self._single_color_hit(source):
-                return ImportResult(
-                    status="skipped", skip_reason=SkipReason.single_color,
-                    error=(f"one color dominates >"
-                           f"{self.settings.single_color_threshold:.0%}"),
-                )
-
         # Artist anchored to the attribution path (folder→artist), resolved
         # UP-FRONT so the enrich-on-duplicate branches link provenance with the
         # right artist even when the sidecar carries none — which is now the norm
@@ -942,10 +925,6 @@ class Importer:
         # Thumbnail is queued separately by the calling task; the importer
         # does not generate thumbnails inline so the import queue stays moving.
 
-        # Title-based WIP auto-tag (task #1458): fresh import only, after the
-        # sidecar has linked the post so record.primary_post_id / its title exist.
-        self._maybe_apply_wip_title(record)
-
         self.session.commit()
         return ImportResult(status="imported", image_id=record.id)
 
@@ -989,36 +968,6 @@ class Importer:
         self.session.commit()
         return ImportResult(status="refreshed", image_id=existing.id)
 
-    def _maybe_apply_wip_title(self, record: ImageRecord) -> None:
-        """Auto-apply the `wip` system tag to a FRESHLY-imported image when its
-        primary post's TITLE explicitly declares work-in-progress (task #1458 —
-        the artist's own "WIP" / "work in progress" label).
-
-        Called ONLY from the two new-record paths (never deep-scan / supersede),
-        so a manually-removed WIP tag is never re-applied by a routine re-scan —
-        removal sticks. The existing catalogue is covered separately by the
-        operator-triggered backfill sweep. Gated by the settings toggle, and
-        best-effort: any failure is logged, never allowed to fail the import."""
-        if not self.settings.wip_title_tagging_enabled:
-            return
-        if record.primary_post_id is None:
-            return
-        try:
-            title = self.session.execute(
-                select(Post.post_title).where(Post.id == record.primary_post_id)
-            ).scalar_one_or_none()
-            if not matches_wip_title(title):
-                return
-            if self._wip_tag_id is _UNSET:
-                self._wip_tag_id = resolve_wip_tag_id(self.session)
-            if self._wip_tag_id is None:
-                return
-            apply_wip_image_tags(self.session, [record.id], self._wip_tag_id)
-        except Exception as exc:  # noqa: BLE001 — a tag must never fail an import
-            log.warning(
-                "wip-title auto-tag failed for image %s: %s", record.id, exc
-            )
-
     def _apply_post_fields(self, post: Post, sd) -> None:
         """Write a parsed sidecar's post-level fields onto a Post — the SINGLE
         predicate shared by BOTH ingest paths: the per-media path (_apply_sidecar)
@@ -1174,13 +1123,6 @@ class Importer:
                         status="skipped", skip_reason=SkipReason.too_transparent,
                         error=f"{pct:.2%} transparent",
                     )
-
-            if self.settings.skip_single_color and self._single_color_hit(path):
-                return ImportResult(
-                    status="skipped", skip_reason=SkipReason.single_color,
-                    error=(f"one color dominates >"
-                           f"{self.settings.single_color_threshold:.0%}"),
-                )
         else:
             # Best-effort probe for dims + duration so downloaded videos can dedup
             # (#871). LENIENT: unlike _import_media this path does not reject on a
@@ -1296,10 +1238,6 @@ class Importer:
         # per-post Source row.
         self._apply_sidecar(record, path, artist, explicit_source=source)
 
-        # Title-based WIP auto-tag (task #1458): fresh import only, see the
-        # matching call in _import_media.
-        self._maybe_apply_wip_title(record)
-
         self.session.commit()
         return ImportResult(status="imported", image_id=record.id)
 
@@ -1360,113 +1298,6 @@ class Importer:
         self.session.commit()
         return True
 
-    def link_existing_image_to_post(
-        self, path: Path, external_post_id: str, *,
-        source: Source | None = None, artist: Artist | None = None,
-    ) -> bool:
-        """Recapture back-link: associate an ALREADY-on-disk image (matched by
-        its stored path) with its post — the link _apply_sidecar normally makes
-        at per-media import time.
-
-        Recapture disk-skips downloaded media, so a pre-existing image (e.g. one
-        pulled under the old gallery-dl path, imported as a bare record with no
-        post) never gets its `image_provenance` row / `primary_post_id`. This
-        backfills it from the walk's (on-disk path, post external id) pairing:
-        find the ImageRecord by path, find/attach the Post by (source,
-        external_post_id), upsert provenance. Idempotent (issue #1288). Returns
-        True when a record matched and was linked; no-op when the file has no
-        record or no id is given."""
-        if not external_post_id:
-            return False
-        record = self.session.execute(
-            select(ImageRecord).where(ImageRecord.path == str(path))
-        ).scalar_one_or_none()
-        if record is None:
-            return False
-        artist_id = record.artist_id or (artist.id if artist else None)
-        if artist_id is None:
-            return False
-        if record.artist_id is None:
-            record.artist_id = artist_id
-        post = self._find_or_create_post(
-            source_id=source.id if source else None,
-            external_post_id=str(external_post_id),
-            artist_id=artist_id,
-        )
-        self._attach_provenance(
-            record, post, source_id=source.id if source else None,
-        )
-        self.session.commit()
-        return True
-
-    def _attach_provenance(
-        self, record: ImageRecord, post: Post, *,
-        source_id: int | None, captured_metadata: dict | None = None,
-    ) -> None:
-        """Upsert the (image, post) `image_provenance` link + keep
-        `primary_post_id` and the denormalized gallery sort keys aligned. Shared
-        by _apply_sidecar (fresh per-media import) and link_existing_image_to_post
-        (recapture back-link, #1288) so the two paths can't diverge. Idempotent.
-
-        Race-safe (image_record_id, post_id) upsert — mirrors the
-        _find_or_create_source/post savepoint pattern. The plain
-        SELECT-then-INSERT pattern lost a race when two workers ran on the same
-        (image, post) pair (e.g. the 5-min recovery sweep re-enqueued a
-        still-running long import), planting duplicates that then broke
-        .scalar_one_or_none() on every later deep-scan rederive
-        (MultipleResultsFound). Alembic 0021's uq_image_provenance_image_post
-        UNIQUE makes this savepoint trip on collision."""
-        exists = self.session.execute(
-            select(ImageProvenance.id).where(
-                ImageProvenance.image_record_id == record.id,
-                ImageProvenance.post_id == post.id,
-            )
-        ).scalar_one_or_none()
-        if exists is None:
-            sp = self.session.begin_nested()
-            try:
-                self.session.add(
-                    ImageProvenance(
-                        image_record_id=record.id,
-                        post_id=post.id,
-                        source_id=source_id,
-                        captured_metadata=captured_metadata,
-                    )
-                )
-                self.session.flush()
-                sp.commit()
-            except IntegrityError:
-                sp.rollback()
-        if record.primary_post_id is None:
-            record.primary_post_id = post.id
-        # Keep the denormalized gallery sort key (alembic 0035) aligned with
-        # the primary post's publish date so /scroll orders off
-        # ix_image_record_effective_date instead of COALESCE-ing across the
-        # post join. Only override when THIS post is the primary AND carries
-        # a date; otherwise the column keeps its created_at-equivalent server
-        # default (matches the old COALESCE(post_date, created_at) fallback).
-        if record.primary_post_id == post.id and post.post_date is not None:
-            record.effective_date = post.post_date
-        # earliest_post_date (alembic 0071) = MIN(post_date) across ALL of this
-        # image's provenance posts, not just the primary — so the gallery can
-        # sort by original publish rather than the download/repost the primary
-        # points at. Recompute from provenance whenever a dated post is linked;
-        # the provenance row for THIS post was committed above, so the MIN
-        # includes it. Leaves the created_at default when no linked post is dated.
-        if post.post_date is not None:
-            earliest = self.session.execute(
-                select(func.min(Post.post_date))
-                .select_from(ImageProvenance)
-                .join(Post, Post.id == ImageProvenance.post_id)
-                .where(
-                    ImageProvenance.image_record_id == record.id,
-                    Post.post_date.is_not(None),
-                )
-            ).scalar_one_or_none()
-            if earliest is not None:
-                record.earliest_post_date = earliest
-        self.session.flush()
-
     def _apply_sidecar(
         self,
         record: ImageRecord,
@@ -1540,12 +1371,47 @@ class Importer:
         if not self.post_first:
             self._apply_post_fields(post, sd)
 
-        # Link the (image, post) provenance + keep primary_post_id / the gallery
+        # Race-safe (image_record_id, post_id) upsert — mirrors the
-        # sort keys aligned — shared with the recapture back-link path (#1288).
+        # _find_or_create_source/post savepoint pattern. The plain
-        self._attach_provenance(
+        # SELECT-then-INSERT pattern lost a race when two workers ran
-            record, post, source_id=src.id if src else None,
+        # _apply_sidecar on the same (image, post) pair (e.g. the 5-min
-            captured_metadata=sd.raw,
+        # recovery sweep re-enqueued a still-running long import), planting
-        )
+        # duplicates that then broke .scalar_one_or_none() on every later
+        # deep-scan rederive (MultipleResultsFound). Alembic 0021 adds the
+        # uq_image_provenance_image_post UNIQUE so this savepoint actually
+        # trips on collision.
+        exists = self.session.execute(
+            select(ImageProvenance.id).where(
+                ImageProvenance.image_record_id == record.id,
+                ImageProvenance.post_id == post.id,
+            )
+        ).scalar_one_or_none()
+        if exists is None:
+            sp = self.session.begin_nested()
+            try:
+                self.session.add(
+                    ImageProvenance(
+                        image_record_id=record.id,
+                        post_id=post.id,
+                        source_id=src.id if src else None,
+                        captured_metadata=sd.raw,
+                    )
+                )
+                self.session.flush()
+                sp.commit()
+            except IntegrityError:
+                sp.rollback()
+        if record.primary_post_id is None:
+            record.primary_post_id = post.id
+        # Keep the denormalized gallery sort key (alembic 0035) aligned with
+        # the primary post's publish date so /scroll orders off
+        # ix_image_record_effective_date instead of COALESCE-ing across the
+        # post join. Only override when THIS post is the primary AND carries
+        # a date; otherwise the column keeps its created_at-equivalent server
+        # default (matches the old COALESCE(post_date, created_at) fallback).
+        if record.primary_post_id == post.id and post.post_date is not None:
+            record.effective_date = post.post_date
+        self.session.flush()
 
     def _copy_to_library(
         self, source: Path, sha: str, attribution_path: Path
@@ -1654,33 +1520,6 @@ class Importer:
                 # Benign orphan; the DB swap already committed. Don't undo it.
                 pass
 
-    # Matches the Cleanup audit card's default tolerance: the import-side
-    # filter and the retroactive audit must agree on what "single color" MEANS
-    # (Euclidean RGB distance to the dominant color); only the match threshold
-    # is operator-tunable per surface.
-    _SINGLE_COLOR_TOLERANCE = 30
-
-    def _single_color_hit(self, source: Path) -> bool:
-        """True when one color dominates beyond the configured threshold — the
-        same canonical predicate the Cleanup audit runs (audits.single_color,
-        whose docstring anticipated this adoption; the skip_single_color
-        setting existed but was never wired until 2026-07-02). Never raises:
-        unreadable files were already rejected by verify() upstream, and a
-        residual decode error just declines to match (the import proceeds)."""
-        try:
-            with Image.open(source) as im:
-                if getattr(im, "is_animated", False):
-                    # Frame 0 only would misjudge animations; skip like the
-                    # transparency check does.
-                    return False
-                return single_color.evaluate(
-                    im,
-                    threshold=self.settings.single_color_threshold,
-                    tolerance=self._SINGLE_COLOR_TOLERANCE,
-                )
-        except Exception:
-            return False
-
     def _transparency_pct(self, source: Path) -> float:
         """Fraction of fully-transparent pixels in the image. 0.0 if no alpha.
 

backend/app/services/ingest_core.py

@@ -90,7 +90,6 @@ class Ingester:
         platform: str,
         error_base: type[Exception],
         drift_label: str | None = None,
-        body_canary: bool = True,
     ):
         self.client = client
         self.downloader = downloader
@@ -106,12 +105,6 @@ class Ingester:
         # update"). Defaults to the platform name; adapters pass a richer phrase
         # (e.g. "Patreon API", "SubscribeStar markup").
         self._drift_label = drift_label or platform
-        # #862 canary opt-out: platforms whose posts legitimately have empty
-        # bodies across large samples (pixiv — caption-less artists are common)
-        # would false-positive the zero-bodies-means-drift alarm; their clients
-        # catch drift structurally (response-shape checks) instead. The
-        # "bodies X/N" summary line still surfaces the ratio either way.
-        self._body_canary = body_canary
 
     # -- public ------------------------------------------------------------
 
@@ -179,11 +172,10 @@ class Ingester:
         written: list[str] = []
         post_records: list[str] = []
         quarantined_paths: list[str] = []
-        # #830 recapture: (on-disk path, CDN source_url, post_id) triples for
+        # #830 recapture: (on-disk path, CDN source_url) pairs for already-present
-        # already-present media, so phase 3 can (a) backfill the ImageRecord's
+        # media, so phase 3 can backfill the ImageRecord's source_filehash WITHOUT
-        # source_filehash and (b) link the on-disk image to its Post (#1288) —
+        # re-downloading or unlinking the file. Empty outside recapture mode.
-        # WITHOUT re-downloading or unlinking the file. Empty outside recapture.
+        relink: list[tuple[str, str]] = []
-        relink: list[tuple[str, str, str]] = []
         downloaded = 0
         errors = 0
         quarantined = 0
@@ -411,15 +403,12 @@ class Ingester:
                         to_clear.append(key)
                         skipped_count += 1
                         consecutive_seen += 1
-                        # #830/#1288 recapture: surface (on-disk path, CDN url,
+                        # #830 recapture: surface (on-disk path, CDN url) so phase
-                        # post_id) so phase 3 can backfill source_filehash AND link
+                        # 3 can backfill source_filehash for inline-image
-                        # the on-disk image to its Post — a SEPARATE non-deleting
+                        # localization — a SEPARATE non-deleting channel, never the
-                        # channel, never the import list (which would unlink the
+                        # import list (which would unlink the file, per above).
-                        # file, per above).
                         if recapture and outcome.path is not None:
-                            relink.append(
+                            relink.append((str(outcome.path), media_item.url))
-                                (str(outcome.path), media_item.url, media_item.post_id)
-                            )
                     elif outcome.status == "skipped_seen":
                         skipped_count += 1
                         consecutive_seen += 1
@@ -547,11 +536,7 @@ class Ingester:
         # creds") so the breakage screams instead of silently archiving empties.
         # Only reached on an otherwise-clean walk (timeout/stop/error returned
         # above), so it never masks a more specific failure.
-        if (
+        if posts_recorded >= _CANARY_MIN_SAMPLE and posts_with_body == 0:
-            self._body_canary
-            and posts_recorded >= _CANARY_MIN_SAMPLE
-            and posts_with_body == 0
-        ):
             msg = (
                 f"Post-body canary: extracted a body from 0 of {posts_recorded} "
                 "posts — Patreon's body field shape likely changed; the ingester "
@@ -577,6 +562,73 @@ class Ingester:
             error_type=None, error_message=None,
         )
 
+    # -- preview (dry-run) -------------------------------------------------
+
+    def preview(
+        self,
+        source_id: int,
+        campaign_id: str,
+        *,
+        page_limit: int = 3,
+        sample_size: int = 10,
+    ) -> dict:
+        """Dry-run (plan #708 B4): walk up to `page_limit` pages and count media
+        NOT already in the seen/dead ledgers, WITHOUT downloading anything.
+
+        Read-only — only the seen/dead SELECTs touch the DB (short sessions). Lets
+        an operator gauge "is this source worth a backfill?" cheaply. Returns:
+            {total_new, posts_scanned, pages_scanned, has_more,
+             sample: [{title, date, new}, ...]}  # sample = posts with new media
+        A client-level failure (auth/drift) propagates to the caller.
+        """
+        total_new = 0
+        posts_scanned = 0
+        pages_scanned = 0
+        has_more = False
+        sample: list[dict] = []
+        unset = object()
+        last_page: object = unset
+        # #874: same gated-post gate as run() — the preview must not count
+        # blurred locked-preview media as "new", or it would overstate a gated
+        # source's backlog (preview/apply parity, rule 93).
+        post_is_gated = getattr(self.client, "post_is_gated", None)
+        for post, included, page_cursor in self.client.iter_posts(
+            campaign_id, cursor=None
+        ):
+            if page_cursor != last_page:
+                last_page = page_cursor
+                pages_scanned += 1
+                if pages_scanned > page_limit:
+                    has_more = True
+                    pages_scanned = page_limit
+                    break
+            posts_scanned += 1
+            if post_is_gated and post_is_gated(post):
+                continue
+            media = self.client.extract_media(post, included)
+            if not media:
+                continue
+            keys = [self._ledger_key(m) for m in media]
+            skip = self._seen_keys(source_id, keys) | self._dead_keys(source_id, keys)
+            new_count = sum(1 for m in media if self._ledger_key(m) not in skip)
+            total_new += new_count
+            if new_count > 0 and len(sample) < sample_size:
+                meta = self.client.post_meta(post)
+                sample.append(
+                    {
+                        "title": meta.get("title") or "(untitled)",
+                        "date": meta.get("date"),
+                        "new": new_count,
+                    }
+                )
+        return {
+            "total_new": total_new,
+            "posts_scanned": posts_scanned,
+            "pages_scanned": pages_scanned,
+            "has_more": has_more,
+            "sample": sample,
+        }
+
     # -- failure mapping (adapter overrides) -------------------------------
 
     def _failure_result(self, exc: Exception, _result) -> DownloadResult:

backend/app/services/interpreter_client.py

@@ -1,152 +0,0 @@
-"""Interpreter translation client (milestone 143) — a thin SYNC wrapper over the
-self-hosted Interpreter LAN service (LibreTranslate-compatible `/v1/translate`).
-
-Sync (requests) because the only caller is the sync celery translate sweep, and
-it mirrors FC's other platform clients. The service knows nothing about Curator —
-all Curator logic stays here. base_url is operator-configured (empty until set;
-behind a reverse proxy). Full API contract: Scribe note #1347.
-"""
-from __future__ import annotations
-
-from datetime import UTC, datetime
-from email.utils import parsedate_to_datetime
-
-import requests
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
-
-
-class InterpreterUnavailable(Exception):
-    """The translation engine is down / unreachable / draining — a connection
-    error, HTTP 429, or a 5xx (commonly 502/503/504 through a reverse proxy while
-    the service restarts). Retry later, don't drop the item. ``retry_after`` holds
-    the server's Retry-After hint in seconds when it sent one, so the caller can
-    back off exactly as long as it's asked to."""
-
-    def __init__(self, message: str, *, retry_after: float | None = None):
-        super().__init__(message)
-        self.retry_after = retry_after
-
-
-class InterpreterBadRequest(Exception):
-    """Bad request params (HTTP 400)."""
-
-
-def _url(base_url: str, path: str) -> str:
-    return f"{base_url.rstrip('/')}{path}"
-
-
-def _parse_retry_after(resp) -> float | None:
-    """Parse a Retry-After header (RFC 7231: delta-seconds or an HTTP-date) into
-    non-negative seconds, or None if absent/unparseable — so a gracefully-
-    draining Interpreter can tell Curator exactly how long to wait before it
-    tries again."""
-    raw = (resp.headers.get("Retry-After") or "").strip()
-    if not raw:
-        return None
-    try:
-        return max(0.0, float(int(raw)))       # delta-seconds form
-    except ValueError:
-        pass
-    try:                                        # HTTP-date form
-        when = parsedate_to_datetime(raw)
-    except (TypeError, ValueError):
-        return None
-    if when is None:
-        return None
-    if when.tzinfo is None:
-        when = when.replace(tzinfo=UTC)
-    return max(0.0, (when - datetime.now(UTC)).total_seconds())
-
-
-# A shared session pools the keep-alive connection across the per-post sweep
-# calls, and retries CONNECT failures only (connect=2, short backoff) — smoothing
-# the instant a reverse proxy reloads. Status codes are deliberately NOT retried
-# (status=0, raise_on_status=False): translate() maps 429/5xx → InterpreterUnavailable
-# itself, and letting urllib3 retry a draining 503 would defeat the Retry-After
-# backoff we honour upstream.
-_retry = Retry(
-    total=None, connect=2, read=0, redirect=0, status=0,
-    backoff_factor=0.3, raise_on_status=False,
-)
-session = requests.Session()
-_adapter = HTTPAdapter(max_retries=_retry)
-session.mount("http://", _adapter)
-session.mount("https://", _adapter)
-
-
-def health(base_url: str, *, timeout: float = 5.0) -> bool:
-    """True iff the Interpreter LLM engine is up. Any error (unset URL, network,
-    non-200, engine down) → False, so the sweep just no-ops rather than raising."""
-    if not base_url:
-        return False
-    try:
-        r = session.get(_url(base_url, "/v1/health"), timeout=timeout)
-    except requests.RequestException:
-        return False
-    if r.status_code != 200:
-        return False
-    engines = (r.json() or {}).get("engines") or {}
-    return bool(engines.get("llm"))
-
-
-def translate(
-    texts: list[str], *, base_url: str, target: str = "en",
-    source: str = "auto", timeout: float = 120.0,
-) -> dict:
-    """Translate a batch. Returns::
-
-        {"translations": [str, ...],        # SAME order & length as `texts`
-         "detected_lang": str | None,       # aggregate: describes the FIRST item
-         "detected_confidence": float | None,  # detector's confidence, if given
-         "engine": str | None,
-         "engine_version": str | None}
-
-    Interpreter batch metadata is aggregate (first item only) — fine for one
-    post's ``[title, description]`` batch since they share a language. Passthrough
-    items (already target-language / emoji-only) come back UNCHANGED in their
-    slot. Raises InterpreterUnavailable on a connection error / 429 / 5xx (retry
-    later, honouring Retry-After), InterpreterBadRequest on 400. (Scribe note
-    #1347.)
-    """
-    if not texts:
-        return {"translations": [], "detected_lang": None,
-                "detected_confidence": None,
-                "engine": None, "engine_version": None}
-    try:
-        r = session.post(
-            _url(base_url, "/v1/translate"),
-            json={"q": list(texts), "source": source,
-                  "target": target, "engine": "auto"},
-            timeout=timeout,
-        )
-    except requests.RequestException as e:
-        raise InterpreterUnavailable(str(e)) from e
-    if r.status_code == 400:
-        raise InterpreterBadRequest((r.json() or {}).get("error", "bad request"))
-    # A gracefully-draining service — often behind a reverse proxy — returns 429
-    # or a 5xx gateway error (502/503/504), not always a clean 503. Treat every
-    # one as "unavailable, retry later" and honour any Retry-After it sends, so a
-    # rolling Interpreter restart interrupts the sweep cleanly (instead of raising
-    # an opaque HTTPError) and resumes exactly when the service says it's ready.
-    if r.status_code == 429 or r.status_code >= 500:
-        raise InterpreterUnavailable(
-            f"interpreter unavailable (HTTP {r.status_code})",
-            retry_after=_parse_retry_after(r),
-        )
-    r.raise_for_status()
-    body = r.json() or {}
-    out = body.get("translatedText")
-    # q was an array → translatedText is an array (same order & length). Guard a
-    # scalar reply just in case (shouldn't happen for an array q).
-    if not isinstance(out, list):
-        out = [out]
-    interp = body.get("interpreter") or {}
-    detected = body.get("detectedLanguage") or {}
-    return {
-        "translations": out,
-        "detected_lang": detected.get("language"),
-        "detected_confidence": detected.get("confidence"),
-        "engine": interp.get("engine"),
-        "engine_version": interp.get("engine_version"),
-    }

backend/app/services/ml/__init__.py

@@ -1,3 +1 @@
-"""ML pipeline services: embedders, heads (the learning suggester), suggestions,
+"""ML pipeline services: tagger, embedder, suggestions, centroids, allowlist, aliases."""
-GPU-job queue + failure triage, CCIP characters, crops/regions, allowlist and
-aliases."""

backend/app/services/ml/allowlist.py

@@ -12,11 +12,13 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from ...models import TagSuggestionRejection
 from ...models.tag import image_tag
+from .aliases import AliasService
 
 
 class AllowlistService:
     def __init__(self, session: AsyncSession):
         self.session = session
+        self.aliases = AliasService(session)
 
     async def _apply_image_tag(self, image_id: int, tag_id: int, source: str):
         stmt = insert(image_tag).values(
@@ -39,6 +41,18 @@ class AllowlistService:
         await self._apply_image_tag(image_id, tag_id, source="ml_accepted")
         await self._clear_rejection(image_id, tag_id)
 
+    async def add_alias_and_accept(
+        self,
+        image_id: int,
+        alias_string: str,
+        alias_category: str,
+        canonical_tag_id: int,
+    ) -> None:
+        await self.aliases.create(
+            alias_string, alias_category, canonical_tag_id
+        )
+        await self.accept(image_id, canonical_tag_id)
+
     async def dismiss(self, image_id: int, tag_id: int) -> None:
         stmt = insert(TagSuggestionRejection).values(
             image_record_id=image_id, tag_id=tag_id

backend/app/services/ml/ccip.py

@@ -13,20 +13,11 @@ exact CCIP difference metric/threshold gets validated against the model during
 the hands-on eval. numpy is imported lazily (API worker has it via pgvector).
 """
 
-from sqlalchemy import exists, func, select
+from sqlalchemy import func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from ...models import (
+from ...models import ImageRegion, MLSettings, Tag, TagKind
-    CcipPrototypeState,
-    CharacterPrototype,
-    ImageRegion,
-    MLSettings,
-    Tag,
-    TagKind,
-    TagPositiveConfirmation,
-)
 from ...models.tag import image_tag
-from .training_data import _AUTO_SOURCES
 
 # Cosine-similarity floor to call a figure the same character. The live setting
 # (ml_settings.ccip_match_threshold) drives it; this is only the fallback when no
@@ -71,18 +62,6 @@ def _single_character_images():
     )
 
 
-def _hygiene_tagged_images():
-    """Subquery of image ids carrying any SYSTEM tag (wip / banner / editor
-    screenshot). Training hygiene (#128): such images never contribute
-    reference prototypes — a faceless wip's figure region would otherwise
-    become an identity reference for the character it's tagged with."""
-    return (
-        select(image_tag.c.image_record_id)
-        .join(Tag, Tag.id == image_tag.c.tag_id)
-        .where(Tag.is_system.is_(True))
-    )
-
-
 async def _ref_signature(session: AsyncSession) -> tuple:
     n_tags = (
         await session.execute(
@@ -100,28 +79,7 @@ async def _ref_signature(session: AsyncSession) -> tuple:
             )
         )
     ).one()
-    # Hygiene applications must invalidate too: tagging an image `wip` changes
+    return (n_tags, n_regs, max_id)
-    # the reference set without touching character-tag or region counts.
-    n_hygiene = (
-        await session.execute(
-            select(func.count())
-            .select_from(image_tag)
-            .join(Tag, Tag.id == image_tag.c.tag_id)
-            .where(Tag.is_system.is_(True))
-        )
-    ).scalar_one()
-    return (n_tags, n_regs, max_id, n_hygiene)
-
-
-def _positive_char_tag():
-    """Condition on the joined character image_tag: HUMAN-applied or operator-
-    confirmed — NOT an unconfirmed auto-apply. Keeps an auto-tagged character from
-    self-seeding CCIP references, so a ccip_auto misfire can't reinforce itself
-    (milestone 139) — mirrors the head-training positive exclusion."""
-    return image_tag.c.source.not_in(_AUTO_SOURCES) | exists().where(
-        TagPositiveConfirmation.image_record_id == image_tag.c.image_record_id,
-        TagPositiveConfirmation.tag_id == image_tag.c.tag_id,
-    )
 
 
 async def character_references(session: AsyncSession) -> dict[int, list]:
@@ -141,13 +99,9 @@ async def character_references(session: AsyncSession) -> dict[int, list]:
             )
             .join(Tag, Tag.id == image_tag.c.tag_id)
             .where(Tag.kind == TagKind.character)
-            .where(_positive_char_tag())
             .where(ImageRegion.kind.in_(_FIGURE_KINDS))
             .where(ImageRegion.ccip_embedding.is_not(None))
             .where(ImageRegion.image_record_id.in_(_single_character_images()))
-            .where(
-                ImageRegion.image_record_id.not_in(_hygiene_tagged_images())
-            )
         )
     ).all()
     refs: dict[int, list] = {}
@@ -169,60 +123,6 @@ async def _tag_names(session: AsyncSession, tag_ids: list[int]) -> dict[int, str
     )
 
 
-# Per-character normalized prototype matrices, cached per process and refreshed
-# INCREMENTALLY: only characters whose ccip_prototype_state.updated_at advanced
-# are reloaded. This replaces the request-path rebuild of the ENTIRE reference
-# blob (the ~4s stall, #1317) — the prototypes are precomputed off the request
-# path by services.ml.character_prototypes (a beat + after each retrain).
-_PROTO_CACHE: dict = {"mats": {}, "ver": {}}
-
-
-async def _load_prototypes(session: AsyncSession) -> dict:
-    """{tag_id: (P, D) L2-normalized prototype matrix} from character_prototype,
-    served from the in-process cache and reloading ONLY the characters whose
-    updated_at changed. Empty dict when the store isn't populated yet (cold start
-    → match_image falls back to the legacy on-the-fly reference build)."""
-    import numpy as np
-
-    versions = dict(
-        (
-            await session.execute(
-                select(CcipPrototypeState.tag_id, CcipPrototypeState.updated_at)
-            )
-        ).all()
-    )
-    mats = _PROTO_CACHE["mats"]
-    ver = _PROTO_CACHE["ver"]
-    # Forget characters that no longer have prototypes.
-    for tag_id in [t for t in mats if t not in versions]:
-        mats.pop(tag_id, None)
-        ver.pop(tag_id, None)
-    # Reload only the characters whose prototypes changed since we cached them.
-    stale = [t for t, u in versions.items() if ver.get(t) != u]
-    if stale:
-        rows = (
-            await session.execute(
-                select(
-                    CharacterPrototype.tag_id, CharacterPrototype.ccip_embedding
-                ).where(CharacterPrototype.tag_id.in_(stale))
-            )
-        ).all()
-        by_tag: dict[int, list] = {}
-        for tag_id, vec in rows:
-            by_tag.setdefault(tag_id, []).append(
-                np.asarray(vec, dtype=np.float32)
-            )
-        for tag_id in stale:
-            vecs = by_tag.get(tag_id)
-            if vecs:
-                mats[tag_id] = _l2norm(np.vstack(vecs), np)
-                ver[tag_id] = versions[tag_id]
-            else:
-                mats.pop(tag_id, None)
-                ver.pop(tag_id, None)
-    return mats
-
-
 async def match_image(
     session: AsyncSession, image_id: int, threshold: float | None = None
 ) -> list[dict]:
@@ -236,30 +136,18 @@ async def match_image(
     if threshold is None:
         threshold = await _settings_threshold(session)
 
-    # Keep each figure region's bbox alongside its vector so a match can point at
+    qvecs = (
-    # the figure that matched (#1206 grounding), not just the score.
-    fig_rows = (
         await session.execute(
-            select(
+            select(ImageRegion.ccip_embedding).where(
-                ImageRegion.ccip_embedding,
-                ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh,
-                ImageRegion.kind, ImageRegion.detector_version,
-            ).where(
                 ImageRegion.image_record_id == image_id,
                 ImageRegion.kind.in_(_FIGURE_KINDS),
                 ImageRegion.ccip_embedding.is_not(None),
             )
         )
-    ).all()
+    ).scalars().all()
-    if not fig_rows:
+    if not qvecs:
         return []
-    # Prefer the precomputed prototype store (fast, incremental). On a cold start
+    refs = await character_references(session)
-    # (store not yet populated post-deploy) fall back to the legacy on-the-fly
-    # reference build so character suggestions work immediately — the background
-    # refresh populates the store within ~15 min, after which this path is used
-    # and the per-accept ~4s rebuild is gone (#1317).
-    protos = await _load_prototypes(session)
-    refs = protos if protos else await character_references(session)
     if not refs:
         return []
     applied = set(
@@ -273,25 +161,13 @@ async def match_image(
     )
     names = await _tag_names(session, [t for t in refs if t not in applied])
 
-    qvecs = [r[0] for r in fig_rows]
-    fig_meta = [
-        {"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector}
-        for _v, rx, ry, rw, rh, kind, detector in fig_rows
-    ]
     Q = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in qvecs]), np)
     out = []
     for tag_id, vecs in refs.items():
         if tag_id in applied:
             continue
-        # Prototype matrices are already L2-normalized; legacy refs are raw
+        R = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np)
-        # vector lists that still need stacking + normalizing.
+        best = float((Q @ R.T).max())   # best (query figure, reference) cosine
-        R = vecs if protos else _l2norm(
-            np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np
-        )
-        sims = Q @ R.T                       # (n_query_figures, n_references)
-        per_figure = sims.max(axis=1)        # best reference cosine per figure
-        best_figure = int(per_figure.argmax())
-        best = float(per_figure[best_figure])
         if best >= threshold:
             out.append({
                 "tag_id": tag_id,
@@ -299,8 +175,6 @@ async def match_image(
                 "category": "character",
                 "score": round(best, 4),
                 "source": "ccip",
-                # the figure region that matched → grounds the character tag.
-                "grounding": fig_meta[best_figure],
             })
     out.sort(key=lambda d: d["score"], reverse=True)
     return out

backend/app/services/ml/character_prototypes.py

@@ -1,266 +0,0 @@
-"""Precomputed CCIP character prototypes — incremental builder (#1317, m138).
-
-Turns the per-character CCIP reference set into a precomputed artifact so the
-matcher never rebuilds it on the request path. Sync (runs in the celery ml
-worker via tasks.ml.refresh_character_prototypes); the async matcher only READS
-the character_prototype table.
-
-`refresh_character_prototypes`:
-  1. Cheap GLOBAL gate — a few COUNTs (`_global_signature`). Unchanged + not
-     `full` → no-op (nothing that affects references changed since last refresh).
-  2. Per-character fingerprint diff (one GROUP BY) → rebuild ONLY the characters
-     whose references changed (or are new); drop characters that lost all refs.
-Each rebuild loads just ONE character's reference vectors, caps them to
-MLSettings.ccip_prototype_cap, and replaces that character's prototype rows — so
-cost scales with WHAT changed, not with the library size.
-
-The reference PREDICATE (single-character, non-hygiene, figure CCIP) is imported
-from ccip so the prototypes match exactly what the legacy matcher selected.
-"""
-
-import random
-from datetime import UTC, datetime
-
-from sqlalchemy import delete, func, select
-from sqlalchemy.orm import Session
-
-from ...models import (
-    CcipPrototypeState,
-    CharacterPrototype,
-    ImageRegion,
-    MLSettings,
-    Tag,
-    TagKind,
-    TagPositiveConfirmation,
-)
-from ...models.tag import image_tag
-from .ccip import (
-    _FIGURE_KINDS,
-    _hygiene_tagged_images,
-    _l2norm,
-    _positive_char_tag,
-    _single_character_images,
-)
-
-# Deterministic per-tag capping so a rebuild of an UNCHANGED reference set
-# resamples identically (stable prototypes, no churn between refreshes).
-_SAMPLE_SEED = 1317
-
-
-def _global_signature(session: Session) -> str:
-    """Cheap 'could any references have changed' gate: character-tag count,
-    figure-CCIP region count + max id, hygiene-tag count. A few COUNTs — the same
-    quantities the legacy per-request signature used, now computed once per
-    refresh instead of on every /suggestions call."""
-    n_tags = session.execute(
-        select(func.count())
-        .select_from(image_tag)
-        .join(Tag, Tag.id == image_tag.c.tag_id)
-        .where(Tag.kind == TagKind.character)
-    ).scalar_one()
-    n_regs, max_id = session.execute(
-        select(func.count(), func.max(ImageRegion.id)).where(
-            ImageRegion.kind.in_(_FIGURE_KINDS),
-            ImageRegion.ccip_embedding.is_not(None),
-        )
-    ).one()
-    n_hygiene = session.execute(
-        select(func.count())
-        .select_from(image_tag)
-        .join(Tag, Tag.id == image_tag.c.tag_id)
-        .where(Tag.is_system.is_(True))
-    ).scalar_one()
-    # Character confirmations affect the reference set now that auto-tags only
-    # seed references once confirmed (milestone 139) — so a confirm must trip the
-    # gate, or the per-character diff (which reflects it) never runs.
-    n_conf = session.execute(
-        select(func.count())
-        .select_from(TagPositiveConfirmation)
-        .join(Tag, Tag.id == TagPositiveConfirmation.tag_id)
-        .where(Tag.kind == TagKind.character)
-    ).scalar_one()
-    return f"{n_tags}:{n_regs}:{max_id or 0}:{n_hygiene}:{n_conf}"
-
-
-def _current_fingerprints(session: Session) -> dict[int, str]:
-    """Per-character (reference count, max reference region id) over the SAME
-    predicate the matcher's references use. One GROUP BY → the change detector:
-    a character whose fingerprint moved (gained/lost a reference) needs a
-    rebuild; everyone else is left untouched."""
-    rows = session.execute(
-        select(
-            image_tag.c.tag_id,
-            func.count(ImageRegion.id),
-            func.max(ImageRegion.id),
-        )
-        .select_from(ImageRegion)
-        .join(
-            image_tag,
-            image_tag.c.image_record_id == ImageRegion.image_record_id,
-        )
-        .join(Tag, Tag.id == image_tag.c.tag_id)
-        .where(Tag.kind == TagKind.character)
-        .where(_positive_char_tag())
-        .where(ImageRegion.kind.in_(_FIGURE_KINDS))
-        .where(ImageRegion.ccip_embedding.is_not(None))
-        .where(ImageRegion.image_record_id.in_(_single_character_images()))
-        .where(ImageRegion.image_record_id.not_in(_hygiene_tagged_images()))
-        .group_by(image_tag.c.tag_id)
-    ).all()
-    return {tag_id: f"{cnt}:{mx}" for tag_id, cnt, mx in rows}
-
-
-def _rebuild_one(session: Session, tag_id: int, cap: int) -> int:
-    """Replace ONE character's prototype rows from its current references, capped
-    to `cap`. Loads only this character's vectors (bounded by its popularity)."""
-    rows = session.execute(
-        select(ImageRegion.id, ImageRegion.ccip_embedding)
-        .select_from(ImageRegion)
-        .join(
-            image_tag,
-            image_tag.c.image_record_id == ImageRegion.image_record_id,
-        )
-        .where(image_tag.c.tag_id == tag_id)
-        .where(_positive_char_tag())
-        .where(ImageRegion.kind.in_(_FIGURE_KINDS))
-        .where(ImageRegion.ccip_embedding.is_not(None))
-        .where(ImageRegion.image_record_id.in_(_single_character_images()))
-        .where(ImageRegion.image_record_id.not_in(_hygiene_tagged_images()))
-    ).all()
-    # Cap for bounded MATCH cost. A random sample (not most-recent) keeps the
-    # prototypes representative of the whole reference set; a fixed per-tag seed
-    # makes an unchanged set resample identically.
-    if cap > 0 and len(rows) > cap:
-        rows = random.Random(f"{_SAMPLE_SEED}:{tag_id}").sample(rows, cap)
-    session.execute(
-        delete(CharacterPrototype).where(CharacterPrototype.tag_id == tag_id)
-    )
-    for region_id, vec in rows:
-        session.add(
-            CharacterPrototype(
-                tag_id=tag_id, region_id=region_id, ccip_embedding=vec
-            )
-        )
-    return len(rows)
-
-
-def refresh_character_prototypes(
-    session: Session, *, full: bool = False
-) -> dict[str, int | bool]:
-    """Incrementally refresh the prototype store. `full=True` rebuilds every
-    character regardless of the gate/fingerprints (nightly reconcile). Returns
-    {skipped, rebuilt, removed}; commits."""
-    settings = session.execute(
-        select(MLSettings).where(MLSettings.id == 1)
-    ).scalar_one()
-    sig = _global_signature(session)
-    if not full and settings.ccip_ref_signature == sig:
-        return {"skipped": True, "rebuilt": 0, "removed": 0}
-
-    cap = settings.ccip_prototype_cap
-    current = _current_fingerprints(session)
-    stored = dict(
-        session.execute(
-            select(CcipPrototypeState.tag_id, CcipPrototypeState.fingerprint)
-        ).all()
-    )
-    now = datetime.now(UTC)
-    rebuilt = 0
-    for tag_id, fp in current.items():
-        if full or stored.get(tag_id) != fp:
-            _rebuild_one(session, tag_id, cap)
-            state = session.get(CcipPrototypeState, tag_id)
-            if state is None:
-                state = CcipPrototypeState(tag_id=tag_id)
-                session.add(state)
-            state.fingerprint = fp
-            state.updated_at = now
-            rebuilt += 1
-    # Characters that lost every reference (refs removed / re-kinded / image now
-    # multi-character) → drop their prototypes + state so they stop matching.
-    removed = 0
-    for tag_id in set(stored) - set(current):
-        session.execute(
-            delete(CharacterPrototype).where(CharacterPrototype.tag_id == tag_id)
-        )
-        session.execute(
-            delete(CcipPrototypeState).where(CcipPrototypeState.tag_id == tag_id)
-        )
-        removed += 1
-
-    settings.ccip_ref_signature = sig
-    session.commit()
-    return {"skipped": False, "rebuilt": rebuilt, "removed": removed}
-
-
-def retract_auto_applied_ccip(session: Session) -> int:
-    """Soft auto-apply for CCIP character tags (milestone 139): re-score every
-    standing source='ccip_auto' character tag against that character's prototypes
-    and REMOVE the ones whose best figure match is now BELOW
-    ccip_auto_apply_threshold. Skips operator-confirmed tags. SILENT — a low score
-    isn't proof the tag was wrong (that's reserved for an operator removal). No-op
-    unless ccip_auto_apply_enabled. A character with no prototypes yet, or an image
-    with no figure vectors, is left alone (can't judge → keep). Returns
-    n_retracted."""
-    import numpy as np
-
-    settings = session.execute(
-        select(MLSettings).where(MLSettings.id == 1)
-    ).scalar_one()
-    if not settings.ccip_auto_apply_enabled:
-        return 0
-    thr = float(settings.ccip_auto_apply_threshold)
-    pairs = session.execute(
-        select(image_tag.c.image_record_id, image_tag.c.tag_id)
-        .where(image_tag.c.source == "ccip_auto")
-    ).all()
-    if not pairs:
-        return 0
-    confirmed = {
-        (iid, tid) for iid, tid in session.execute(
-            select(
-                TagPositiveConfirmation.image_record_id,
-                TagPositiveConfirmation.tag_id,
-            )
-        ).all()
-    }
-    # Each involved character's normalized prototype matrix, loaded once.
-    proto: dict[int, object] = {}
-    for tid in {tid for _iid, tid in pairs}:
-        vecs = [
-            v for (v,) in session.execute(
-                select(CharacterPrototype.ccip_embedding)
-                .where(CharacterPrototype.tag_id == tid)
-            )
-        ]
-        if vecs:
-            proto[tid] = _l2norm(
-                np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np
-            )
-    retracted = 0
-    for iid, tid in pairs:
-        if (iid, tid) in confirmed or tid not in proto:
-            continue                                   # confirmed / no prototypes
-        qvecs = [
-            v for (v,) in session.execute(
-                select(ImageRegion.ccip_embedding)
-                .where(ImageRegion.image_record_id == iid)
-                .where(ImageRegion.kind.in_(_FIGURE_KINDS))
-                .where(ImageRegion.ccip_embedding.is_not(None))
-            )
-        ]
-        if not qvecs:
-            continue                                   # no figure vectors → keep
-        Q = _l2norm(
-            np.vstack([np.asarray(v, dtype=np.float32) for v in qvecs]), np
-        )
-        if float((Q @ proto[tid].T).max()) < thr:
-            session.execute(
-                image_tag.delete()
-                .where(image_tag.c.image_record_id == iid)
-                .where(image_tag.c.tag_id == tid)
-                .where(image_tag.c.source == "ccip_auto")
-            )
-            retracted += 1
-    session.commit()
-    return retracted

backend/app/services/ml/gpu_jobs.py

@@ -12,9 +12,8 @@ and the lease itself reclaims expired leases as a final backstop. Result-writing
 
 from datetime import UTC, datetime, timedelta
 
-from sqlalchemy import and_, delete, exists, func, or_, select, update
+from sqlalchemy import and_, or_, select, update
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import aliased
 
 from ...models import GpuJob
 
@@ -25,107 +24,6 @@ DEFAULT_LEASE_TTL = 180     # seconds an agent holds a job before it can be re-l
 DEFAULT_BATCH = 8
 MAX_ATTEMPTS = 3
 
-# Poison-loop backstops. `attempts` counts LEASES GRANTED (incremented in
-# lease()), but fail()'s MAX_ATTEMPTS cap only fires when the agent reports a
-# failure — a job that keeps coming back via release() (transient handback) or
-# lease expiry (agent crash/wedge) never gets a verdict and would cycle forever.
-# The orphan sweep converts those to 'error': an expired lease that has already
-# been granted EXPIRED_POISON_CAP leases is presumed to kill/wedge the agent,
-# and a pending job granted PENDING_POISON_CAP leases without ever completing is
-# presumed poisoned (e.g. a transfer that stalls every time). Both stay
-# resurrectable via /retry_errors, which resets attempts.
-EXPIRED_POISON_CAP = MAX_ATTEMPTS + 2
-PENDING_POISON_CAP = 10
-
-
-def error_dedupe_statements():
-    """DELETEs enforcing: at most ONE error row per (image, task), and none that
-    a live or succeeded row makes moot. The 2026-07-02 tombstone loop (backfill
-    skip-lists lacked 'error') minted a duplicate error row per bad file per
-    hour; running these before every backfill and inside /retry_errors keeps the
-    error count reading as "distinct failing files" and stops a retry fanning
-    one file out into several duplicate pending jobs. Shared by the sync beat
-    task and the async API route so both prune by the SAME predicate.
-    Execution order matters: moot rows first, then older duplicates (the newest
-    error — the freshest reason — survives)."""
-    other = aliased(GpuJob)
-    same_pair = and_(
-        other.image_record_id == GpuJob.image_record_id,
-        other.task == GpuJob.task,
-    )
-    moot = (
-        delete(GpuJob)
-        .where(
-            GpuJob.status == "error",
-            exists().where(
-                same_pair, other.status.in_(["pending", "leased", "done"]),
-            ),
-        )
-        .execution_options(synchronize_session=False)
-    )
-    older_dupe = (
-        delete(GpuJob)
-        .where(
-            GpuJob.status == "error",
-            exists().where(
-                same_pair,
-                other.status == "error",
-                or_(
-                    other.updated_at > GpuJob.updated_at,
-                    and_(other.updated_at == GpuJob.updated_at,
-                         other.id > GpuJob.id),
-                ),
-            ),
-        )
-        .execution_options(synchronize_session=False)
-    )
-    return [moot, older_dupe]
-
-
-def recover_statements(now: datetime) -> dict:
-    """UPDATEs for the orphan sweep, keyed by outcome; insertion order IS the
-    required execution order ('recovered' must run after 'poison_expired', which
-    claims the crash-loopers out of the same expired-lease pool)."""
-    expired = and_(GpuJob.status == "leased", GpuJob.lease_expires_at < now)
-    unlease = {"lease_token": None, "leased_at": None, "lease_expires_at": None,
-               "updated_at": now}
-    return {
-        "poison_expired": (
-            update(GpuJob)
-            .where(expired, GpuJob.attempts >= EXPIRED_POISON_CAP)
-            .values(
-                status="error",
-                # Keep the job's last stored failure reason — it's the triage
-                # signal for WHY the loop happened.
-                error=func.concat(
-                    f"poisoned: lease expired after {EXPIRED_POISON_CAP}+ lease "
-                    "attempts (job repeatedly crashes or wedges the agent?); "
-                    "last error: ",
-                    func.coalesce(GpuJob.error, "none"),
-                ),
-                **unlease,
-            )
-        ),
-        "recovered": update(GpuJob).where(expired).values(
-            status="pending", **unlease,
-        ),
-        "poison_pending": (
-            update(GpuJob)
-            .where(GpuJob.status == "pending",
-                   GpuJob.attempts >= PENDING_POISON_CAP)
-            .values(
-                status="error",
-                error=func.concat(
-                    f"poisoned: {PENDING_POISON_CAP}+ lease attempts without "
-                    "ever completing (transfer stalls every time?); "
-                    "last error: ",
-                    func.coalesce(GpuJob.error, "none"),
-                ),
-                updated_at=now,
-            )
-        ),
-    }
-
 
 class GpuJobService:
     def __init__(self, session: AsyncSession):
@@ -153,33 +51,25 @@ class GpuJobService:
     async def lease(
         self, token: str, batch_size: int = DEFAULT_BATCH, ttl: int = DEFAULT_LEASE_TTL
     ) -> list[GpuJob]:
-        """Claim up to batch_size pending (or expired-leased) jobs for `token`.
+        """Claim up to batch_size pending (or expired-leased) jobs for `token`."""
-
-        Two phases so each hits a partial index (0070) and stays O(batch) no
-        matter how many done/error rows have accumulated: the pending pool is the
-        hot path; expired leases are reclaimed only when pending can't fill the
-        batch (a crashed agent's work — rare). The old single OR-query walked the
-        primary key past the whole done-prefix in id order → O(done), which is
-        why leasing crawled — and the DB saturated — as the run progressed."""
         now = datetime.now(UTC)
-
+        picked = (
-        async def _claim(condition, limit: int) -> list[int]:
+            await self.session.execute(
-            return list(
+                select(GpuJob.id)
-                (
+                .where(
-                    await self.session.execute(
+                    or_(
-                        select(GpuJob.id).where(condition)
+                        GpuJob.status == "pending",
-                        .order_by(GpuJob.id).limit(limit)
+                        and_(
-                        .with_for_update(skip_locked=True)
+                            GpuJob.status == "leased",
+                            GpuJob.lease_expires_at < now,
+                        ),
                     )
-                ).scalars().all()
+                )
-            )
+                .order_by(GpuJob.id)
-
+                .limit(batch_size)
-        picked = await _claim(GpuJob.status == "pending", batch_size)
+                .with_for_update(skip_locked=True)
-        if len(picked) < batch_size:     # pending exhausted → reclaim expired leases
-            picked += await _claim(
-                and_(GpuJob.status == "leased", GpuJob.lease_expires_at < now),
-                batch_size - len(picked),
             )
+        ).scalars().all()
         if not picked:
             return []
         await self.session.execute(
@@ -272,11 +162,16 @@ class GpuJobService:
 
     async def recover_orphaned(self) -> int:
         """Reset every expired lease back to pending — catches agents that died
-        mid-job (no graceful release) — and convert poison-loopers to 'error'
+        mid-job (no graceful release). Run on a short beat so the queue recovers
-        (see the *_POISON_CAP rationale above). Run on a short beat so the queue
+        + reads honestly even when no worker is actively leasing. Returns rows
-        recovers + reads honestly even when no worker is actively leasing.
+        recovered."""
-        Returns rows recovered to pending (poison conversions are extra)."""
+        now = datetime.now(UTC)
-        counts = {}
+        res = await self.session.execute(
-        for name, stmt in recover_statements(datetime.now(UTC)).items():
+            update(GpuJob)
-            counts[name] = (await self.session.execute(stmt)).rowcount or 0
+            .where(GpuJob.status == "leased", GpuJob.lease_expires_at < now)
-        return counts["recovered"]
+            .values(
+                status="pending", lease_token=None, leased_at=None,
+                lease_expires_at=None, updated_at=now,
+            )
+        )
+        return res.rowcount or 0

backend/app/services/ml/gpu_triage.py

@@ -1,179 +0,0 @@
-"""GPU-failure triage (#125): classify errored jobs, PROBE the file, recover.
-
-An errored GPU job is a tombstone with a stored reason, but the reason alone is
-a suspicion, not a verdict — a timeout can hit a perfectly fine file, and
-"moov atom not found" can mean a truncated download OR a one-off transfer
-fault. So triage EVALUATES: it runs the real integrity probe (sha256 recompute
-+ PIL/ffprobe — verify_integrity's own machinery) on each errored image ONCE
-and records both verdicts:
-
-  ImageRecord.integrity_status  <- file-level verdict (ok / corrupt / ...)
-  GpuJob.triage_status          <- 'defect'  (file is bad: recovery material,
-                                              excluded from /retry_errors)
-                                   'file_ok' (file passes: the failure was
-                                              operational, safe to retry)
-
-Recovery reuses established primitives: delete the defective copy + record
-(cleanup_service.delete_images — full cascade) and re-poll the image's
-subscription Source (the Layer-2 refetch pattern: gallery-dl re-fetches the
-now-absent file on the next source check). Images without a pollable Source
-report 'no_source' — manual remediation. Every classification is logged at
-WARNING so the operator notices in Logs / System Activity.
-"""
-import logging
-import time
-from datetime import UTC, datetime
-from pathlib import Path
-
-from sqlalchemy import select, update
-from sqlalchemy.orm import Session
-
-from ...models import GpuJob, ImageProvenance, ImageRecord, Source
-from ..cleanup_service import delete_images
-
-log = logging.getLogger(__name__)
-
-# Reason buckets for the triage overview (reporting only — the PROBE decides
-# 'defect', never the string). Ordered: first match wins.
-_REASON_BUCKETS = (
-    ("poisoned", ("poisoned:",)),
-    ("transient", ("gave up after repeated transient", "curator unreachable",
-                   "connection", "read timed out")),
-    ("timeout", ("timed out", "timeout")),
-    ("truncated_or_corrupt", ("moov atom", "invalid data", "end of file",
-                              "header missing", "error reading header",
-                              "truncated", "premature", "corrupt",
-                              "no frames sampled")),
-    ("decode", ("cannot identify", "decompression", "broken data stream",
-                "unrecognized data")),
-)
-
-
-def classify_reason(error: str | None) -> str:
-    """Bucket a stored job-error string for the overview table."""
-    text = (error or "").lower()
-    if not text:
-        return "other"
-    for bucket, needles in _REASON_BUCKETS:
-        if any(n in text for n in needles):
-            return bucket
-    return "other"
-
-
-def triage_errored_jobs(
-    session: Session, *, time_budget_seconds: float = 300.0,
-) -> dict:
-    """Probe every not-yet-triaged errored image and write both verdicts.
-
-    Time-boxed (sha256 of a large original over NFS can take tens of seconds)
-    and inherently resumable: rows are selected by `triage_status IS NULL`, so
-    the next sweep continues exactly where a budget cut stopped. Commits per
-    image so a mid-run crash keeps completed verdicts."""
-    image_ids = session.execute(
-        select(GpuJob.image_record_id)
-        .where(GpuJob.status == "error", GpuJob.triage_status.is_(None))
-        .group_by(GpuJob.image_record_id)
-        .order_by(GpuJob.image_record_id)
-    ).scalars().all()
-    counts = {"probed": 0, "defect": 0, "file_ok": 0, "partial": False}
-    if not image_ids:
-        return counts
-    # Lazy imports: the probe helper lives in the maintenance task module and
-    # the hasher in the importer — importing either at module load would pull
-    # celery into every service consumer.
-    from ...tasks.maintenance import _verify_one
-    from ..importer import _sha256_of
-
-    started = time.monotonic()
-    for image_id in image_ids:
-        if time.monotonic() - started > time_budget_seconds:
-            counts["partial"] = True
-            break
-        rec = session.get(ImageRecord, image_id)
-        if rec is None:      # record deleted since the job errored
-            continue
-        verdict = _verify_one(Path(rec.path), rec.sha256, rec.mime, _sha256_of)
-        # 'ok' means the failure was operational; anything else (corrupt /
-        # failed_verification = missing/unreadable) makes the file itself the
-        # problem — recovery material.
-        triage = "file_ok" if verdict == "ok" else "defect"
-        reason = session.execute(
-            select(GpuJob.error)
-            .where(GpuJob.image_record_id == image_id, GpuJob.status == "error")
-            .limit(1)
-        ).scalar_one_or_none()
-        rec.integrity_status = verdict
-        session.execute(
-            update(GpuJob)
-            .where(GpuJob.image_record_id == image_id, GpuJob.status == "error")
-            .values(triage_status=triage, updated_at=datetime.now(UTC))
-        )
-        session.commit()
-        counts["probed"] += 1
-        counts[triage] += 1
-        log.warning(
-            "gpu triage: image %s (%s) job error %r -> integrity probe %r -> %s",
-            image_id, rec.path, (reason or "")[:120], verdict, triage,
-        )
-    return counts
-
-
-def recover_defective_image(
-    session: Session, image_id: int, *, images_root: Path,
-) -> dict:
-    """Delete the defective copy + record and queue its re-fetch — surgically
-    where possible.
-
-    Two re-fetch layers (operator 2026-07-03: deep back-catalogue items are
-    NEVER re-walked by the normal cadence, so recovery can't rely on it):
-    1. SURGICAL: any ExternalLink rows on the image's post(s) are reset +
-       re-dispatched — this is how external-host files (the common defect
-       case: big videos) come back regardless of post age. Sha-dedupe at
-       import discards payload files that still exist.
-    2. BROAD: a source re-check, which re-fetches gallery-dl-NATIVE files the
-       walk still reaches (recent posts). A native file deeper than the walk
-       needs a per-source backfill/deep scan — reported via links_reset=0 so
-       the caller can say so.
-
-    The record delete cascades the error tombstones with it. 'no_source' when
-    no enabled, real-URL Source resolves via provenance — manual there."""
-    rec = session.get(ImageRecord, image_id)
-    if rec is None:
-        return {"status": "not_found"}
-    src_id = session.execute(
-        select(Source.id)
-        .join(ImageProvenance, ImageProvenance.source_id == Source.id)
-        .where(
-            ImageProvenance.image_record_id == image_id,
-            Source.enabled.is_(True),
-            ~Source.url.like("sidecar:%"),   # synthetic anchor — not pollable
-        )
-        .order_by(Source.id.asc())
-    ).scalars().first()
-    if src_id is None:
-        return {"status": "no_source"}
-    # Capture the post linkage BEFORE the delete cascades provenance away.
-    post_ids = session.execute(
-        select(ImageProvenance.post_id)
-        .where(ImageProvenance.image_record_id == image_id)
-    ).scalars().all()
-    path = rec.path
-    summary = delete_images(session, image_ids=[image_id], images_root=images_root)
-    from ..external_links import refetch_links_for_post
-
-    links_reset = 0
-    for pid in post_ids:
-        links_reset += refetch_links_for_post(session, pid)["links_reset"]
-    # Lazy import (services -> tasks would cycle at module load).
-    from ...tasks.download import download_source
-
-    download_source.delay(src_id)
-    log.warning(
-        "gpu triage recovery: deleted defective image %s (%s); reset %d "
-        "external link(s) and queued a re-check of source %s",
-        image_id, path, links_reset, src_id,
-    )
-    return {
-        "status": "refetch_queued", "source_id": src_id,
-        "links_reset": links_reset, **summary,
-    }

backend/app/services/ml/heads.py

@@ -1,13 +1,12 @@
 """Production heads: train + score the per-concept classifiers (#114).
 
-The eval harness (#1130) proved the spine, then retired 2026-07-02 once the
+The eval (#1130, tag_eval.py) proved the spine; this is its production form.
-tagging system was accepted; this is the production form.
 - TRAIN (sync, ml worker — needs scikit-learn): for every general/character tag
   with enough labelled positives, fit a logistic-regression head on the FROZEN
   SigLIP embeddings (positives + negatives = rejections + sampled unlabeled),
   derive an honest suggest threshold + earned-auto-apply point from CROSS-
-  VALIDATED scores, and upsert a TagHead row. Uses the eval-proven data loaders
+  VALIDATED scores, and upsert a TagHead row. Reuses tag_eval's proven data
-  + metric helpers (training_data.py) so heads match the measured numbers.
+  loaders + metric helpers so production heads match the eval's measured numbers.
 - SCORE (async, API worker — numpy via pgvector, NO scikit-learn): score one
   image's embedding against all current heads → the suggestions the rail shows,
   REPLACING Camie predictions + per-tag centroids.
@@ -22,7 +21,7 @@ import logging
 from datetime import UTC, datetime
 from typing import Any
 
-from sqlalchemy import delete, exists, func, select
+from sqlalchemy import delete, func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import Session
 
@@ -32,18 +31,14 @@ from ...models import (
     ImageRecord,
     ImageRegion,
     MLSettings,
-    PresentationReview,
     Tag,
     TagHead,
     TagKind,
-    TagPositiveConfirmation,
     TagSuggestionRejection,
 )
-from ...models.tag import CHROME_SYSTEM_TAGS, PROCESS_SYSTEM_TAGS, image_tag
+from ...models.tag import image_tag
-from .training_data import (
+from .tag_eval import (
-    _AUTO_SOURCES,
     _auto_apply_point,
-    _hygiene_excluded_ids,
     _ids_with_tag,
     _l2norm,
     _load_embeddings,
@@ -66,17 +61,6 @@ _HEAD_KINDS = (TagKind.general, TagKind.character)
 # tag.kind -> the suggestion category the rail groups under.
 _CATEGORY = {TagKind.general: "general", TagKind.character: "character"}
 
-# System-tag (wip/banner/editor screenshot) heads surface as suggestions at
-# this FLAT confidence floor instead of their auto-derived (precision-tuned)
-# suggest threshold. The auto threshold is high, so it hides the borderline /
-# false-positive guesses — which are exactly the cases the operator wants to
-# SEE and REJECT to sharpen these heads (hard-negative mining: "negatively
-# reinforce what isn't a system tag"). Operator-set 0.65 (2026-07-03): high
-# enough not to spam near-zero scores, low enough to surface real mistakes.
-# Content-tag heads keep their own thresholds; the typed-dropdown's
-# threshold_override still overrides everything (show-all mode).
-_SYSTEM_TAG_SUGGEST_FLOOR = 0.65
-
 
 class HeadTrainingAlreadyRunning(Exception):
     """Raised by start_head_training_run when a run is already in flight."""
@@ -140,148 +124,42 @@ def _embedder_version(session: Session) -> str:
 
 
 def _eligible_tag_ids(session: Session, min_pos: int) -> list[int]:
-    """Concept tags (general/character) with >= min_pos POSITIVE images — the set
+    """Concept tags (general/character) with >= min_pos labelled images — the
-    that gets a head. Counts human-applied + operator-confirmed tags only;
+    set that gets a head. Counts all sources; source-aware filtering (#1133) is
-    unconfirmed auto-applied predictions do NOT count toward eligibility (they
+    a separate, optional refinement."""
-    don't train the head — milestone 139), so a concept can't graduate on its own
-    guesses."""
-    confirmed = exists().where(
-        TagPositiveConfirmation.image_record_id == image_tag.c.image_record_id,
-        TagPositiveConfirmation.tag_id == image_tag.c.tag_id,
-    )
     rows = session.execute(
         select(Tag.id)
         .join(image_tag, image_tag.c.tag_id == Tag.id)
         .where(Tag.kind.in_(_HEAD_KINDS))
-        .where(image_tag.c.source.not_in(_AUTO_SOURCES) | confirmed)
         .group_by(Tag.id)
         .having(func.count(image_tag.c.image_record_id) >= min_pos)
     ).all()
     return [r[0] for r in rows]
 
 
-def _head_fingerprints(session: Session, tag_ids: list[int]) -> dict[int, str]:
-    """Per-tag training-data fingerprint: (positive count, latest positive
-    created_at) + (rejection count, latest rejected_at). It moves whenever a tag
-    gains/loses a positive or a rejection — the incremental-retrain change
-    detector (#1317 p2). A newly-added positive/rejection always has the latest
-    timestamp, so even a remove-one-add-one (unchanged count) is caught. The
-    sampled-unlabeled negative pool + the hygiene set drift GLOBALLY and are
-    reconciled by the nightly full run, not captured here."""
-    if not tag_ids:
-        return {}
-    pos = session.execute(
-        select(
-            image_tag.c.tag_id,
-            func.count(image_tag.c.image_record_id),
-            func.max(image_tag.c.created_at),
-        )
-        .where(image_tag.c.tag_id.in_(tag_ids))
-        .group_by(image_tag.c.tag_id)
-    ).all()
-    pos_map = {t: (c, m) for t, c, m in pos}
-    rej = session.execute(
-        select(
-            TagSuggestionRejection.tag_id,
-            func.count(),
-            func.max(TagSuggestionRejection.rejected_at),
-        )
-        .where(TagSuggestionRejection.tag_id.in_(tag_ids))
-        .group_by(TagSuggestionRejection.tag_id)
-    ).all()
-    rej_map = {t: (c, m) for t, c, m in rej}
-    # Confirmations promote an auto-applied tag to a positive (milestone 139), so
-    # a confirm must move the fingerprint too — else a manual Retrain right after
-    # confirming wouldn't fold the tag in (the nightly full run would).
-    conf = session.execute(
-        select(TagPositiveConfirmation.tag_id, func.count())
-        .where(TagPositiveConfirmation.tag_id.in_(tag_ids))
-        .group_by(TagPositiveConfirmation.tag_id)
-    ).all()
-    conf_map = dict(conf)
-    out = {}
-    for t in tag_ids:
-        pc, pm = pos_map.get(t, (0, None))
-        rc, rm = rej_map.get(t, (0, None))
-        out[t] = f"{pc}:{pm}:{rc}:{rm}:{conf_map.get(t, 0)}"
-    return out
-
-
-def _heads_needing_retrain(
-    session: Session, eligible: list[int], embedding_version: str,
-    fps: dict[int, str], full: bool,
-) -> list[int]:
-    """The eligible tag_ids to (re)fit: no head yet, a head trained in a DIFFERENT
-    embedding space (a model swap), or a changed training-data fingerprint.
-    full=True forces every eligible tag. sklearn-free (train_head itself needs
-    scikit-learn) so the incremental decision is unit-testable on its own."""
-    if full:
-        return list(eligible)
-    existing = {
-        tag_id: (fp, ev)
-        for tag_id, fp, ev in session.execute(
-            select(
-                TagHead.tag_id, TagHead.train_fingerprint,
-                TagHead.embedding_version,
-            )
-        ).all()
-    }
-    out = []
-    for tag_id in eligible:
-        prev = existing.get(tag_id)
-        if (
-            prev is None
-            or prev[1] != embedding_version
-            or prev[0] != fps.get(tag_id)
-        ):
-            out.append(tag_id)
-    return out
-
-
 def train_all_heads(
     session: Session, params: dict[str, Any], run: HeadTrainingRun | None = None
 ) -> dict[str, int]:
-    """(Re)train eligible concept heads, INCREMENTALLY by default (#1317 p2):
+    """(Re)train a head for every eligible concept; prune heads whose tag is no
-    refit only the tags whose training data changed since last fit, so a manual
+    longer eligible. Commits per head so a SIGKILL leaves trained heads durable
-    Retrain click is fast. `params["full"]=True` (the nightly run) refits every
+    (training is idempotent). Returns {n_trained, n_skipped}."""
-    head to reconcile sampled-negative + hygiene drift. Prunes heads whose tag is
-    no longer eligible. Commits per head so a SIGKILL leaves trained heads durable.
-    Returns {n_trained, n_skipped} (n_skipped = unchanged + too-few-examples)."""
     import numpy as np
 
     cfg = _normalize_params(session, params)
     embedding_version = _embedder_version(session)
-    full = bool((params or {}).get("full"))
     eligible = _eligible_tag_ids(session, cfg["min_positives"])
     eligible_set = set(eligible)
-    # Computed once per run, not per head — the hygiene set is identical for
-    # every non-system concept.
-    hygiene = _hygiene_excluded_ids(session)
-    fps = _head_fingerprints(session, eligible)
-    to_train = set(
-        _heads_needing_retrain(session, eligible, embedding_version, fps, full)
-    )
     trained = 0
-    failed = 0
+    skipped = 0
     for i, tag_id in enumerate(eligible):
-        if tag_id not in to_train:
-            continue
         try:
-            ok = train_head(
+            ok = train_head(session, tag_id, embedding_version, cfg, np)
-                session, tag_id, embedding_version, cfg, np, hygiene=hygiene
-            )
         except Exception:
             log.exception("train_head failed for tag %d", tag_id)
             ok = False
-        if ok:
-            # Stamp the fingerprint we trained against so an unchanged tag is
-            # skipped on the next incremental run.
-            head = session.get(TagHead, tag_id)
-            if head is not None:
-                head.train_fingerprint = fps.get(tag_id)
         session.commit()
         trained += int(ok)
-        failed += int(not ok)
+        skipped += int(not ok)
         if run is not None and i % 10 == 0:
             run.last_progress_at = datetime.now(UTC)
             session.commit()
@@ -292,63 +170,30 @@ def train_all_heads(
     else:
         session.execute(delete(TagHead))
     session.commit()
-    # n_skipped = unchanged (not attempted) + failed-to-fit (too few examples).
+    return {"n_trained": trained, "n_skipped": skipped}
-    return {
-        "n_trained": trained,
-        "n_skipped": (len(eligible) - len(to_train)) + failed,
-    }
-
-
-def head_training_ids(
-    session: Session, tag_id: int, cfg: dict, hygiene: set[int] | None = None,
-) -> tuple[list[int], list[int]] | None:
-    """Select (pos_ids, neg_ids) for one head. Split out of train_head and
-    kept sklearn-free so the hygiene exclusion is testable in the CI env
-    (sklearn only exists in the ml image). Returns None when the concept has
-    too few usable positives.
-
-    Training hygiene (#128): images carrying a system tag are ABSENT from
-    every other concept's training — dropped as positives AND kept out of
-    the rejection/sampled negative pool (see _hygiene_excluded_ids). A system
-    tag's own head trains on them unfiltered: its positives ARE the hygiene
-    images."""
-    tag = session.get(Tag, tag_id)
-    if tag is not None and tag.is_system:
-        hygiene = set()
-    elif hygiene is None:
-        hygiene = _hygiene_excluded_ids(session)
-
-    pos_ids = [i for i in _ids_with_tag(session, tag_id) if i not in hygiene]
-    if len(pos_ids) < cfg["min_positives"]:
-        return None
-
-    pos_set = set(pos_ids)
-    rejected = [
-        i for i in _rejected_ids(session, tag_id)
-        if i not in pos_set and i not in hygiene
-    ]
-    want_neg = max(len(pos_ids) * cfg["neg_ratio"], _EXAMPLES_MIN * 4)
-    sampled = _sample_unlabeled(
-        session, pos_set | set(rejected) | hygiene,
-        min(_UNLABELED_POOL, want_neg),
-    )
-    return pos_ids, rejected + [i for i in sampled if i not in pos_set]
 
 
 def train_head(
-    session: Session, tag_id: int, embedding_version: str, cfg: dict, np,
+    session: Session, tag_id: int, embedding_version: str, cfg: dict, np
-    hygiene: set[int] | None = None,
 ) -> bool:
     """Fit + upsert one head. Returns True if a head was written, False if the
     concept had too few usable examples to train (the row is then removed)."""
     from sklearn.linear_model import LogisticRegression
     from sklearn.model_selection import StratifiedKFold, cross_val_predict
 
-    ids = head_training_ids(session, tag_id, cfg, hygiene)
+    pos_ids = _ids_with_tag(session, tag_id)
-    if ids is None:
+    if len(pos_ids) < cfg["min_positives"]:
         session.execute(delete(TagHead).where(TagHead.tag_id == tag_id))
         return False
-    pos_ids, neg_ids = ids
+
+    pos_set = set(pos_ids)
+    rejected = [i for i in _rejected_ids(session, tag_id) if i not in pos_set]
+    want_neg = max(len(pos_ids) * cfg["neg_ratio"], _EXAMPLES_MIN * 4)
+    sampled = _sample_unlabeled(
+        session, pos_set | set(rejected), min(_UNLABELED_POOL, want_neg)
+    )
+    neg_ids = rejected + [i for i in sampled if i not in pos_set]
+
     emb = _load_embeddings(session, pos_ids + neg_ids)
     pos = [emb[i] for i in pos_ids if i in emb]
     neg = [emb[i] for i in neg_ids if i in emb]
@@ -416,7 +261,7 @@ async def _current_heads(session: AsyncSession, embedding_version: str):
     rows = (
         await session.execute(
             select(
-                TagHead.tag_id, Tag.name, Tag.kind, Tag.is_system,
+                TagHead.tag_id, Tag.name, Tag.kind,
                 TagHead.weights, TagHead.bias,
                 TagHead.suggest_threshold, TagHead.auto_apply_threshold,
             )
@@ -434,12 +279,8 @@ async def _current_heads(session: AsyncSession, embedding_version: str):
             {
                 "tag_id": r.tag_id,
                 "name": r.name,
-                # System tags (wip/banner/editor) are kind=general but group under
+                "category": _CATEGORY.get(r.kind, "general"),
-                # their OWN "system" suggestion category so the operator reviews
-                # them apart from content tags (they still train as general heads).
-                "category": "system" if r.is_system else _CATEGORY.get(r.kind, "general"),
                 "auto_apply_threshold": r.auto_apply_threshold,
-                "is_system": bool(r.is_system),
             }
             for r in rows
         ]
@@ -449,70 +290,14 @@ async def _current_heads(session: AsyncSession, embedding_version: str):
     return loaded
 
 
-async def _image_bag(
-    session: AsyncSession, image_id: int, cur_version: str,
-) -> tuple[list, list[dict | None]]:
-    """The max-over-bag inputs for one image: the whole-image SigLIP vector (when
-    it's in the current model's space) PLUS every concept-region crop embedded in
-    that space. Returns (bag, bag_meta) as PARALLEL lists — bag_meta[i] is None for
-    the whole-image row, else the region's {bbox, kind, detector} so a surfaced tag
-    can point back at the crop that produced it (#1206 grounding).
-
-    Only current-version embeddings enter the bag: mid model-swap (#1190) an image
-    still carrying an OLD-version whole-image vector is skipped rather than scored
-    by heads trained in a different space; a legacy NULL version is treated as
-    current (those predate per-row stamping). Shared by live scoring (score_image)
-    and on-demand applied-tag grounding (ground_applied_tag, #1206 Step 4)."""
-    import numpy as np
-
-    img = await session.get(ImageRecord, image_id)
-    bag: list = []
-    bag_meta: list[dict | None] = []
-    if img is None:
-        return bag, bag_meta
-    if img.siglip_embedding is not None and img.siglip_model_version in (
-        cur_version, None,
-    ):
-        bag.append(np.asarray(img.siglip_embedding, dtype=np.float32))
-        bag_meta.append(None)
-    region_rows = (
-        await session.execute(
-            select(
-                ImageRegion.siglip_embedding,
-                ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh,
-                ImageRegion.kind, ImageRegion.detector_version,
-            )
-            .where(ImageRegion.image_record_id == image_id)
-            .where(ImageRegion.siglip_embedding.is_not(None))
-            .where(ImageRegion.embedding_version == cur_version)
-        )
-    ).all()
-    for vec, rx, ry, rw, rh, kind, detector in region_rows:
-        if vec is not None:
-            bag.append(np.asarray(vec, dtype=np.float32))
-            bag_meta.append(
-                {"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector}
-            )
-    return bag, bag_meta
-
-
 async def score_image(
     session: AsyncSession, image_id: int, threshold_override: float | None = None,
 ) -> list[dict]:
     """Suggestions for one image from the trained heads: [{tag_id, name,
-    category, score, above_threshold, grounding}], ranked. A concept is INCLUDED
+    category, score}], ranked. A concept surfaces when its score clears the
-    when its score clears the head's own suggest_threshold — or, when
+    head's own suggest_threshold — or, when threshold_override is given (the
-    threshold_override is given (the typed-dropdown "show everything" mode), that
+    typed-dropdown "show everything" mode), that flat floor instead (0 → every
-    flat floor instead (0 → every head). System-tag heads (wip/banner/editor)
+    head). Empty if the image has no embedding or no heads exist yet.
-    instead use a flat _SYSTEM_TAG_SUGGEST_FLOOR so their false positives surface
-    for rejection (still overridden by threshold_override). Empty if the image has
-    no embedding or no heads exist yet.
-
-    ``above_threshold`` is reported SEPARATELY from inclusion: it's always whether
-    the score cleared the head's NATURAL cut (suggest_threshold, or the system
-    floor), regardless of any override. So the single min=0 fetch returns every
-    head, and the caller can split panel (above_threshold) from dropdown (all)
-    without a second request.
 
     MAX-OVER-BAG: the image is scored as a BAG of embeddings — the whole-image
     vector PLUS every concept-region crop the agent embedded (same model
@@ -522,12 +307,35 @@ async def score_image(
     always in the bag, so this can never score lower than whole-image alone."""
     import numpy as np
 
+    img = await session.get(ImageRecord, image_id)
+    if img is None:
+        return []
     settings = await _settings_async(session)
     cur_version = settings.embedder_model_version
     heads = await _current_heads(session, cur_version)
     if heads["W"] is None:
         return []
-    bag, bag_meta = await _image_bag(session, image_id, cur_version)
+
+    # Only embeddings in the CURRENT model's space enter the bag. Mid model-swap
+    # (#1190), an image still carrying the OLD-version whole-image vector is
+    # skipped rather than scored by heads trained in a different space; a legacy
+    # NULL version is treated as current (those predate per-row stamping).
+    bag = []
+    if img.siglip_embedding is not None and img.siglip_model_version in (
+        cur_version, None,
+    ):
+        bag.append(np.asarray(img.siglip_embedding, dtype=np.float32))
+    region_vecs = (
+        await session.execute(
+            select(ImageRegion.siglip_embedding)
+            .where(ImageRegion.image_record_id == image_id)
+            .where(ImageRegion.siglip_embedding.is_not(None))
+            .where(ImageRegion.embedding_version == cur_version)
+        )
+    ).all()
+    for (vec,) in region_vecs:
+        if vec is not None:
+            bag.append(np.asarray(vec, dtype=np.float32))
     if not bag:
         return []
 
@@ -536,83 +344,22 @@ async def score_image(
     norms[norms == 0] = 1.0
     Xn = X / norms
     Z = Xn @ heads["W"].T + heads["b"]                  # (B, H)
-    probs_bag = 1.0 / (1.0 + np.exp(-Z))                # (B, H)
+    probs = (1.0 / (1.0 + np.exp(-Z))).max(axis=0)      # (H,) best over the bag
-    probs = probs_bag.max(axis=0)                       # (H,) best over the bag
-    # ARGMAX beside the max: WHICH bag row won each head → the region that grounds
-    # the tag (bag_meta[win]); None when the whole-image vector won (#1206).
-    winners = probs_bag.argmax(axis=0)                  # (H,)
     out = []
     for i, p in enumerate(probs):
-        m = heads["meta"][i]
+        cut = threshold_override if threshold_override is not None else heads["thr"][i]
-        # The head's NATURAL suggest cut — system tags use the flat floor (see
-        # _SYSTEM_TAG_SUGGEST_FLOOR) so their false positives show up for the
-        # operator to reject; content heads use their own precision-tuned
-        # threshold. This is what "above threshold" means (drives the panel).
-        natural = (
-            _SYSTEM_TAG_SUGGEST_FLOOR if m["is_system"] else float(heads["thr"][i])
-        )
-        # INCLUSION is looser under threshold_override (dropdown show-all,
-        # override=0): every head comes back so a low-confidence concept can still
-        # be typed + picked, each carrying its own above_threshold flag.
-        cut = threshold_override if threshold_override is not None else natural
         if p >= cut:
+            m = heads["meta"][i]
             out.append({
                 "tag_id": m["tag_id"],
                 "name": m["name"],
                 "category": m["category"],
                 "score": float(p),
-                "above_threshold": bool(p >= natural),
-                "grounding": bag_meta[int(winners[i])],
             })
     out.sort(key=lambda d: d["score"], reverse=True)
     return out
 
 
-async def ground_applied_tag(
-    session: AsyncSession, image_id: int, tag_id: int,
-) -> tuple[dict | None, bool]:
-    """On-demand grounding for an ALREADY-APPLIED tag (#1206 Step 4). Applied tags
-    aren't scored live, so recompute the max-over-bag argmax for just this tag's
-    head — which crop region best explains the tag on this image — mirroring what
-    score_image records for live suggestions. Returns (grounding, has_head):
-
-    - has_head False → the tag has no head in the current embedding space (manual/
-      artist/meta tags, or a concept below the head floor). Nothing to localize
-      with, so the UI shows no overlay (distinct from "the whole image won").
-    - grounding None (has_head True) → the whole-image vector best explains it,
-      not any crop; the UI shows the subtle whole-image frame.
-    - grounding {bbox, kind, detector} → the winning region.
-
-    Character heads are covered too (character is a head kind); this deliberately
-    reuses the SigLIP head bag rather than the CCIP figure path so every applied
-    concept grounds through one consistent mechanism."""
-    import numpy as np
-
-    cur_version = (await _settings_async(session)).embedder_model_version
-    row = (
-        await session.execute(
-            select(TagHead.weights, TagHead.bias).where(
-                TagHead.tag_id == tag_id,
-                TagHead.embedding_version == cur_version,
-            )
-        )
-    ).one_or_none()
-    if row is None:
-        return None, False
-    bag, bag_meta = await _image_bag(session, image_id, cur_version)
-    if not bag:
-        return None, True
-
-    X = np.vstack(bag)
-    norms = np.linalg.norm(X, axis=1, keepdims=True)
-    norms[norms == 0] = 1.0
-    Xn = X / norms
-    # The sigmoid is monotonic in the logit, so the highest-probability bag row is
-    # just argmax of the raw score — no need to exponentiate to pick the winner.
-    z = Xn @ np.asarray(row.weights, dtype=np.float32) + float(row.bias)   # (B,)
-    return bag_meta[int(z.argmax())], True
-
-
 async def _settings_async(session: AsyncSession) -> MLSettings:
     return (
         await session.execute(select(MLSettings).where(MLSettings.id == 1))
@@ -660,11 +407,8 @@ def start_head_auto_apply_run(session: Session, params: dict[str, Any]) -> int:
 
 
 def _auto_apply_heads(session: Session, embedding_version: str, min_pos: int):
-    """Eligible CONTENT heads to fire: graduated (auto_apply_threshold set),
+    """Eligible heads to fire: graduated (auto_apply_threshold set), enough
-    enough support, current embedding, NON-system. System tags never auto-apply
+    support, current embedding. Returns the row list (tag_id/name/weights/...)."""
-    via this path — `wip` never auto-applies at all, and banner/editor screenshot
-    go through the presentation path at their own flat threshold (#141). Returns
-    the row list (tag_id/name/weights/...)."""
     return session.execute(
         select(
             TagHead.tag_id, Tag.name, TagHead.weights, TagHead.bias,
@@ -674,7 +418,6 @@ def _auto_apply_heads(session: Session, embedding_version: str, min_pos: int):
         .where(TagHead.embedding_version == embedding_version)
         .where(TagHead.auto_apply_threshold.is_not(None))
         .where(TagHead.n_pos >= min_pos)
-        .where(~Tag.is_system)
     ).all()
 
 
@@ -756,253 +499,3 @@ def auto_apply_sweep(
         for h in range(len(rows))
     ]
     return {"n_applied": sum(applied), "concepts": concepts}
-
-
-_PRESENTATION_SOURCE = "presentation_auto"
-_PROCESS_SOURCE = "process_auto"
-
-# System-tag auto-apply modes (#1464). Both modes run the identical sweep — apply
-# a system tag at a flat threshold with a PROVISIONAL source + a ring-loud review
-# guard — and differ ONLY in which tags, which settings knobs, and which
-# source/review-mode. 'chrome' (banner) is HIDDEN from the gallery; 'process'
-# (wip / editor screenshot) stays VISIBLE (the hide is a gallery-query effect of
-# the tag's group membership, not of this sweep).
-_SWEEP_MODES = {
-    "chrome": {
-        "names": CHROME_SYSTEM_TAGS,
-        "enabled": "presentation_auto_apply_enabled",
-        "threshold": "presentation_auto_apply_threshold",
-        "conflict": "presentation_conflict_threshold",
-        "source": _PRESENTATION_SOURCE,
-    },
-    "process": {
-        "names": PROCESS_SYSTEM_TAGS,
-        "enabled": "process_auto_apply_enabled",
-        "threshold": "process_auto_apply_threshold",
-        "conflict": "process_conflict_threshold",
-        "source": _PROCESS_SOURCE,
-    },
-}
-
-
-def _system_tag_heads(session: Session, embedding_version: str, names):
-    """Trained heads for a system-tag group (chrome banner / process wip+editor).
-    They fire at the group's FLAT threshold regardless of graduation — a head
-    exists once the operator has labelled enough (head_min_positives)."""
-    return session.execute(
-        select(TagHead.tag_id, Tag.name, TagHead.weights, TagHead.bias)
-        .join(Tag, Tag.id == TagHead.tag_id)
-        .where(TagHead.embedding_version == embedding_version)
-        .where(Tag.is_system.is_(True))
-        .where(Tag.name.in_(names))
-    ).all()
-
-
-def _conflict_heads(session: Session, embedding_version: str):
-    """ALL content (non-system) heads — the "does this ALSO look like real
-    content" signal for the presentation conflict guard (#141)."""
-    return session.execute(
-        select(TagHead.tag_id, TagHead.weights, TagHead.bias)
-        .join(Tag, Tag.id == TagHead.tag_id)
-        .where(TagHead.embedding_version == embedding_version)
-        .where(~Tag.is_system)
-    ).all()
-
-
-def _valued_image_ids(session: Session) -> set[int]:
-    """Images the operator has shown they value: carrying a HUMAN or CONFIRMED
-    content (non-system) tag. The presentation sweep never auto-hides these
-    (guard 1) — you tagged it, so the model doesn't get to bury it (#141)."""
-    confirmed = exists().where(
-        TagPositiveConfirmation.image_record_id == image_tag.c.image_record_id,
-        TagPositiveConfirmation.tag_id == image_tag.c.tag_id,
-    )
-    rows = session.execute(
-        select(image_tag.c.image_record_id)
-        .join(Tag, Tag.id == image_tag.c.tag_id)
-        .where(~Tag.is_system)
-        .where(image_tag.c.source.not_in(_AUTO_SOURCES) | confirmed)
-    ).all()
-    return {r[0] for r in rows}
-
-
-def system_tag_auto_apply_sweep(
-    session: Session, *, mode: str, dry_run: bool = False
-) -> dict:
-    """Auto-apply a system-tag group at its FLAT threshold. mode='chrome' (banner,
-    #141) hides the image; mode='process' (wip / editor screenshot, #1464) keeps it
-    VISIBLE — the ONLY difference is the tag group's gallery membership, not this
-    sweep. Two guards keep it safe: (1) never touch an image carrying a
-    human/confirmed content tag; (2) if the image ALSO scores >= the conflict
-    threshold on a content head, still apply but flag it (PresentationReview,
-    mode=<mode>) so the review strip surfaces "also looks like <X>". The source is
-    PROVISIONAL so the head never trains on its own output. No-op unless the mode's
-    enabled flag is set. numpy-only (no sklearn). Returns {n_applied, n_flagged,
-    concepts}."""
-    import numpy as np
-    from sqlalchemy.dialects.postgresql import insert as pg_insert
-
-    cfg = _SWEEP_MODES[mode]
-    settings = _settings(session)
-    if not dry_run and not getattr(settings, cfg["enabled"]):
-        return {"n_applied": 0, "n_flagged": 0, "concepts": []}
-    ver = settings.embedder_model_version
-    pres = _system_tag_heads(session, ver, cfg["names"])
-    if not pres:
-        return {"n_applied": 0, "n_flagged": 0, "concepts": []}
-    thr = float(getattr(settings, cfg["threshold"]))
-    conflict_thr = float(getattr(settings, cfg["conflict"]))
-    source = cfg["source"]
-
-    Wp = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in pres])
-    bp = np.asarray([r.bias for r in pres], dtype=np.float32)
-    pres_tag_ids = [r.tag_id for r in pres]
-    pres_names = [r.name for r in pres]
-
-    conf = _conflict_heads(session, ver)
-    Wc = bc = conf_tag_ids = None
-    if conf:
-        Wc = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in conf])
-        bc = np.asarray([r.bias for r in conf], dtype=np.float32)
-        conf_tag_ids = [r.tag_id for r in conf]
-
-    valued = _valued_image_ids(session)
-
-    # Skip images that already carry, or have rejected, each presentation tag.
-    skip = {tid: set() for tid in pres_tag_ids}
-    for tid in pres_tag_ids:
-        for (iid,) in session.execute(
-            select(image_tag.c.image_record_id).where(image_tag.c.tag_id == tid)
-        ):
-            skip[tid].add(iid)
-        for (iid,) in session.execute(
-            select(TagSuggestionRejection.image_record_id).where(
-                TagSuggestionRejection.tag_id == tid
-            )
-        ):
-            skip[tid].add(iid)
-
-    applied = [0] * len(pres)
-    n_flagged = 0
-    scanned = 0
-    all_ids = list(session.execute(
-        select(ImageRecord.id).where(ImageRecord.siglip_embedding.is_not(None))
-    ).scalars())
-    for start in range(0, len(all_ids), _AUTO_APPLY_CHUNK):
-        chunk = all_ids[start:start + _AUTO_APPLY_CHUNK]
-        emb = _load_embeddings(session, chunk)
-        cids = [i for i in chunk if i in emb]
-        if not cids:
-            continue
-        Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
-        probs = 1.0 / (1.0 + np.exp(-(Xn @ Wp.T + bp)))       # (N, P)
-        if Wc is not None:
-            cprobs = 1.0 / (1.0 + np.exp(-(Xn @ Wc.T + bc)))  # (N, C)
-            max_c = cprobs.max(axis=1)
-            arg_c = cprobs.argmax(axis=1)
-        scanned += len(cids)
-        for p in range(len(pres)):
-            tid = pres_tag_ids[p]
-            for idx in np.where(probs[:, p] >= thr)[0]:
-                iid = cids[int(idx)]
-                if iid in skip[tid] or iid in valued:
-                    continue
-                skip[tid].add(iid)
-                applied[p] += 1
-                if not dry_run:
-                    session.execute(
-                        pg_insert(image_tag)
-                        .values(
-                            image_record_id=iid, tag_id=tid,
-                            source=source,
-                        )
-                        .on_conflict_do_nothing()
-                    )
-                # Guard 2: also looks like real content → still apply, but flag it
-                # for the review strip instead of silently marking (chrome hides,
-                # process stays visible — either way the operator gets a heads-up).
-                if Wc is not None and float(max_c[idx]) >= conflict_thr:
-                    n_flagged += 1
-                    if not dry_run:
-                        session.execute(
-                            pg_insert(PresentationReview)
-                            .values(
-                                image_record_id=iid, tag_id=tid,
-                                conflict_tag_id=conf_tag_ids[int(arg_c[idx])],
-                                conflict_score=float(max_c[idx]),
-                                mode=mode,
-                            )
-                            .on_conflict_do_nothing()
-                        )
-        if not dry_run:
-            session.commit()
-
-    concepts = [
-        {"tag_id": pres_tag_ids[p], "name": pres_names[p],
-         "applied": applied[p], "scanned": scanned, "threshold": thr}
-        for p in range(len(pres))
-    ]
-    return {
-        "n_applied": sum(applied), "n_flagged": n_flagged, "concepts": concepts,
-    }
-
-
-def retract_auto_applied_heads(session: Session) -> int:
-    """Soft auto-apply (milestone 139): re-score every standing source='head_auto'
-    tag against its CURRENT head and REMOVE the ones now BELOW the head's
-    auto_apply_threshold — i.e. the head sharpened (or the operator raised the bar)
-    and no longer supports them. Skips operator-confirmed tags
-    (TagPositiveConfirmation). SILENT: a low score isn't proof the tag was wrong,
-    so no hard negative is recorded — that's reserved for an operator removal.
-    No-op unless head_auto_apply_enabled. Only re-scores the images that ALREADY
-    carry the auto-tag (bounded), never the whole library. Returns n_retracted."""
-    import numpy as np
-
-    settings = _settings(session)
-    if not settings.head_auto_apply_enabled:
-        return 0
-    heads = session.execute(
-        select(
-            TagHead.tag_id, TagHead.weights, TagHead.bias,
-            TagHead.auto_apply_threshold,
-        )
-        .where(TagHead.embedding_version == settings.embedder_model_version)
-        .where(TagHead.auto_apply_threshold.is_not(None))
-    ).all()
-    retracted = 0
-    for tag_id, weights, bias, thr in heads:
-        auto_ids = [
-            iid for (iid,) in session.execute(
-                select(image_tag.c.image_record_id)
-                .where(image_tag.c.tag_id == tag_id)
-                .where(image_tag.c.source == "head_auto")
-            )
-        ]
-        if not auto_ids:
-            continue
-        confirmed = {
-            iid for (iid,) in session.execute(
-                select(TagPositiveConfirmation.image_record_id)
-                .where(TagPositiveConfirmation.tag_id == tag_id)
-                .where(TagPositiveConfirmation.image_record_id.in_(auto_ids))
-            )
-        }
-        candidates = [i for i in auto_ids if i not in confirmed]
-        emb = _load_embeddings(session, candidates)
-        cids = [i for i in candidates if i in emb]
-        if not cids:
-            continue
-        Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
-        w = np.asarray(weights, dtype=np.float32)
-        probs = 1.0 / (1.0 + np.exp(-(Xn @ w + float(bias))))
-        below = [cids[k] for k in np.where(probs < float(thr))[0]]
-        for iid in below:
-            session.execute(
-                image_tag.delete()
-                .where(image_tag.c.image_record_id == iid)
-                .where(image_tag.c.tag_id == tag_id)
-                .where(image_tag.c.source == "head_auto")
-            )
-            retracted += 1
-        session.commit()
-    return retracted

backend/app/services/ml/suggestions.py

@@ -22,30 +22,27 @@ from .heads import score_image
 
 @dataclass(frozen=True)
 class Suggestion:
-    # Every suggestion is a canonical Tag: heads/CCIP only score EXISTING concept
+    # canonical_tag_id is None when this is a raw Camie tag with no alias and
-    # tags (tagging-v2, #114). The old raw-model-key / creates-new / alias-remap
+    # no existing Tag row — accepting it will create the tag.
-    # cases are gone — a suggestion always maps to a real tag id.
+    canonical_tag_id: int | None
-    canonical_tag_id: int
     display_name: str
     category: str
     score: float
     source: str  # 'head' | 'ccip' | 'both' (Camie tagger/centroid removed in v2)
-    # above_threshold = the score cleared the head's own suggest cut (or the
+    creates_new_tag: bool
-    # system floor). The Suggestions PANEL shows only these; the typed-tag
+    # raw_name = the booru model vocab key behind this suggestion. It's the key
-    # dropdown fetches ALL suggestions (every head, min=0) and just annotates each
+    # an alias MUST be stored under (resolution looks up the raw key), so the
-    # matching row with its score, so a low-confidence concept can still be typed
+    # modal needs it to author an alias correctly. None for centroid-only hits
-    # and picked. CCIP character matches are always above their match threshold.
+    # (no underlying prediction → nothing to alias).
-    above_threshold: bool
+    raw_name: str | None = None
+    # via_alias = this suggestion was surfaced because an operator alias remapped
+    # the raw prediction to this canonical tag. Lets the UI mark it + offer undo.
+    via_alias: bool = False
     # rejected = the operator dismissed this tag for this image (a stored
     # TagSuggestionRejection). It stays in the list — flagged, not dropped — so
     # the rejection is VISIBLE and REVERSIBLE in the rail (misclick recovery,
     # operator-asked 2026-06-27) instead of silently vanishing or re-suggesting.
     rejected: bool = False
-    # grounding = the crop region that produced this suggestion (#1206):
-    # {bbox:[x,y,w,h] normalized, kind, detector}. None when the whole-image
-    # vector won (not localized) or for a CCIP-only hit (figure grounding TBD).
-    # Lets the rail highlight the exact region on hover.
-    grounding: dict | None = None
 
 
 @dataclass
@@ -106,8 +103,6 @@ class SuggestionService:
         for h in hits:
             merged[(h["category"], h["tag_id"])] = {
                 "name": h["name"], "score": h["score"], "source": "head",
-                "above_threshold": h["above_threshold"],
-                "grounding": h.get("grounding"),
             }
         for c in ccip_hits:
             key = ("character", c["tag_id"])
@@ -115,17 +110,9 @@ class SuggestionService:
             if ex is not None:
                 ex["source"] = "both"
                 ex["score"] = max(ex["score"], c["score"])
-                # CCIP only returns matches above its own threshold, so a CCIP
-                # corroboration always makes the merged suggestion above-threshold.
-                ex["above_threshold"] = True
-                # Keep the head's localized crop if it had one; else fall back to
-                # the CCIP figure so a corroborated character still grounds (#1206).
-                ex["grounding"] = ex.get("grounding") or c.get("grounding")
             else:
                 merged[key] = {
                     "name": c["name"], "score": c["score"], "source": "ccip",
-                    "above_threshold": True,
-                    "grounding": c.get("grounding"),
                 }
 
         result = SuggestionList()
@@ -139,9 +126,8 @@ class SuggestionService:
                     category=cat,
                     score=m["score"],
                     source=m["source"],
-                    above_threshold=m["above_threshold"],
+                    creates_new_tag=False,
                     rejected=tag_id in rejected,
-                    grounding=m.get("grounding"),
                 )
             )
         for cat in result.by_category:
@@ -160,7 +146,8 @@ class SuggestionService:
         was suggested for (or already applied to) >= threshold fraction of
         the selection AND was acceptable on >= 1 image. Confidence is the
         mean over images where it was suggested. Aggregated by
-        canonical_tag_id (every suggestion is a canonical tag now)."""
+        canonical_tag_id; creates-new (no canonical id) suggestions are
+        skipped (bulk Accept applies by tag id)."""
         if not image_ids:
             return {}
         threshold = min(1.0, max(0.0, threshold))
@@ -171,6 +158,8 @@ class SuggestionService:
             sl = await self.for_image(image_id)
             for category, items in sl.by_category.items():
                 for s in items:
+                    if s.canonical_tag_id is None or s.creates_new_tag:
+                        continue
                     # for_image keeps rejected tags (flagged) for the rail;
                     # bulk consensus must still ignore them — a tag dismissed on
                     # an image isn't a suggestion for that image.

backend/app/services/ml/tag_eval.py

@@ -0,0 +1,430 @@
+"""Head-vs-centroid tagging eval (#1130, milestone #114 slice 1).
+
+Proves the "frozen embedding + small trained head (with negatives)" spine on the
+operator's OWN data, reusing the SigLIP embeddings already stored on
+image_record. For each concept tag it compares:
+  - CENTROID baseline (the old approach): cosine to the mean of positive vectors.
+  - HEAD (the new approach): logistic regression trained on positives + negatives.
+and reports cross-validated precision/recall/AP for both, a LEARNING CURVE
+(accuracy as the number of tagged positives grows), and example image ids to
+eyeball.
+
+numpy + scikit-learn are imported LAZILY inside run_eval so the API worker (base
+image, no ML stack) can still import start_tag_eval_run to enqueue the ml-queue
+task — the heavy compute only runs on the ml worker.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import UTC, datetime
+from typing import Any
+
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from ...models import (
+    ImageRecord,
+    Tag,
+    TagEvalRun,
+    TagKind,
+    TagPositiveConfirmation,
+    TagSuggestionRejection,
+)
+from ...models.tag import image_tag
+
+log = logging.getLogger(__name__)
+
+# The operator's real concept list (mix of whole-ish + small/local cues). The
+# admin trigger can override; this is the default eval set.
+DEFAULT_CONCEPTS = [
+    "glasses", "cat", "dog", "horse", "goblin",
+    "cum", "lactation", "fellatio", "xray", "stomach bulge",
+]
+DEFAULT_CURVE_POINTS = [10, 30, 100, 300]
+DEFAULT_NEG_RATIO = 3       # negatives per positive (rejections + sampled unlabeled)
+DEFAULT_CV_FOLDS = 5
+MIN_POSITIVES = 8           # below this, a concept can't be evaluated meaningfully
+_UNLABELED_POOL = 4000      # cap on sampled unlabeled rows pulled per concept
+_EXAMPLES_K = 12
+
+
+def start_tag_eval_run(session: Session, params: dict[str, Any]) -> int:
+    """Create a TagEvalRun (status='running') and dispatch the ml-queue task.
+    Returns the new run id. Light guard: one running eval at a time."""
+    existing = session.execute(
+        select(TagEvalRun.id).where(TagEvalRun.status == "running")
+    ).scalar_one_or_none()
+    if existing is not None:
+        raise EvalAlreadyRunning(existing)
+    norm = _normalize_params(params)
+    run = TagEvalRun(params=norm, status="running", last_progress_at=datetime.now(UTC))
+    session.add(run)
+    session.flush()
+    run_id = run.id
+    # Same enqueue-by-import pattern api/suggestions.py uses for ml tasks; the
+    # commit happens in the API handler so row + dispatch are visible together.
+    from ...tasks.ml import tag_eval_run as _task
+    _task.delay(run_id)
+    return run_id
+
+
+class EvalAlreadyRunning(Exception):
+    """Raised by start_tag_eval_run when an eval is already in flight."""
+
+
+def _normalize_params(params: dict[str, Any] | None) -> dict[str, Any]:
+    params = params or {}
+    concepts = [str(c).strip() for c in (params.get("concepts") or []) if str(c).strip()]
+    try:
+        neg_ratio = max(1, int(params.get("neg_ratio", DEFAULT_NEG_RATIO)))
+    except (TypeError, ValueError):
+        neg_ratio = DEFAULT_NEG_RATIO
+    try:
+        cv_folds = max(2, int(params.get("cv_folds", DEFAULT_CV_FOLDS)))
+    except (TypeError, ValueError):
+        cv_folds = DEFAULT_CV_FOLDS
+    try:
+        auto_top_n = min(max(int(params.get("auto_top_n", 0) or 0), 0), 200)
+    except (TypeError, ValueError):
+        auto_top_n = 0
+    try:
+        precision_target = min(max(float(params.get("precision_target", 0.97)), 0.5), 0.999)
+    except (TypeError, ValueError):
+        precision_target = 0.97
+    # No explicit concepts and auto-discovery off → fall back to the hand list.
+    if not concepts and not auto_top_n:
+        concepts = list(DEFAULT_CONCEPTS)
+    curve = params.get("curve_points") or DEFAULT_CURVE_POINTS
+    curve = sorted({int(n) for n in curve if int(n) > 0})
+    return {
+        "concepts": concepts,
+        "neg_ratio": neg_ratio,
+        "cv_folds": cv_folds,
+        "auto_top_n": auto_top_n,
+        "precision_target": round(precision_target, 4),
+        "curve_points": curve,
+    }
+
+
+def _top_general_concepts(session: Session, n: int, min_count: int) -> list[str]:
+    """The n most-tagged general (concept) tags with >= min_count images — a fast
+    server-side way to broaden the eval beyond the hand-picked list (counts all
+    sources; source-aware filtering is a separate concern)."""
+    rows = session.execute(
+        select(Tag.name)
+        .join(image_tag, image_tag.c.tag_id == Tag.id)
+        .where(Tag.kind == TagKind.general)
+        .group_by(Tag.id)
+        .having(func.count(image_tag.c.image_record_id) >= min_count)
+        .order_by(func.count(image_tag.c.image_record_id).desc())
+        .limit(n)
+    ).all()
+    return [r[0] for r in rows]
+
+
+def _resolve_tag_id(session: Session, name: str) -> int | None:
+    """Case-insensitive tag-name match; if several share a name, take the one
+    applied to the most images (the one the operator actually uses)."""
+    rows = session.execute(
+        select(Tag.id, func.count(image_tag.c.image_record_id))
+        .outerjoin(image_tag, image_tag.c.tag_id == Tag.id)
+        .where(func.lower(Tag.name) == name.lower())
+        .group_by(Tag.id)
+        .order_by(func.count(image_tag.c.image_record_id).desc())
+    ).all()
+    return rows[0][0] if rows else None
+
+
+def _ids_with_tag(session: Session, tag_id: int) -> list[int]:
+    return [
+        r[0] for r in session.execute(
+            select(image_tag.c.image_record_id).where(image_tag.c.tag_id == tag_id)
+        ).all()
+    ]
+
+
+def _rejected_ids(session: Session, tag_id: int) -> list[int]:
+    return [
+        r[0] for r in session.execute(
+            select(TagSuggestionRejection.image_record_id)
+            .where(TagSuggestionRejection.tag_id == tag_id)
+        ).all()
+    ]
+
+
+def _confirmed_ids(session: Session, tag_id: int) -> set[int]:
+    """Positives the operator explicitly affirmed ('keep') — excluded from the
+    doubts list so confirmed-correct images don't resurface every run."""
+    return {
+        r[0] for r in session.execute(
+            select(TagPositiveConfirmation.image_record_id)
+            .where(TagPositiveConfirmation.tag_id == tag_id)
+        ).all()
+    }
+
+
+def _sample_unlabeled(session: Session, exclude: set[int], limit: int) -> list[int]:
+    """Random image ids (with an embedding) NOT carrying the tag. Concepts are
+    sparse, so an untagged image is almost always a true negative."""
+    stmt = (
+        select(ImageRecord.id)
+        .where(ImageRecord.siglip_embedding.is_not(None))
+        .order_by(func.random())
+        .limit(limit)
+    )
+    if exclude:
+        stmt = stmt.where(ImageRecord.id.not_in(exclude))
+    return [r[0] for r in session.execute(stmt).all()]
+
+
+def _load_embeddings(session: Session, ids: list[int]) -> dict[int, Any]:
+    import numpy as np
+
+    out: dict[int, Any] = {}
+    if not ids:
+        return out
+    # Chunk the IN list to stay well under psycopg's parameter ceiling.
+    for i in range(0, len(ids), 2000):
+        chunk = ids[i:i + 2000]
+        for rid, emb in session.execute(
+            select(ImageRecord.id, ImageRecord.siglip_embedding)
+            .where(ImageRecord.id.in_(chunk))
+            .where(ImageRecord.siglip_embedding.is_not(None))
+        ).all():
+            out[rid] = np.asarray(emb, dtype=np.float32)
+    return out
+
+
+def run_eval(session: Session, params: dict[str, Any]) -> dict[str, Any]:
+    """Compute the full report. Per-concept failures are captured, not fatal."""
+    import numpy as np
+
+    cfg = _normalize_params(params)
+    # Auto-discovery: union the explicit concepts with the top-N most-tagged
+    # general tags (server-side, fast) so the eval can broaden itself.
+    concepts = list(cfg["concepts"])
+    if cfg["auto_top_n"]:
+        seen = {c.lower() for c in concepts}
+        for name in _top_general_concepts(session, cfg["auto_top_n"], MIN_POSITIVES):
+            if name.lower() not in seen:
+                concepts.append(name)
+                seen.add(name.lower())
+    cfg["concepts"] = concepts
+    concepts_out = []
+    for name in cfg["concepts"]:
+        try:
+            concepts_out.append(_eval_concept(session, name, cfg, np))
+        except Exception as exc:  # one bad concept shouldn't kill the run
+            log.exception("tag-eval concept %r failed", name)
+            concepts_out.append({"name": name, "skipped": f"error: {exc}"})
+    return {
+        "generated_at": datetime.now(UTC).isoformat(),
+        "params": cfg,
+        "concepts": concepts_out,
+    }
+
+
+def _eval_concept(session: Session, name: str, cfg: dict, np) -> dict[str, Any]:
+    tag_id = _resolve_tag_id(session, name)
+    if tag_id is None:
+        return {"name": name, "skipped": "no such tag"}
+    pos_ids = _ids_with_tag(session, tag_id)
+    if len(pos_ids) < MIN_POSITIVES:
+        return {"name": name, "tag_id": tag_id, "n_pos": len(pos_ids),
+                "skipped": f"too few positives (<{MIN_POSITIVES})"}
+
+    neg_ratio = cfg["neg_ratio"]
+    pos_set = set(pos_ids)
+    rejected = [i for i in _rejected_ids(session, tag_id) if i not in pos_set]
+    want_neg = max(len(pos_ids) * neg_ratio, _EXAMPLES_K * 4)
+    sampled = _sample_unlabeled(session, pos_set | set(rejected),
+                                min(_UNLABELED_POOL, want_neg))
+    neg_ids = rejected + [i for i in sampled if i not in pos_set]
+
+    emb = _load_embeddings(session, pos_ids + neg_ids)
+    pos = [(i, emb[i]) for i in pos_ids if i in emb]
+    neg = [(i, emb[i]) for i in neg_ids if i in emb]
+    if len(pos) < MIN_POSITIVES or len(neg) < MIN_POSITIVES:
+        return {"name": name, "tag_id": tag_id, "n_pos": len(pos),
+                "n_neg": len(neg), "skipped": "too few embedded examples"}
+
+    ids = np.array([i for i, _ in pos] + [i for i, _ in neg])
+    X = np.vstack([v for _, v in pos] + [v for _, v in neg]).astype(np.float32)
+    y = np.array([1] * len(pos) + [0] * len(neg))
+    Xn = _l2norm(X, np)
+
+    head = _eval_head(Xn, y, cfg["cv_folds"], cfg["precision_target"], np)
+    centroid = _eval_centroid(Xn, y, cfg["cv_folds"], np)
+    curve = _learning_curve(Xn, y, cfg["curve_points"], neg_ratio, np)
+    confirmed = _confirmed_ids(session, tag_id)
+    examples = _examples(session, Xn, y, ids, np, set(rejected), confirmed)
+
+    return {
+        "name": name, "tag_id": tag_id,
+        "n_pos": len(pos), "n_neg": len(neg),
+        "n_rejected": len(rejected),
+        "head": head, "centroid": centroid,
+        "curve": curve, "examples": examples,
+    }
+
+
+def _l2norm(X, np):
+    n = np.linalg.norm(X, axis=1, keepdims=True)
+    n[n == 0] = 1.0
+    return X / n
+
+
+def _metrics_from_scores(y, scores, np) -> dict[str, float]:
+    from sklearn.metrics import average_precision_score, precision_recall_curve
+
+    ap = float(average_precision_score(y, scores))
+    prec, rec, thr = precision_recall_curve(y, scores)
+    f1 = (2 * prec * rec) / np.clip(prec + rec, 1e-9, None)
+    best = int(np.argmax(f1))
+    # thr has len = len(prec)-1; map best index safely.
+    t = float(thr[min(best, len(thr) - 1)]) if len(thr) else 0.5
+    return {
+        "ap": round(ap, 4),
+        "precision": round(float(prec[best]), 4),
+        "recall": round(float(rec[best]), 4),
+        "f1": round(float(f1[best]), 4),
+        "threshold": round(t, 4),
+    }
+
+
+def _safe_folds(y, folds, np) -> int:
+    minority = int(min(np.bincount(y)))
+    return max(2, min(folds, minority))
+
+
+def _eval_head(Xn, y, folds, target, np) -> dict[str, float]:
+    from sklearn.linear_model import LogisticRegression
+    from sklearn.model_selection import StratifiedKFold, cross_val_predict
+
+    clf = LogisticRegression(max_iter=1000, class_weight="balanced")
+    cv = StratifiedKFold(n_splits=_safe_folds(y, folds, np), shuffle=True,
+                         random_state=0)
+    probs = cross_val_predict(clf, Xn, y, cv=cv, method="predict_proba")[:, 1]
+    m = _metrics_from_scores(y, probs, np)
+    m["auto_apply"] = _auto_apply_point(y, probs, target, np)
+    return m
+
+
+def _auto_apply_point(y, scores, target, np) -> dict | None:
+    """The auto-apply operating point: the threshold that yields the MOST recall
+    while holding precision >= target. This answers 'could this concept fire
+    without a human, and how much would it catch?' Returns None if no threshold
+    reaches the precision target (concept not auto-apply-ready)."""
+    from sklearn.metrics import precision_recall_curve
+
+    prec, rec, thr = precision_recall_curve(y, scores)
+    best = None  # (threshold, precision, recall) maximizing recall s.t. prec>=target
+    for i in range(len(thr)):  # thr[i] corresponds to prec[i], rec[i]
+        if prec[i] >= target and (best is None or rec[i] > best[2]):
+            best = (float(thr[i]), float(prec[i]), float(rec[i]))
+    if best is None:
+        return None
+    return {
+        "target": round(float(target), 4),
+        "threshold": round(best[0], 4),
+        "precision": round(best[1], 4),
+        "recall": round(best[2], 4),
+    }
+
+
+def _eval_centroid(Xn, y, folds, np) -> dict[str, float]:
+    """Cross-validated cosine-to-positive-mean — the OLD method's quality."""
+    from sklearn.model_selection import StratifiedKFold
+
+    cv = StratifiedKFold(n_splits=_safe_folds(y, folds, np), shuffle=True,
+                         random_state=0)
+    scores = np.zeros(len(y), dtype=np.float32)
+    for train, test in cv.split(Xn, y):
+        c = Xn[train][y[train] == 1].mean(axis=0)
+        cn = c / (np.linalg.norm(c) or 1.0)
+        scores[test] = Xn[test] @ cn
+    return _metrics_from_scores(y, scores, np)
+
+
+def _learning_curve(Xn, y, points, neg_ratio, np) -> list[dict[str, float]]:
+    """Hold out a fixed test split; train the head on a growing number of
+    positives and watch AP/F1 climb — answers 'does tagging more sharpen it?'"""
+    from sklearn.linear_model import LogisticRegression
+    from sklearn.model_selection import train_test_split
+
+    rng = np.random.default_rng(0)
+    idx = np.arange(len(y))
+    try:
+        tr, te = train_test_split(idx, test_size=0.3, stratify=y, random_state=0)
+    except ValueError:
+        return []
+    tr_pos = tr[y[tr] == 1]
+    tr_neg = tr[y[tr] == 0]
+    out = []
+    for n in points:
+        if n > len(tr_pos):
+            break
+        sp = rng.choice(tr_pos, size=n, replace=False)
+        nn = min(len(tr_neg), n * neg_ratio)
+        sn = rng.choice(tr_neg, size=nn, replace=False)
+        sub = np.concatenate([sp, sn])
+        clf = LogisticRegression(max_iter=1000, class_weight="balanced")
+        clf.fit(Xn[sub], y[sub])
+        prob = clf.predict_proba(Xn[te])[:, 1]
+        m = _metrics_from_scores(y[te], prob, np)
+        out.append({"n_pos": int(n), "ap": m["ap"], "f1": m["f1"]})
+    return out
+
+
+def _examples(session, Xn, y, ids, np, rejected_set, confirmed_set) -> dict[str, list[dict]]:
+    """Train on all data, then surface: top-scoring negatives the operator has
+    NOT already rejected (= fresh suggestions) and lowest-scoring POSITIVES the
+    operator has NOT already confirmed (= unreviewed doubts). Excluding rejected
+    ids stops an adjudicated near-miss from resurfacing in 'would suggest';
+    excluding confirmed ids stops a 'kept' correct positive from resurfacing in
+    'head doubts' every run. Resolves thumbnail urls for a self-contained report."""
+    from sklearn.linear_model import LogisticRegression
+
+    clf = LogisticRegression(max_iter=1000, class_weight="balanced")
+    clf.fit(Xn, y)
+    s = clf.predict_proba(Xn)[:, 1]
+    neg_idx = np.where(y == 0)[0]
+    pos_idx = np.where(y == 1)[0]
+    top_neg = []
+    for i in neg_idx[np.argsort(s[neg_idx])[::-1]]:  # high score → low
+        rid = int(ids[i])
+        if rid in rejected_set:
+            continue  # already told the head 'no' — don't re-suggest it
+        top_neg.append(rid)
+        if len(top_neg) >= _EXAMPLES_K:
+            break
+    low_pos = []
+    for i in pos_idx[np.argsort(s[pos_idx])]:  # low score → high
+        rid = int(ids[i])
+        if rid in confirmed_set:
+            continue  # already kept/confirmed — don't re-doubt it
+        low_pos.append(rid)
+        if len(low_pos) >= _EXAMPLES_K:
+            break
+    thumbs = _resolve_thumbs(session, top_neg + low_pos)
+    return {
+        "head_would_suggest": [thumbs[i] for i in top_neg if i in thumbs],
+        "head_doubts_positive": [thumbs[i] for i in low_pos if i in thumbs],
+    }
+
+
+def _resolve_thumbs(session, ids: list[int]) -> dict[int, dict]:
+    from ..gallery_service import thumbnail_url
+
+    out: dict[int, dict] = {}
+    if not ids:
+        return out
+    for rid, tp, sha, mime in session.execute(
+        select(
+            ImageRecord.id, ImageRecord.thumbnail_path,
+            ImageRecord.sha256, ImageRecord.mime,
+        ).where(ImageRecord.id.in_(ids))
+    ).all():
+        out[rid] = {"id": rid, "thumbnail_url": thumbnail_url(tp, sha, mime)}
+    return out

backend/app/services/ml/training_data.py

@@ -1,172 +0,0 @@
-"""Shared data-selection + validated-metric helpers for the heads trainer.
-
-Born in the head-vs-centroid eval harness (#1130, tag_eval.py) that proved the
-"frozen embedding + small trained head (with negatives)" spine; the harness was
-retired 2026-07-02 (operator: the tagging system is proven, the eval isn't
-needed) and these survivors moved here — they ARE the heads' production data
-pipeline (heads.py trains and scores with them nightly).
-
-numpy/scikit-learn are imported lazily inside the functions that need them so
-the API worker (base image, no ML stack) can import this module.
-"""
-
-from __future__ import annotations
-
-from typing import Any
-
-from sqlalchemy import func, select
-from sqlalchemy.orm import Session
-
-from ...models import (
-    ImageRecord,
-    Tag,
-    TagPositiveConfirmation,
-    TagSuggestionRejection,
-)
-from ...models.tag import image_tag
-
-# Auto-apply sources whose tags are PROVISIONAL: they never train a head (or seed
-# a CCIP reference) unless the operator confirms them (milestone 139). Keeping
-# auto-applied predictions out of training is what makes them "soft" — a misfire
-# can't reinforce itself, so the retraction sweep can actually drop it.
-# `process_auto` (#1464): wip/editor screenshot applied by the process sweep are
-# ALSO provisional — the head must learn only from title (`wip_title`) + manual
-# labels, never its own auto-applied output, or it would runaway (operator 2026-07-12).
-_AUTO_SOURCES = ("head_auto", "ccip_auto", "ml_auto", "presentation_auto", "process_auto")
-
-
-def _hygiene_excluded_ids(session: Session) -> set[int]:
-    """Ids of images carrying ANY system tag (wip / banner / editor
-    screenshot — milestone #128). These images are excluded from OTHER
-    concepts' head training entirely: not positives (a rough wip tagged as a
-    character drags that head toward 'generic sketch') and not sampled or
-    rejection negatives (a wip OF character X is not evidence against X) —
-    simply absent. A system tag's OWN head trains on them unchanged; that is
-    what makes auto-flagging banners/editor screenshots work.
-
-    Item-level by design: a wip-tagged process video contributes (or
-    withholds) ALL its sampled frames, though some may show the finished
-    piece. Operator call 2026-07-03: with enough clean data this washes out —
-    no per-frame handling.
-    """
-    return set(
-        session.execute(
-            select(image_tag.c.image_record_id)
-            .join(Tag, Tag.id == image_tag.c.tag_id)
-            .where(Tag.is_system.is_(True))
-        ).scalars().all()
-    )
-
-
-def _ids_with_tag(session: Session, tag_id: int) -> list[int]:
-    """Image ids that count as POSITIVES for this tag's head: human-applied
-    (manual / accepted) tags PLUS any auto-applied tag the operator explicitly
-    confirmed (TagPositiveConfirmation). Unconfirmed auto-applied tags are
-    EXCLUDED — they are provisional and must not train the head that judges
-    them (milestone 139)."""
-    confirmed = (
-        select(TagPositiveConfirmation.image_record_id)
-        .where(TagPositiveConfirmation.tag_id == tag_id)
-    )
-    return [
-        r[0] for r in session.execute(
-            select(image_tag.c.image_record_id)
-            .where(image_tag.c.tag_id == tag_id)
-            .where(
-                image_tag.c.source.not_in(_AUTO_SOURCES)
-                | image_tag.c.image_record_id.in_(confirmed)
-            )
-        ).all()
-    ]
-
-
-def _rejected_ids(session: Session, tag_id: int) -> list[int]:
-    return [
-        r[0] for r in session.execute(
-            select(TagSuggestionRejection.image_record_id)
-            .where(TagSuggestionRejection.tag_id == tag_id)
-        ).all()
-    ]
-
-
-def _sample_unlabeled(session: Session, exclude: set[int], limit: int) -> list[int]:
-    """Random image ids (with an embedding) NOT carrying the tag. Concepts are
-    sparse, so an untagged image is almost always a true negative."""
-    stmt = (
-        select(ImageRecord.id)
-        .where(ImageRecord.siglip_embedding.is_not(None))
-        .order_by(func.random())
-        .limit(limit)
-    )
-    if exclude:
-        stmt = stmt.where(ImageRecord.id.not_in(exclude))
-    return [r[0] for r in session.execute(stmt).all()]
-
-
-def _load_embeddings(session: Session, ids: list[int]) -> dict[int, Any]:
-    import numpy as np
-
-    out: dict[int, Any] = {}
-    if not ids:
-        return out
-    # Chunk the IN list to stay well under psycopg's parameter ceiling.
-    for i in range(0, len(ids), 2000):
-        chunk = ids[i:i + 2000]
-        for rid, emb in session.execute(
-            select(ImageRecord.id, ImageRecord.siglip_embedding)
-            .where(ImageRecord.id.in_(chunk))
-            .where(ImageRecord.siglip_embedding.is_not(None))
-        ).all():
-            out[rid] = np.asarray(emb, dtype=np.float32)
-    return out
-
-
-def _l2norm(X, np):
-    n = np.linalg.norm(X, axis=1, keepdims=True)
-    n[n == 0] = 1.0
-    return X / n
-
-
-def _metrics_from_scores(y, scores, np) -> dict[str, float]:
-    from sklearn.metrics import average_precision_score, precision_recall_curve
-
-    ap = float(average_precision_score(y, scores))
-    prec, rec, thr = precision_recall_curve(y, scores)
-    f1 = (2 * prec * rec) / np.clip(prec + rec, 1e-9, None)
-    best = int(np.argmax(f1))
-    # thr has len = len(prec)-1; map best index safely.
-    t = float(thr[min(best, len(thr) - 1)]) if len(thr) else 0.5
-    return {
-        "ap": round(ap, 4),
-        "precision": round(float(prec[best]), 4),
-        "recall": round(float(rec[best]), 4),
-        "f1": round(float(f1[best]), 4),
-        "threshold": round(t, 4),
-    }
-
-
-def _safe_folds(y, folds, np) -> int:
-    minority = int(min(np.bincount(y)))
-    return max(2, min(folds, minority))
-
-
-def _auto_apply_point(y, scores, target, np) -> dict | None:
-    """The auto-apply operating point: the threshold that yields the MOST recall
-    while holding precision >= target. This answers 'could this concept fire
-    without a human, and how much would it catch?' Returns None if no threshold
-    reaches the precision target (concept not auto-apply-ready)."""
-    from sklearn.metrics import precision_recall_curve
-
-    prec, rec, thr = precision_recall_curve(y, scores)
-    best = None  # (threshold, precision, recall) maximizing recall s.t. prec>=target
-    for i in range(len(thr)):  # thr[i] corresponds to prec[i], rec[i]
-        if prec[i] >= target and (best is None or rec[i] > best[2]):
-            best = (float(thr[i]), float(prec[i]), float(rec[i]))
-    if best is None:
-        return None
-    return {
-        "target": round(float(target), 4),
-        "threshold": round(best[0], 4),
-        "precision": round(best[1], 4),
-        "recall": round(best[2], 4),
-    }

backend/app/services/patreon_ingester.py

@@ -87,14 +87,9 @@ class PatreonIngester(Ingester):
         validate: bool = True,
         rate_limit: float = 0.0,
         request_sleep: float = 0.0,
-        auth_token: str | None = None,
         client: PatreonClient | None = None,
         downloader: PatreonDownloader | None = None,
     ):
-        # auth_token: accepted for the uniform native-ingester construction
-        # (download_backends passes it to every adapter); Patreon
-        # authenticates by cookies, so it's unused here.
-        del auth_token
         self.images_root = Path(images_root)
         self.cookies_path = str(cookies_path) if cookies_path else None
         # Pacing (plan #703): request_sleep paces the API page fetches,

backend/app/services/patreon_resolver.py

@@ -139,32 +139,6 @@ def _lookup_via_api(vanity: str, cookies_path: str | None) -> str | None:
     return campaign_id
 
 
-def resolve_display_name(vanity: str, cookies_path: str | None) -> str | None:
-    """The Patreon campaign's display name for `vanity` via the campaigns API
-    (`fields[campaign]=name`), used to name the Artist at add-time (#130). None
-    on any failure — the caller falls back to the vanity handle. Sync: call from
-    an executor."""
-    jar = _load_cookie_jar(cookies_path)
-    try:
-        resp = requests.get(
-            _CAMPAIGNS_URL,
-            params={"filter[vanity]": vanity, "fields[campaign]": "name"},
-            headers={"User-Agent": _USER_AGENT, "Accept": "application/vnd.api+json"},
-            cookies=jar,
-            timeout=_TIMEOUT_SECONDS,
-        )
-        if resp.status_code != 200:
-            return None
-        data = resp.json().get("data")
-    except (requests.RequestException, ValueError) as exc:
-        log.warning("Patreon name lookup failed for vanity=%s: %s", vanity, exc)
-        return None
-    if not isinstance(data, list) or not data or not isinstance(data[0], dict):
-        return None
-    name = (data[0].get("attributes") or {}).get("name")
-    return name.strip() if isinstance(name, str) and name.strip() else None
-
-
 def _scrape_campaign_id(html: str) -> str | None:
     """First campaign id found in creator-page HTML via the known embeddings."""
     if not isinstance(html, str):

backend/app/services/pixiv_client.py

@@ -1,579 +0,0 @@
-"""Native Pixiv client — the Pixiv adapter's read path.
-
-Pixiv has a real (if unofficial) API: the mobile app API gallery-dl drives
-(`PixivAppAPI`). Per the downloader ground rule — gallery-dl is the
-known-working base — this client mirrors gallery-dl 1.32.5's request profile
-EXACTLY: the same iOS app headers on every request, the same OAuth
-refresh-token dance against oauth.secure.pixiv.net (X-Client-Time +
-X-Client-Hash), and the same `/v1/user/illusts` walk paginated by `next_url`.
-Deviating from that profile is how the SubscribeStar/Patreon spikes broke, so
-any change here should be diffed against gallery-dl's extractor first.
-
-Feed shape (characterized from gallery-dl 1.32.5, extractor/pixiv.py):
-  - `GET /v1/user/illusts?user_id=<id>` returns `{"illusts": [work...],
-    "next_url": "https://app-api...?user_id=..&offset=30" | null}`.
-  - Pagination: re-issue the SAME endpoint with `next_url`'s query params. The
-    query string doubles as our resumable page cursor (re-fetching it re-serves
-    the same page — the ingest-core resume contract).
-  - A work carries id/title/type(illust|manga|ugoira)/caption(HTML)/
-    create_date(ISO+09:00)/tags[{name,translated_name}]/user/page_count/
-    x_restrict/series/total_view/total_bookmarks/meta_single_page/meta_pages.
-  - Files: multi-page → meta_pages[].image_urls.original; single page →
-    meta_single_page.original_image_url; ugoira → `/v1/ugoira/metadata` zip
-    (600x600 → 1920x1080 URL swap, gallery-dl's default non-original mode).
-
-`campaign_id` for Pixiv is the numeric user id (extracted from the source URL
-by `user_id_from_url` — no network resolver needed).
-
-Gated works: pixiv serves a `https://s.pximg.net/common/images/limit_*.png`
-placeholder as the "original" when a work is blocked for this account
-(sanity-level filter, my-pixiv lock, deleted). gallery-dl's fallback for those
-is a web-AJAX scrape that needs PHPSESSID browser cookies — FC stores only the
-OAuth refresh token, so (exactly like our previous gallery-dl configuration,
-which warned "No PHPSESSID cookie set") those works are skipped, via the
-post_is_gated seam. Auth failures are loud (rotate the refresh token); a
-response missing the fields we depend on is DRIFT (update this client).
-
-FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import logging
-import time
-from collections.abc import Iterator
-from dataclasses import dataclass
-from datetime import UTC, datetime
-from urllib.parse import parse_qsl, urlsplit
-
-import requests
-
-from ..utils.paths import safe_ext
-from .native_ingest_common import (
-    _MAX_429_RETRIES,
-    NativeAuthError,
-    NativeDriftError,
-    NativeIngestError,
-    make_session,
-    retry_after_seconds,
-)
-
-log = logging.getLogger(__name__)
-
-_TIMEOUT_SECONDS = 30.0
-_API_ROOT = "https://app-api.pixiv.net"
-_OAUTH_URL = "https://oauth.secure.pixiv.net/auth/token"
-
-# gallery-dl's public Pixiv-app credentials (PixivAppAPI, also pixivpy's) —
-# these identify the official iOS app to the API, NOT the operator; the
-# operator's identity is the OAuth refresh token.
-_CLIENT_ID = "MOBrBDS8blbauoSck0ZfDbtuzpyT"
-_CLIENT_SECRET = "lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj"
-_HASH_SECRET = (
-    "28c1fdd170a5204386cb1313c7077b34"
-    "f83e4aaf4aa829ce78c231e05b0bae2c"
-)
-
-# The exact header set gallery-dl 1.32.5 installs on its session — the proven
-# app-API request profile. The Referer also unlocks i.pximg.net media GETs
-# (403 without it), so the downloader reuses this constant.
-PIXIV_APP_HEADERS = {
-    "App-OS": "ios",
-    "App-OS-Version": "16.7.2",
-    "App-Version": "7.19.1",
-    "User-Agent": "PixivIOSApp/7.19.1 (iOS 16.7.2; iPhone12,8)",
-    "Referer": "https://app-api.pixiv.net/",
-}
-
-# Placeholder image prefix pixiv serves instead of a blocked work's original
-# (limit_sanity_level / limit_mypixiv / limit_unknown variants).
-_LIMIT_URL = "https://s.pximg.net/common/images/limit_"
-
-# The app API reports rate-limiting as an error MESSAGE (often on HTTP 403),
-# not only as HTTP 429. gallery-dl sleeps 300s in-walk; sleeping that long
-# inside our time-boxed chunk would eat the whole budget, so we surface it as
-# a typed 429 and let download_service's cooldown machinery honor the wait.
-_RATE_LIMIT_RETRY_AFTER = 300.0
-
-_TITLE_MAX = 50  # gallery-dl pixiv filename template: {title[:50]}
-
-_RATINGS = {0: "General", 1: "R-18", 2: "R-18G"}
-
-
-class PixivAPIError(NativeIngestError):
-    """Base for native Pixiv client failures. status_code / retry_after are
-    inherited from NativeIngestError."""
-
-
-class PixivAuthError(PixivAPIError, NativeAuthError):
-    """Auth failure — missing/expired/revoked OAuth refresh token. Fix =
-    rotate the credential (Settings → Credentials → Pixiv), not update the
-    client. Maps to error_type 'auth_error'."""
-
-
-class PixivDriftError(PixivAPIError, NativeDriftError):
-    """A response did not match the shape this client depends on (missing
-    `illusts`, un-parseable JSON where JSON was promised). Fail loud so the
-    run flags 'the Pixiv app API changed' instead of silently importing
-    nothing. Maps to API_DRIFT."""
-
-
-@dataclass
-class MediaItem:
-    """One resolved downloadable file belonging to a Pixiv work.
-
-    Fields mirror the other native clients' MediaItem so the downloader and
-    ledger are structurally the same. Pixiv original URLs carry no content
-    hash, so `filehash` is always None and the ledger keys on
-    `<post_id>:<media_id>` where media_id is `p<num>` (page) or `ugoira`
-    (the frame zip) — stable across URL-shape drift.
-    """
-
-    url: str
-    filename: str
-    kind: str
-    filehash: str | None
-    post_id: str
-    media_id: str
-
-
-def user_id_from_url(url: str) -> str | None:
-    """The numeric pixiv user id from a source URL, or None.
-
-    Handles the modern forms FC accepts as sources
-    (https://www.pixiv.net/users/<id>, /en/users/<id>) plus the legacy
-    member.php?id=<id>. This IS the campaign id — no network resolver.
-    """
-    parts = urlsplit(url or "")
-    if "pixiv.net" not in parts.netloc:
-        return None
-    segs = [s for s in parts.path.split("/") if s]
-    if segs and segs[0] == "en":
-        segs = segs[1:]
-    if len(segs) >= 2 and segs[0] == "users" and segs[1].isdigit():
-        return segs[1]
-    if segs and segs[0] == "member.php":
-        qid = dict(parse_qsl(parts.query)).get("id", "")
-        if qid.isdigit():
-            return qid
-    return None
-
-
-def _work_filename(work: dict, num: int, url: str) -> str:
-    """gallery-dl layout parity: `{id}_{title[:50]}_{num:>02}.{extension}`
-    (the downloader sanitizes the final segment)."""
-    title = work.get("title")
-    title50 = (title if isinstance(title, str) else "")[:_TITLE_MAX]
-    ext = safe_ext(urlsplit(url).path.rsplit("/", 1)[-1])
-    return f"{work.get('id')}_{title50}_{num:02d}{ext}"
-
-
-class PixivClient:
-    """Synchronous Pixiv app-API read client. Construct with the operator's
-    OAuth refresh token (the same token-type Credential the gallery-dl path
-    consumed as `extractor.pixiv.refresh-token`)."""
-
-    def __init__(
-        self,
-        refresh_token: str | None,
-        *,
-        request_sleep: float = 0.0,
-        max_retries: int = _MAX_429_RETRIES,
-        session: requests.Session | None = None,
-    ):
-        self.refresh_token = refresh_token
-        self._request_sleep = request_sleep or 0.0
-        self._max_retries = max_retries
-        # No cookies — the app API authenticates via the Bearer token _login
-        # installs. make_session still supplies the retry/UA plumbing; the
-        # extra_headers overwrite its browser UA with the app profile.
-        self._session = (
-            session if session is not None
-            else make_session(None, extra_headers=PIXIV_APP_HEADERS)
-        )
-        self._authed_user: dict = {}
-        # Monotonic deadline after which the access token must be refreshed;
-        # 0 forces a refresh on first use.
-        self._token_deadline = 0.0
-
-    # -- auth ----------------------------------------------------------------
-
-    def _login(self) -> None:
-        """Exchange the refresh token for a Bearer access token (gallery-dl's
-        `_login_impl`, including the X-Client-Time/X-Client-Hash pair the
-        endpoint validates). No-op while the current token is still fresh."""
-        if time.monotonic() < self._token_deadline:
-            return
-        if not self.refresh_token:
-            raise PixivAuthError(
-                "No Pixiv refresh token configured — add the OAuth refresh "
-                "token as the Pixiv credential (token type)."
-            )
-        # gallery-dl stamps naive-UTC with a literal +00:00 suffix.
-        now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S+00:00")
-        headers = {
-            "X-Client-Time": now,
-            "X-Client-Hash": hashlib.md5(
-                (now + _HASH_SECRET).encode()
-            ).hexdigest(),
-        }
-        data = {
-            "client_id": _CLIENT_ID,
-            "client_secret": _CLIENT_SECRET,
-            "grant_type": "refresh_token",
-            "refresh_token": self.refresh_token,
-            "get_secure_url": "1",
-        }
-        try:
-            resp = self._session.post(
-                _OAUTH_URL, data=data, headers=headers,
-                timeout=_TIMEOUT_SECONDS,
-            )
-        except requests.RequestException as exc:
-            raise PixivAPIError(f"Pixiv OAuth request failed: {exc}") from exc
-        if resp.status_code >= 400:
-            raise PixivAuthError(
-                "Pixiv rejected the refresh token (HTTP "
-                f"{resp.status_code}) — rotate the Pixiv credential.",
-                status_code=resp.status_code,
-            )
-        try:
-            payload = resp.json()["response"]
-            access = payload["access_token"]
-        except (ValueError, KeyError, TypeError) as exc:
-            raise PixivDriftError(
-                f"Pixiv OAuth response shape changed: {exc}"
-            ) from exc
-        self._authed_user = payload.get("user") or {}
-        self._session.headers["Authorization"] = f"Bearer {access}"
-        # expires_in is 3600 today; refresh 60s early so a long walk never
-        # rides an expiring token into a spurious 400.
-        expires_in = payload.get("expires_in")
-        lifetime = float(expires_in) if isinstance(expires_in, (int, float)) else 3600.0
-        self._token_deadline = time.monotonic() + max(60.0, lifetime - 60.0)
-
-    # -- request -------------------------------------------------------------
-
-    def _call(self, endpoint: str, params: dict) -> dict:
-        """Authenticated app-API GET → parsed JSON body, with the shared 429
-        backoff and the loud auth/drift/rate-limit mapping."""
-        self._login()
-        if self._request_sleep > 0:
-            time.sleep(self._request_sleep)
-        url = _API_ROOT + endpoint
-        attempt = 0
-        while True:
-            try:
-                resp = self._session.get(
-                    url, params=params, timeout=_TIMEOUT_SECONDS
-                )
-            except requests.RequestException as exc:
-                raise PixivAPIError(
-                    f"Pixiv request failed ({endpoint}): {exc}"
-                ) from exc
-            if resp.status_code == 429 and attempt < self._max_retries:
-                attempt += 1
-                delay = retry_after_seconds(resp, attempt)
-                log.warning(
-                    "Pixiv 429 (%s) — backing off %.1fs (retry %d/%d)",
-                    endpoint, delay, attempt, self._max_retries,
-                )
-                time.sleep(delay)
-                continue
-            break
-
-        try:
-            body = resp.json()
-        except ValueError as exc:
-            raise PixivDriftError(
-                f"Pixiv returned non-JSON for {endpoint} "
-                f"(HTTP {resp.status_code})"
-            ) from exc
-
-        error = body.get("error") if isinstance(body, dict) else None
-        message = ""
-        if isinstance(error, dict):
-            message = str(
-                error.get("user_message") or error.get("message") or ""
-            )
-        # Rate limiting first: the app API reports it as an error MESSAGE
-        # (often on HTTP 403), which must not be mistaken for an auth failure.
-        if resp.status_code == 429 or "rate limit" in message.lower():
-            raise PixivAPIError(
-                f"Pixiv rate limit hit ({endpoint}): {message or 'HTTP 429'}",
-                status_code=429,
-                retry_after=_RATE_LIMIT_RETRY_AFTER,
-            )
-        if resp.status_code in (400, 401, 403):
-            # Invalid/expired access token surfaces as 400 invalid_grant-style
-            # errors on the app API; 401/403 are straight auth rejections.
-            raise PixivAuthError(
-                f"Pixiv rejected the request ({endpoint}, HTTP "
-                f"{resp.status_code}): {message or 'auth rejected'} — "
-                "rotate the Pixiv refresh token.",
-                status_code=resp.status_code,
-            )
-        if resp.status_code >= 400:
-            raise PixivAPIError(
-                f"Pixiv API error ({endpoint}, HTTP {resp.status_code}): "
-                f"{message or 'unknown error'}",
-                status_code=resp.status_code,
-            )
-        if error:
-            # HTTP 200 carrying an error object — unexpected, but never
-            # silently treat it as data.
-            raise PixivAPIError(
-                f"Pixiv API error ({endpoint}): {message or error}",
-                status_code=resp.status_code,
-            )
-        return body
-
-    # -- normalization -------------------------------------------------------
-
-    @staticmethod
-    def _normalize(work: dict) -> dict:
-        """Wrap an app-API work in the `{"id", "attributes", ...}` post shape
-        the platform-agnostic core and shared helpers read. The raw work rides
-        along under `_work` for extract_media / the post record."""
-        title = work.get("title")
-        caption = work.get("caption")
-        wtype = work.get("type")
-        return {
-            "id": work.get("id"),
-            "attributes": {
-                "title": title if isinstance(title, str) else "",
-                "content": caption if isinstance(caption, str) else "",
-                "published_at": work.get("create_date"),
-                "post_type": wtype if isinstance(wtype, str) else "illust",
-            },
-            "_work": work,
-        }
-
-    # -- post-first seams ----------------------------------------------------
-
-    @staticmethod
-    def post_record_key(post: dict) -> tuple[str, str] | None:
-        """`(ledger_key, post_id)` gating post-record capture through the seen
-        ledger (`post:<id>` synthetic key), or None when the work has no id."""
-        pid = post.get("id")
-        pid = str(pid) if pid is not None else ""
-        if not pid:
-            return None
-        return (f"post:{pid}", pid)
-
-    @staticmethod
-    def post_meta(post: dict) -> dict:
-        attrs = post.get("attributes") or {}
-        return {"title": attrs.get("title") or None, "date": attrs.get("published_at")}
-
-    @staticmethod
-    def post_is_gated(post: dict) -> bool:
-        """True when this account cannot fetch the work's real files: pixiv
-        substitutes a `limit_*` placeholder for the original (sanity-level
-        filter / my-pixiv lock / deleted), or zeroes the author (deleted
-        account). Mirrors #874 semantics: gated content leaves NO trace — a
-        placeholder thumbnail and an empty stub would only pollute the
-        archive. (gallery-dl's PHPSESSID web-scrape fallback for these is out
-        of scope: FC holds no pixiv browser cookies — module docstring.)"""
-        work = post.get("_work") or {}
-        user = work.get("user") or {}
-        if not user.get("id"):
-            return True
-        if work.get("meta_pages"):
-            return False
-        single = work.get("meta_single_page") or {}
-        original = single.get("original_image_url")
-        return isinstance(original, str) and original.startswith(_LIMIT_URL)
-
-    # -- media ---------------------------------------------------------------
-
-    def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]:
-        """Resolve a work's downloadable files (gallery-dl's `_extract_files`):
-        multi-page originals, the single-page original, or the ugoira frame
-        zip. `included_index` is unused (pixiv works are self-contained)."""
-        work = post.get("_work") or {}
-        pid = str(post.get("id") or "")
-        if not pid or self.post_is_gated(post):
-            return []
-
-        if work.get("type") == "ugoira":
-            return self._ugoira_media(work, pid)
-
-        meta_pages = work.get("meta_pages") or []
-        if meta_pages:
-            items = []
-            for num, page in enumerate(meta_pages):
-                urls = page.get("image_urls") or {}
-                url = urls.get("original")
-                if not isinstance(url, str) or not url:
-                    continue
-                items.append(
-                    MediaItem(
-                        url=url,
-                        filename=_work_filename(work, num, url),
-                        kind="image",
-                        filehash=None,
-                        post_id=pid,
-                        media_id=f"p{num}",
-                    )
-                )
-            return items
-
-        single = work.get("meta_single_page") or {}
-        url = single.get("original_image_url")
-        if not isinstance(url, str) or not url or url.startswith(_LIMIT_URL):
-            return []
-        return [
-            MediaItem(
-                url=url,
-                filename=_work_filename(work, 0, url),
-                kind="image",
-                filehash=None,
-                post_id=pid,
-                media_id="p0",
-            )
-        ]
-
-    def _ugoira_meta(self, work: dict, pid: str) -> dict | None:
-        """Fetch + memoize the ugoira metadata (frames + zip urls) for a work.
-
-        Idempotent and cached on the work dict, so the post record and the
-        media extraction share ONE `/v1/ugoira/metadata` call regardless of
-        which runs first (the core writes the post record BEFORE it extracts
-        media). Returns None — and caches the miss — on a non-auth failure
-        (matching gallery-dl's downgrade); auth failures stay loud."""
-        if "_ugoira_meta" in work:
-            return work["_ugoira_meta"]
-        try:
-            body = self._call("/v1/ugoira/metadata", {"illust_id": pid})
-            meta = body["ugoira_metadata"]
-        except PixivAuthError:
-            raise
-        except (PixivAPIError, KeyError, TypeError) as exc:
-            log.warning("Pixiv ugoira metadata failed for %s: %s", pid, exc)
-            work["_ugoira_meta"] = None
-            return None
-        work["_ugoira_meta"] = meta
-        # Frame delays: a future ugoira→video conversion needs the timings (the
-        # zip alone has none), so the post record captures them.
-        work["_ugoira_frames"] = meta.get("frames") or []
-        return meta
-
-    def fetch_ugoira_frames(self, post: dict) -> None:
-        """Populate `post['_work']['_ugoira_frames']` for an ugoira post (no-op
-        otherwise). The core writes the post record BEFORE extract_media, so
-        without this the frame timings would never reach the record; this
-        fetches (and memoizes, so extract_media reuses it) the metadata. Injected
-        into the downloader by the ingester, mirroring Patreon's content_fetcher.
-        Auth errors propagate; other failures leave frames unset."""
-        work = post.get("_work") or {}
-        if work.get("type") != "ugoira":
-            return
-        pid = str(post.get("id") or "")
-        if pid:
-            self._ugoira_meta(work, pid)
-
-    def _ugoira_media(self, work: dict, pid: str) -> list[MediaItem]:
-        """The ugoira frame zip (gallery-dl's default non-original mode):
-        `/v1/ugoira/metadata` → zip_urls.medium with the 600x600→1920x1080
-        swap. A metadata failure downgrades to 'no media' with a warning
-        (matching gallery-dl) instead of failing the walk — except auth
-        failures, which stay loud."""
-        meta = self._ugoira_meta(work, pid)
-        if meta is None:
-            return []
-        try:
-            zip_url = meta["zip_urls"]["medium"]
-        except (KeyError, TypeError) as exc:
-            log.warning("Pixiv ugoira zip url missing for %s: %s", pid, exc)
-            return []
-        url = zip_url.replace("_ugoira600x600", "_ugoira1920x1080", 1)
-        return [
-            MediaItem(
-                url=url,
-                filename=_work_filename(work, 0, url),
-                kind="ugoira",
-                filehash=None,
-                post_id=pid,
-                media_id="ugoira",
-            )
-        ]
-
-    # -- iteration -----------------------------------------------------------
-
-    def iter_posts(
-        self, campaign_id: str, cursor: str | None = None
-    ) -> Iterator[tuple[dict, dict, str | None]]:
-        """Yield (post, {}, page_cursor) for every work in the user's feed.
-
-        `campaign_id` is the numeric pixiv user id. `cursor` is the query
-        string of the app API's `next_url` (offset pagination); None fetches
-        page 1. The yielded `page_cursor` is the cursor that FETCHED this
-        work's page, so the core checkpoints a value that re-serves the same
-        page on resume (the shared cursor contract)."""
-        if not str(campaign_id or "").isdigit():
-            raise PixivDriftError(
-                f"Pixiv campaign id must be a numeric user id, got "
-                f"{campaign_id!r}"
-            )
-        current = cursor
-        while True:
-            page_cursor = current
-            if current is None:
-                params: dict = {"user_id": campaign_id}
-            else:
-                params = dict(parse_qsl(current))
-            data = self._call("/v1/user/illusts", params)
-            works = data.get("illusts")
-            if not isinstance(works, list):
-                raise PixivDriftError(
-                    "Pixiv user-illusts response had no 'illusts' list "
-                    f"(keys: {sorted(data)[:8]})"
-                )
-            for work in works:
-                if not isinstance(work, dict):
-                    continue
-                yield self._normalize(work), {}, page_cursor
-            next_url = data.get("next_url")
-            if not next_url:
-                return
-            current = str(next_url).rpartition("?")[2]
-
-    # -- user detail ---------------------------------------------------------
-
-    def resolve_display_name(self, user_id: str) -> str | None:
-        """The pixiv user's display name via `/v1/user/detail` (gallery-dl's
-        user_detail) — used to name the Artist when a source is added by numeric
-        id. None on any failure (the caller falls back to the id)."""
-        try:
-            body = self._call("/v1/user/detail", {"user_id": str(user_id)})
-        except PixivAPIError:
-            return None
-        name = (body.get("user") or {}).get("name") if isinstance(body, dict) else None
-        return name if isinstance(name, str) and name.strip() else None
-
-    # -- verify --------------------------------------------------------------
-
-    def verify_auth(self) -> tuple[bool | None, str]:
-        """Cheap credential probe: run the OAuth refresh (the thing that fails
-        when the token is bad) without walking any feed."""
-        try:
-            self._token_deadline = 0.0  # force a real refresh
-            self._login()
-        except PixivAuthError as exc:
-            return False, f"Pixiv rejected the credential — {exc}"
-        except PixivAPIError as exc:
-            return None, f"Couldn't verify (network/HTTP issue): {exc}"
-        account = self._authed_user.get("account") or self._authed_user.get("name")
-        suffix = f" as {account}" if account else ""
-        return True, f"Credentials valid — Pixiv OAuth refresh succeeded{suffix}."
-
-
-def rating_label(x_restrict) -> str | None:
-    """Human rating from pixiv's x_restrict (0/1/2) — written into the post
-    record so the archive keeps the R-18 flag without the reader needing to
-    know pixiv's numeric scheme."""
-    if isinstance(x_restrict, bool) or not isinstance(x_restrict, int):
-        return None
-    return _RATINGS.get(x_restrict)

backend/app/services/pixiv_downloader.py

@@ -1,276 +0,0 @@
-"""Native Pixiv media downloader — the Pixiv counterpart to
-patreon_downloader / subscribestar_downloader.
-
-Given a normalized Pixiv work and its resolved `MediaItem`s
-(pixiv_client.extract_media), download the originals to gallery-dl's on-disk
-layout (so pre-cutover gallery-dl downloads are recognized on disk and not
-re-fetched), write the post-first sidecars the importer consumes, and report
-per-media outcomes.
-
-On-disk layout (matches FC's gallery-dl pixiv config, PLATFORM_DEFAULTS:
-base-directory `<images_root>/<artist_slug>/pixiv` + `directory:
-["{category}"]` + filename `{id}_{title[:50]}_{num:>02}.{extension}`):
-
-    <images_root>/<artist_slug>/pixiv/pixiv/<id>_<title50>_<NN>.<ext>
-
-— note the intentional DOUBLE `pixiv` segment: gallery-dl appended
-`{category}` under a base-directory that already ended in the platform name,
-and tier-2 disk-skip parity requires reproducing that exactly. The layout is
-FLAT (no per-post directory), so the post-first record is `_post_<id>.json`
-in the same directory (the id suffix prevents the collisions a bare
-`_post.json` would have here; phase 3 receives explicit post_record_paths, so
-the name is a convention, not a discovery key).
-
-Simpler than Patreon (no Mux/yt-dlp video branch) — the one special file is
-the ugoira frame zip, downloaded as-is; FC's archive-containment import
-extracts the frames, and the frame DELAYS ride the post record (the zip
-carries none — a future ugoira→video conversion needs them).
-
-PURE: no DB; the seen-skip is an injected predicate. FC runs on a plain-HTTP
-homelab; nothing here uses a secure-context Web API.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import re
-import time
-from collections.abc import Callable
-from pathlib import Path
-
-import requests
-
-from .native_ingest_common import (
-    BaseNativeDownloader,
-    MediaOutcome,
-    PostRecordOutcome,
-)
-from .pixiv_client import PIXIV_APP_HEADERS, rating_label
-
-log = logging.getLogger(__name__)
-
-# Control chars (0x00–0x1f + 0x7f DEL) — gallery-dl's default `path-remove`.
-_GDL_PATH_REMOVE_RE = re.compile(r"[\x00-\x1f\x7f]")
-
-
-def gdl_clean_filename(name: str) -> str:
-    """Reproduce gallery-dl's on-disk filename EXACTLY as it wrote it on this
-    Linux host, so the tier-2 disk-skip recognizes pre-cutover files instead of
-    re-downloading them.
-
-    gallery-dl's PathFormat.build_filename is `clean_path(clean_segment(name))`.
-    On Linux (verified against gallery-dl 1.32.5 path.py) the defaults resolve to:
-      - path-restrict "auto" → "/"          → clean_segment replaces ONLY "/" → "_"
-      - path-remove   "\\x00-\\x1f\\x7f"       → clean_path DELETES control chars
-      - path-strip    "auto" → ""           → NO trailing dot/space stripping
-    Crucially it does NOT touch the Windows-forbidden set (<>:"|?*) — those stay
-    raw in titles on disk. A stricter sanitizer here would rename any such title,
-    miss the on-disk match, and re-pull the whole work. Order mirrors gallery-dl
-    (segment inner, path outer); for these disjoint char sets it's commutative.
-    """
-    return _GDL_PATH_REMOVE_RE.sub("", name.replace("/", "_"))
-
-# Enrichment keys copied verbatim from the app-API work dict into the post
-# record (they're already JSON scalars/objects). Everything lands in
-# Post.raw_metadata via the importer, so the archive keeps pixiv's stats and
-# structure without a schema change.
-_WORK_PASSTHROUGH_KEYS = (
-    "type",
-    "page_count",
-    "width",
-    "height",
-    "total_view",
-    "total_bookmarks",
-    "total_comments",
-    "is_bookmarked",
-    "illust_ai_type",
-    "series",
-)
-
-
-class PixivDownloader(BaseNativeDownloader):
-    """Download resolved Pixiv media to gallery-dl's on-disk layout.
-    Subclasses BaseNativeDownloader for the shared streaming GET
-    (transient-retry + Range-resume) and validation/quarantine. PURE: no DB."""
-
-    def __init__(
-        self,
-        images_root: Path,
-        cookies_path: str | None = None,
-        *,
-        validate: bool = True,
-        rate_limit: float = 0.0,
-        session: requests.Session | None = None,
-        ugoira_frames_fetcher: Callable[[dict], None] | None = None,
-    ):
-        super().__init__(
-            images_root, cookies_path, platform="pixiv",
-            validate=validate, rate_limit=rate_limit, session=session,
-        )
-        # Injected by the ingester (client.fetch_ugoira_frames) so write_post_record
-        # can populate frame timings — which extract_media memoizes, but the core
-        # writes the post record FIRST. Mirrors Patreon's content_fetcher.
-        self._ugoira_frames_fetcher = ugoira_frames_fetcher
-        if session is None:
-            # i.pximg.net 403s any GET without the app Referer; mirror the
-            # client's full app-header profile (gallery-dl serves media off
-            # the same session it drives the API with). An injected session
-            # (tests) owns its own headers.
-            self.session.headers.update(PIXIV_APP_HEADERS)
-
-    # -- public ------------------------------------------------------------
-
-    def download_post(
-        self,
-        post: dict,
-        media_items: list,
-        artist_slug: str,
-        *,
-        is_seen: Callable[[object], bool] = lambda m: False,
-        should_stop: Callable[[], bool] = lambda: False,
-        recapture: bool = False,
-    ) -> list[MediaOutcome]:
-        """Download every media item of one work; return per-item outcomes.
-        Mirrors SubscribeStarDownloader.download_post (two-tier skip, mid-post
-        time-box, recapture surfacing)."""
-        flat_dir = self._flat_dir(artist_slug)
-        outcomes: list[MediaOutcome] = []
-        for media in media_items:
-            if should_stop():
-                break
-            try:
-                outcomes.append(
-                    self._download_one(
-                        post, media, flat_dir, artist_slug, is_seen,
-                        recapture=recapture,
-                    )
-                )
-            except Exception as exc:  # resilient: isolate one item's failure
-                log.warning(
-                    "Pixiv media failed (work %s, %s): %s",
-                    post.get("id"), getattr(media, "media_id", "?"), exc,
-                )
-                outcomes.append(
-                    MediaOutcome(media=media, status="error", path=None, error=str(exc))
-                )
-        return outcomes
-
-    def _flat_dir(self, artist_slug: str) -> Path:
-        # Double platform segment — gallery-dl layout parity (module docstring).
-        return self.images_root / artist_slug / "pixiv" / "pixiv"
-
-    # -- per-item ----------------------------------------------------------
-
-    def _download_one(
-        self,
-        post: dict,
-        media,
-        flat_dir: Path,
-        artist_slug: str,
-        is_seen: Callable[[object], bool],
-        *,
-        recapture: bool = False,
-    ) -> MediaOutcome:
-        seen = is_seen(media)
-        if seen and not recapture:
-            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
-
-        # The client's filename already carries the {id}_{title50}_{NN} shape
-        # (raw title, gallery-dl-template order); clean it to the byte-exact
-        # name gallery-dl wrote on disk so tier-2 disk-skip matches (else a
-        # re-download of the whole work). See gdl_clean_filename.
-        media_path = flat_dir / gdl_clean_filename(media.filename)
-
-        if media_path.exists():  # tier-2: already on disk
-            return MediaOutcome(
-                media=media, status="skipped_disk", path=media_path, error=None
-            )
-        # recapture: a seen item not on disk is NOT re-downloaded (recovery's job).
-        if seen:
-            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
-
-        flat_dir.mkdir(parents=True, exist_ok=True)
-        if self._rate_limit > 0:
-            time.sleep(self._rate_limit)
-
-        out_path = self._fetch_get(media.url, media_path)
-        reason, quarantine_dest = self._validate_path(out_path, artist_slug, media.url)
-        if reason is not None:
-            return MediaOutcome(
-                media=media, status="quarantined", path=quarantine_dest, error=reason,
-            )
-        self._write_minimal_sidecar(post, out_path, source_url=media.url)
-        return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
-
-    # -- post record ---------------------------------------------------------
-
-    def write_post_record(self, post: dict, artist_slug: str) -> PostRecordOutcome:
-        """Write the post-first `_post_<id>.json` — the sole writer of the post
-        body/metadata on the native path. Beyond the standard body fields, the
-        record carries pixiv's own structure (tags + EN translations, rating,
-        series, view/bookmark counts, AI flag, dimensions, author, ugoira frame
-        delays) so the archive keeps what the platform knows about the work."""
-        attrs = post.get("attributes") or {}
-        work = post.get("_work") or {}
-        title = attrs.get("title") if isinstance(attrs.get("title"), str) else None
-        post_type = attrs.get("post_type") if isinstance(attrs.get("post_type"), str) else None
-        pid = str(post.get("id") or "")
-        if not pid:
-            return PostRecordOutcome(
-                path=None, post_type=post_type, title=title, body_chars=0,
-            )
-
-        content = attrs.get("content")
-        content = content if isinstance(content, str) else ""
-        data: dict = {
-            "category": "pixiv",
-            "id": pid,
-            "title": title or "",
-            "content": content,
-            "published_at": attrs.get("published_at"),
-            # The post permalink is synthesized by platforms/pixiv.py
-            # derive_post_url from `id` at parse time — no url key here.
-            "rating": rating_label(work.get("x_restrict")),
-        }
-        for key in _WORK_PASSTHROUGH_KEYS:
-            if key in work:
-                data[key] = work[key]
-        tags = work.get("tags")
-        if isinstance(tags, list):
-            data["tags"] = [
-                {
-                    "name": t.get("name"),
-                    "translated_name": t.get("translated_name"),
-                }
-                for t in tags
-                if isinstance(t, dict)
-            ]
-        user = work.get("user")
-        if isinstance(user, dict):
-            data["user"] = {
-                "id": user.get("id"),
-                "account": user.get("account"),
-                "name": user.get("name"),
-            }
-        # Ugoira frame timings. extract_media memoizes these, but the core writes
-        # the post record BEFORE extracting media, so fetch them here (shared +
-        # idempotent via the client's memoization) so the record actually keeps
-        # them — the zip carries no timings.
-        if (
-            work.get("type") == "ugoira"
-            and not work.get("_ugoira_frames")
-            and self._ugoira_frames_fetcher is not None
-        ):
-            self._ugoira_frames_fetcher(post)
-        frames = work.get("_ugoira_frames")
-        if frames:
-            data["ugoira_frames"] = frames
-
-        flat_dir = self._flat_dir(artist_slug)
-        flat_dir.mkdir(parents=True, exist_ok=True)
-        path = flat_dir / f"_post_{pid}.json"
-        path.write_text(json.dumps(data, indent=2, ensure_ascii=False))
-        return PostRecordOutcome(
-            path=path, post_type=post_type, title=title, body_chars=len(content),
-        )

backend/app/services/pixiv_ingester.py

@@ -1,121 +0,0 @@
-"""Native Pixiv ingester — the Pixiv ADAPTER over the platform-agnostic core
-(`ingest_core.Ingester`).
-
-Thin counterpart to patreon_ingester / subscribestar_ingester: wires the Pixiv
-client/downloader/ledger models/constraints/key into the core and supplies the
-Pixiv failure mapping. The modes (tick / backfill / recovery / recapture), the
-seen + dead-letter ledgers, cursor checkpointing, and the post-first capture
-all live in the core. `download_service.download_source` drives
-`PixivIngester.run` exactly as it drives the other two.
-
-`campaign_id` is the numeric pixiv user id (download_backends extracts it from
-the source URL — no network resolver). Auth is the operator's OAuth refresh
-token (the token-type Credential), passed as `auth_token` — pixiv is the first
-native platform authenticating by token rather than cookies, so the uniform
-constructor accepts both and ignores what it doesn't need.
-
-FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-from collections.abc import Callable
-from pathlib import Path
-
-from ..models import PixivFailedMedia, PixivSeenMedia
-from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester
-from .pixiv_client import MediaItem, PixivAPIError, PixivClient
-from .pixiv_downloader import PixivDownloader
-
-__all__ = [
-    "DEAD_LETTER_THRESHOLD",
-    "PixivIngester",
-    "_ledger_key",
-    "verify_pixiv_credential",
-]
-
-log = logging.getLogger(__name__)
-
-_LEDGER_KEY_MAX = 128
-
-
-def _ledger_key(media: MediaItem) -> str:
-    """Stable per-media identity for the cross-run seen-ledger. Pixiv original
-    URLs carry no content hash, so the key is the page/zip identity scoped to
-    its work: `<illust_id>:p<num>` / `<illust_id>:ugoira`. Bounded to the
-    column width."""
-    if media.filehash:
-        return media.filehash
-    return f"{media.post_id}:{media.media_id}"[:_LEDGER_KEY_MAX]
-
-
-class PixivIngester(Ingester):
-    """Walk a pixiv user's works, download unseen originals, return a
-    `DownloadResult`. A thin adapter over `ingest_core.Ingester`; `client` /
-    `downloader` are injectable seams so unit tests run without network."""
-
-    def __init__(
-        self,
-        images_root: Path,
-        cookies_path: str | None,
-        session_factory: Callable[[], object],
-        *,
-        validate: bool = True,
-        rate_limit: float = 0.0,
-        request_sleep: float = 0.0,
-        auth_token: str | None = None,
-        client: PixivClient | None = None,
-        downloader: PixivDownloader | None = None,
-    ):
-        self.images_root = Path(images_root)
-        self.cookies_path = str(cookies_path) if cookies_path else None
-        resolved_client = (
-            client
-            if client is not None
-            else PixivClient(auth_token, request_sleep=request_sleep)
-        )
-        resolved_downloader = (
-            downloader
-            if downloader is not None
-            else PixivDownloader(
-                self.images_root, cookies_path, validate=validate, rate_limit=rate_limit,
-                # write_post_record runs before extract_media in the core, so it
-                # fetches ugoira frame timings via the SAME client (shared,
-                # memoized) — else the record's ugoira_frames stays empty.
-                ugoira_frames_fetcher=resolved_client.fetch_ugoira_frames,
-            )
-        )
-        super().__init__(
-            client=resolved_client,
-            downloader=resolved_downloader,
-            session_factory=session_factory,
-            seen_model=PixivSeenMedia,
-            failed_model=PixivFailedMedia,
-            seen_constraint="uq_pixiv_seen_media_source_id",
-            failed_constraint="uq_pixiv_failed_media_source_id",
-            ledger_key=_ledger_key,
-            platform="pixiv",
-            error_base=PixivAPIError,
-            # API_DRIFT message phrasing; the base Ingester._failure_result owns
-            # the auth/drift/HTTP→error_type mapping (shared across platforms).
-            drift_label="Pixiv app API",
-            # Captions are legitimately empty for many pixiv artists, so the
-            # zero-bodies #862 canary would false-positive here; the client's
-            # response-shape checks (missing `illusts` → drift) cover the same
-            # failure class structurally.
-            body_canary=False,
-        )
-
-
-async def verify_pixiv_credential(
-    auth_token: str | None,
-) -> tuple[bool | None, str]:
-    """Native Pixiv credential probe — one OAuth refresh via
-    PixivClient.verify_auth (the exact call that fails when the token is
-    bad; no feed walk). Returns the uniform `(ok, message)` contract so
-    download_backends.verify_source_credential treats it like the others."""
-    client = PixivClient(auth_token)
-    loop = asyncio.get_running_loop()
-    return await loop.run_in_executor(None, client.verify_auth)

backend/app/services/platforms/pixiv.py

@@ -1,14 +1,8 @@
 """Pixiv — one quirk.
 
-post_url: the sidecar's `url` (legacy gallery-dl era) is the image URL
+post_url: gallery-dl's `url` is the image URL on `i.pximg.net`. The
-on `i.pximg.net`, and the native post record (#129) writes no url key
+post permalink follows /artworks/<id>. external_post_id (= `id`) was
-at all — the permalink is synthesized from `id` here either way:
+already correct, so no override there.
-/artworks/<id>. external_post_id (= `id`) was already correct, so no
-override there.
-
-Downloads run through the native ingester (pixiv_ingester.py), not
-gallery-dl; this registry entry still owns URL validation, sidecar
-parsing, and the credential surface (the OAuth refresh token).
 """
 
 from .base import GD_DEFAULTS, PlatformInfo, str_id_value

backend/app/services/platforms/subscribestar.py

@@ -14,11 +14,9 @@
    `_personalization_id` age-confirmation cookie that the user can't
    easily refresh — SubscribeStar's frontend JS uses localStorage to
    suppress the age popup once dismissed. gallery-dl's own login flow
-   sidesteps this by setting `18_plus_agreement_generic=true`; we mirror
+   sidesteps this by setting `18_plus_agreement_generic=true` on
-   that for cookies captured via the extension — on EVERY SubscribeStar
+   `.subscribestar.adult`; we mirror that for cookies captured via the
-   domain, because cookies are domain-scoped and the wall exists on all
+   extension.
-   of them: an .adult-only line never rides along to a .art creator page
-   (Elasid, event #54116 — every poll 302'd to /age_confirmation_warning).
 """
 
 from .base import GD_DEFAULTS, PlatformInfo, str_id_value
@@ -31,31 +29,20 @@ def derive_post_url(data: dict) -> str | None:
     return None
 
 
-# All domains SubscribeStar serves creators from; the age wall gates each.
-_SS_DOMAINS = (".subscribestar.com", ".subscribestar.adult", ".subscribestar.art")
-
-
 def augment_cookies(netscape: str) -> str:
+    if "18_plus_agreement_generic" in netscape:
+        return netscape
     # Far-future expiry — gallery-dl's own login flow sets this with no
     # explicit expiry; the server only checks presence/value.
     expiry = 4102444800  # 2100-01-01 UTC
+    line = "\t".join([
+        ".subscribestar.adult", "TRUE", "/", "TRUE",
+        str(expiry), "18_plus_agreement_generic", "true",
+    ])
     body = netscape.rstrip("\n")
     if not body:
         body = "# Netscape HTTP Cookie File"
-    lines = netscape.splitlines()
+    return body + "\n" + line + "\n"
-    for domain in _SS_DOMAINS:
-        # Per-domain presence check: captured cookies carrying the age
-        # cookie for one domain must not suppress injection on the others.
-        if any(
-            line.startswith(domain + "\t") and "18_plus_agreement_generic" in line
-            for line in lines
-        ):
-            continue
-        body += "\n" + "\t".join([
-            domain, "TRUE", "/", "TRUE",
-            str(expiry), "18_plus_agreement_generic", "true",
-        ])
-    return body + "\n"
 
 
 INFO = PlatformInfo(
@@ -64,11 +51,10 @@ INFO = PlatformInfo(
     description="Download posts from SubscribeStar creators",
     auth_type="cookies",
     requires_auth=True,
-    url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult|art)/",
+    url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult)/",
     url_examples=[
         "https://subscribestar.adult/example_artist",
         "https://www.subscribestar.com/example_artist",
-        "https://subscribestar.art/example_artist",
     ],
     default_config={**GD_DEFAULTS, "content_types": ["all"]},
     derive_post_url=derive_post_url,

backend/app/services/post_feed_service.py

@@ -200,8 +200,6 @@ class PostFeedService:
         atts_map = await self._attachments_for([post.id])
         item = self._to_dict(post, artist, source, thumbs_map, atts_map)
         item["description_full"] = html_to_plain(post.description)
-        # Full (uncapped) translated description for the detail view (#143).
-        item["description_translated_full"] = post.description_translated
         # Sanitized HTML body for faithful (semantic) rendering in the post view;
         # detail-only (the feed list stays lightweight plain text). None when the
         # post has no body. Inline `<img>` sources are remapped to locally-served
@@ -373,12 +371,6 @@ class PostFeedService:
             description_plain, truncated = None, False
         else:
             description_plain, truncated = truncate_at_word(plain_full, DESCRIPTION_LIMIT)
-        # Translation (#143): the stored translated description is already plain
-        # text; truncate it the same way for the card.
-        desc_trans = post.description_translated
-        desc_trans_short = (
-            truncate_at_word(desc_trans, DESCRIPTION_LIMIT)[0] if desc_trans else None
-        )
         thumbs_entry = thumbs_map.get(post.id, {"thumbs": [], "more": 0})
         # `source` is null for filesystem-imported posts with no live
         # subscription (alembic 0030). Frontend renders that as a
@@ -392,13 +384,6 @@ class PostFeedService:
             "downloaded_at": post.downloaded_at.isoformat(),
             "description_plain": description_plain,
             "description_truncated": truncated,
-            # Translation-forward fields (#143): shown by default when present;
-            # UI toggles back to the originals above. Source lang labels the toggle.
-            "post_title_translated": post.post_title_translated,
-            "description_translated": desc_trans_short,
-            "translated_source_lang": post.translated_source_lang,
-            # Sticky per-post translation choice (auto/force/original, #155).
-            "translation_override": post.translation_override,
             "artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
             "source": (
                 {"id": source.id, "platform": source.platform}

backend/app/services/provenance_service.py

@@ -29,13 +29,6 @@ def _post_dict(p: Post) -> dict:
         "date": p.post_date.isoformat() if p.post_date else None,
         "description_html": sanitize_post_html(p.description),
         "attachment_count": p.attachment_count,
-        # Translation (#143): the English title/description shown by default when
-        # a translation exists; the UI toggles to the original. Source lang labels
-        # the original.
-        "title_translated": p.post_title_translated,
-        "description_translated": p.description_translated,
-        "translated_source_lang": p.translated_source_lang,
-        "translation_override": p.translation_override,
     }
 
 

backend/app/services/source_service.py

@@ -4,18 +4,11 @@ is_subscription auto-flip on first add / last delete.
 
 from dataclasses import dataclass
 
-from sqlalchemy import func, select, update
+from sqlalchemy import func, select
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from ..models import (
+from ..models import Artist, ImportSettings, Source
-    Artist,
-    ImageProvenance,
-    ImageRecord,
-    ImportSettings,
-    Post,
-    Source,
-)
 from .platforms import known_platform_keys
 from .scheduler_service import compute_next_check_at
 
@@ -305,15 +298,6 @@ class SourceService:
 
         for key, value in fields.items():
             setattr(source, key, value)
-        # Disabling a source clears its failure state (operator: disable the subs
-        # you're not paying for without them lingering as "failing"). Re-enabling
-        # then starts clean; the next real run re-derives health. Only on the
-        # explicit disable — an unrelated edit to an already-disabled source
-        # leaves its (already-clear) state alone.
-        if fields.get("enabled") is False:
-            source.last_error = None
-            source.error_type = None
-            source.consecutive_failures = 0
         try:
             await self.session.flush()
         except IntegrityError as exc:
@@ -427,70 +411,6 @@ class SourceService:
         await self.session.commit()
         return await self._row_to_record(source)
 
-    async def reassign(self, source_id: int, target_artist_id: int) -> SourceRecord:
-        """Move a Source — and the content it brought in — to another artist
-        (#130). The slug/storage path is IMMUTABLE, so NO files move: only the
-        artist attribution changes (reads use ImageRecord.path). Re-points the
-        source's Posts and the ImageRecords it contributed (those still
-        attributed to the old artist — images shared with another artist are
-        left alone). If the old artist is left fully empty (no sources, images,
-        or posts) it's deleted (ArtistVisit cascades); if it just lost its last
-        source, its is_subscription flag clears. No-op when already on target."""
-        source = (await self.session.execute(
-            select(Source).where(Source.id == source_id)
-        )).scalar_one_or_none()
-        if source is None:
-            raise LookupError(f"source id={source_id} not found")
-        target = await self._artist_or_raise(target_artist_id)
-        old_artist_id = source.artist_id
-        if old_artist_id == target_artist_id:
-            return await self._row_to_record(source)
-
-        source.artist_id = target_artist_id
-        target.is_subscription = True
-        # Re-attribute this source's posts (Post.artist_id is denormalized).
-        await self.session.execute(
-            update(Post).where(Post.source_id == source_id)
-            .values(artist_id=target_artist_id)
-        )
-        # Re-attribute the images this source contributed that are still on the
-        # OLD artist. Scoping to artist_id == old avoids stealing an image that
-        # a different artist's source also contributed.
-        contributed = select(ImageProvenance.image_record_id).where(
-            ImageProvenance.source_id == source_id
-        )
-        await self.session.execute(
-            update(ImageRecord)
-            .where(
-                ImageRecord.artist_id == old_artist_id,
-                ImageRecord.id.in_(contributed),
-            )
-            .values(artist_id=target_artist_id)
-        )
-        await self.session.flush()
-
-        old = (await self.session.execute(
-            select(Artist).where(Artist.id == old_artist_id)
-        )).scalar_one_or_none()
-        if old is not None:
-            n_src = (await self.session.execute(
-                select(func.count(Source.id)).where(Source.artist_id == old_artist_id)
-            )).scalar_one()
-            n_img = (await self.session.execute(
-                select(func.count(ImageRecord.id)).where(
-                    ImageRecord.artist_id == old_artist_id
-                )
-            )).scalar_one()
-            n_post = (await self.session.execute(
-                select(func.count(Post.id)).where(Post.artist_id == old_artist_id)
-            )).scalar_one()
-            if n_src == 0 and n_img == 0 and n_post == 0:
-                await self.session.delete(old)      # ArtistVisit cascades
-            elif n_src == 0:
-                old.is_subscription = False
-        await self.session.commit()
-        return await self._row_to_record(source)
-
     async def delete(self, source_id: int) -> None:
         source = (await self.session.execute(
             select(Source).where(Source.id == source_id)

backend/app/services/subscribestar_client.py

@@ -226,35 +226,13 @@ def _attachment_item(
     )
 
 
-def _normalize_ss_host(netloc: str) -> str:
-    """Rewrite the `subscribestar.art` host to `subscribestar.adult`.
-
-    The age wall on the `.art` domain does not clear with the
-    `18_plus_agreement_generic` cookie (unlike `.com`/`.adult`): a `.art`
-    creator page keeps 302'ing to `/age_confirmation_warning` even with the
-    cookie set (Elasid, event #54116). The same creator is reachable on
-    `.adult`, where the cookie works — so `.art` behaves as an alias that
-    doesn't honor the age gate. Normalize it to `.adult` at request time (the
-    stored Source.url is left untouched). `.com`/`.adult` pass through.
-    """
-    host = netloc.lower()
-    if host == "subscribestar.art" or host.endswith(".subscribestar.art"):
-        rewritten = netloc[: -len("art")] + "adult"
-        log.info(
-            "SubscribeStar: rewrote age-gated .art host %r → %r", netloc, rewritten
-        )
-        return rewritten
-    return netloc
-
-
 def _split_creator_url(campaign_id: str) -> tuple[str, str]:
     """`campaign_id` is the creator URL → (base, slug).
 
-    base = scheme://host (preserving .com vs .adult; .art → .adult, see
+    base = scheme://host (preserving .com vs .adult); slug = first path segment.
-    _normalize_ss_host); slug = first path segment.
     """
     parts = urlsplit(campaign_id)
-    base = f"{parts.scheme or 'https'}://{_normalize_ss_host(parts.netloc)}"
+    base = f"{parts.scheme or 'https'}://{parts.netloc}"
     slug = parts.path.strip("/").split("/")[0] if parts.path else ""
     return base, slug
 
@@ -273,30 +251,6 @@ def _parse_ss_datetime(text: str) -> str | None:
     return None
 
 
-_OG_TITLE_RE = re.compile(
-    r'<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']',
-    re.IGNORECASE,
-)
-_TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
-# Trailing " | SubscribeStar" / " on SubscribeStar" the profile <title> carries.
-_SS_TITLE_SUFFIX_RE = re.compile(
-    r"\s*[|·]\s*SubscribeStar.*$|\s+on\s+SubscribeStar.*$", re.IGNORECASE
-)
-
-
-def _extract_creator_name(html: str) -> str | None:
-    """The creator's display name from a SubscribeStar profile page: prefer the
-    og:title meta (it's the bare creator name), else the <title> with the
-    SubscribeStar suffix stripped. None when neither yields anything (#130)."""
-    m = _OG_TITLE_RE.search(html)
-    name = unescape(m.group(1)).strip() if m else ""
-    if not name:
-        t = _TITLE_RE.search(html)
-        raw = unescape(t.group(1)).strip() if t else ""
-        name = _SS_TITLE_SUFFIX_RE.sub("", raw).strip()
-    return name or None
-
-
 class SubscribeStarClient:
     """Synchronous SubscribeStar HTML-scrape read client. Construct with a path
     to a Netscape cookies.txt (the same file CredentialService.get_cookies_path
@@ -628,23 +582,6 @@ class SubscribeStarClient:
             current = next_href
             first_page = False
 
-    # -- display name -------------------------------------------------------
-
-    def resolve_display_name(self, campaign_id: str) -> str | None:
-        """The creator's display name from their profile page, used to name the
-        Artist at add-time (#130). `campaign_id` is the creator URL. None on any
-        failure — the caller falls back to the URL handle. Sync: run in an
-        executor."""
-        base, slug = _split_creator_url(campaign_id)
-        if not slug:
-            return None
-        self._session.headers["Referer"] = f"{base}/"
-        try:
-            html = self._feed_html(f"{base}/{slug}")
-        except SubscribeStarAPIError:
-            return None
-        return _extract_creator_name(html)
-
     # -- verify ------------------------------------------------------------
 
     def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:

backend/app/services/subscribestar_ingester.py

@@ -61,14 +61,9 @@ class SubscribeStarIngester(Ingester):
         validate: bool = True,
         rate_limit: float = 0.0,
         request_sleep: float = 0.0,
-        auth_token: str | None = None,
         client: SubscribeStarClient | None = None,
         downloader: SubscribeStarDownloader | None = None,
     ):
-        # auth_token: accepted for the uniform native-ingester construction
-        # (download_backends passes it to every adapter); SubscribeStar
-        # authenticates by cookies, so it's unused here.
-        del auth_token
         self.images_root = Path(images_root)
         self.cookies_path = str(cookies_path) if cookies_path else None
         resolved_client = (

backend/app/services/tag_directory_service.py

@@ -107,7 +107,6 @@ class TagDirectoryService:
                 "kind": tag.kind.value if hasattr(tag.kind, "value") else tag.kind,
                 "fandom_id": tag.fandom_id,
                 "fandom_name": fandom_name,
-                "is_system": tag.is_system,
                 "image_count": int(image_count),
                 "preview_thumbnails": previews.get(tag.id, []),
             }

backend/app/services/tag_query.py

@@ -67,35 +67,24 @@ def fandom_join_alias():
 
 
 def tag_columns(fandom_alias):
-    """The canonical (id, name, kind, fandom_id, fandom_name, is_system)
+    """The canonical (id, name, kind, fandom_id, fandom_name) column set for a
-    column set for a tag select that outerjoins `fandom_alias` (from
+    tag select that outerjoins `fandom_alias` (from `fandom_join_alias()`)."""
-    `fandom_join_alias()`)."""
     return [
         Tag.id,
         Tag.name,
         Tag.kind,
         Tag.fandom_id,
         fandom_alias.c.name.label("fandom_name"),
-        Tag.is_system,
     ]
 
 
 def serialize_tag(row) -> dict:
     """Serialize a row carrying .id/.name/.kind/.fandom_id/.fandom_name to the
-    canonical tag dict. `kind` may be a TagKind enum or a plain string.
+    canonical tag dict. `kind` may be a TagKind enum or a plain string."""
-    `is_system` defaults False for callers whose selects predate the column."""
     return {
         "id": row.id,
         "name": row.name,
         "kind": row.kind.value if hasattr(row.kind, "value") else row.kind,
         "fandom_id": row.fandom_id,
         "fandom_name": row.fandom_name,
-        "is_system": bool(getattr(row, "is_system", False)),
-        # Applied-tag context: only the image-scoped selects (list_for_image /
-        # get_image_with_tags) provide these; autocomplete / directory don't →
-        # default. `source` drives the auto-applied badge; `confirmed` = the
-        # operator affirmed the tag (a training positive, shielded from the
-        # retraction sweep — milestone 139).
-        "source": getattr(row, "source", None),
-        "confirmed": bool(getattr(row, "confirmed", False)),
     }