From 6cd7281af5a1621810ce6cb383e601a10e4ecc9f Mon Sep 17 00:00:00 2001
From: Bryan Van Deusen <bryan.vandeusen@fabledsword.com>
Date: Sat, 27 Jun 2026 22:56:41 -0400
Subject: [PATCH] =?UTF-8?q?feat(settings):=20tag-eval=20admin=20card=20?=
 =?UTF-8?q?=E2=80=94=20trigger=20+=20persisted=20report=20(survives=20nav)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Frontend for #1130. A maintenance tile in Settings → Tagging:
- Editable concept list + "Run eval" → POST /api/tag-eval (one running at a time).
- Rehydrates on mount via the persisted run (getRun by latest id) and polls while
  running — so the report SURVIVES navigation (operator-flagged); the task runs
  backend-side regardless and the card reconnects to its row.
- Renders the saved report: per-concept head-vs-centroid metrics table (AP/F1/
  precision/recall) with Δ AP, the learning curve (AP @ N positives), and
  thumbnail galleries (head-would-suggest / head-doubts-positive) for eyeballing.

Backend: _examples now stores thumbnail_urls (not just ids) so the report is a
self-contained artifact that renders without per-id lookups on reload.

No new top-level surface — slots into the existing maintenance area.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 backend/app/services/ml/tag_eval.py           |  33 ++-
 .../components/settings/MaintenancePanel.vue  |   2 +
 .../src/components/settings/TagEvalCard.vue   | 211 ++++++++++++++++++
 frontend/src/stores/tagEval.js                |  27 +++
 4 files changed, 266 insertions(+), 7 deletions(-)
 create mode 100644 frontend/src/components/settings/TagEvalCard.vue
 create mode 100644 frontend/src/stores/tagEval.js

diff --git a/backend/app/services/ml/tag_eval.py b/backend/app/services/ml/tag_eval.py
index 132d3f1..2a1cbb5 100644
--- a/backend/app/services/ml/tag_eval.py
+++ b/backend/app/services/ml/tag_eval.py
@@ -201,7 +201,7 @@ def _eval_concept(session: Session, name: str, cfg: dict, np) -> dict[str, Any]:
     head = _eval_head(Xn, y, cfg["cv_folds"], np)
     centroid = _eval_centroid(Xn, y, cfg["cv_folds"], np)
     curve = _learning_curve(Xn, y, cfg["curve_points"], neg_ratio, np)
-    examples = _examples(Xn, y, ids, np)
+    examples = _examples(session, Xn, y, ids, np)
 
     return {
         "name": name, "tag_id": tag_id,
@@ -296,11 +296,13 @@ def _learning_curve(Xn, y, points, neg_ratio, np) -> list[dict[str, float]]:
     return out
 
 
-def _examples(Xn, y, ids, np) -> dict[str, list[int]]:
+def _examples(session, Xn, y, ids, np) -> dict[str, list[dict]]:
     """Train on all data, then surface: top-scoring UNLABELED-ish (highest among
     the negative pool = what the head would newly suggest) and lowest-scoring
     POSITIVES (where the head disagrees with the operator's tag — likely the
-    most informative to review)."""
+    most informative to review). Resolves thumbnail urls so the stored report
+    renders without per-id lookups (survives navigation as a self-contained
+    artifact)."""
     from sklearn.linear_model import LogisticRegression
 
     clf = LogisticRegression(max_iter=1000, class_weight="balanced")
@@ -308,9 +310,26 @@ def _examples(Xn, y, ids, np) -> dict[str, list[int]]:
     s = clf.predict_proba(Xn)[:, 1]
     neg_idx = np.where(y == 0)[0]
     pos_idx = np.where(y == 1)[0]
-    top_neg = neg_idx[np.argsort(s[neg_idx])[::-1][:_EXAMPLES_K]]
-    low_pos = pos_idx[np.argsort(s[pos_idx])[:_EXAMPLES_K]]
+    top_neg = [int(ids[i]) for i in neg_idx[np.argsort(s[neg_idx])[::-1][:_EXAMPLES_K]]]
+    low_pos = [int(ids[i]) for i in pos_idx[np.argsort(s[pos_idx])[:_EXAMPLES_K]]]
+    thumbs = _resolve_thumbs(session, top_neg + low_pos)
     return {
-        "head_would_suggest": [int(ids[i]) for i in top_neg],
-        "head_doubts_positive": [int(ids[i]) for i in low_pos],
+        "head_would_suggest": [thumbs[i] for i in top_neg if i in thumbs],
+        "head_doubts_positive": [thumbs[i] for i in low_pos if i in thumbs],
     }
+
+
+def _resolve_thumbs(session, ids: list[int]) -> dict[int, dict]:
+    from ..gallery_service import thumbnail_url
+
+    out: dict[int, dict] = {}
+    if not ids:
+        return out
+    for rid, tp, sha, mime in session.execute(
+        select(
+            ImageRecord.id, ImageRecord.thumbnail_path,
+            ImageRecord.sha256, ImageRecord.mime,
+        ).where(ImageRecord.id.in_(ids))
+    ).all():
+        out[rid] = {"id": rid, "thumbnail_url": thumbnail_url(tp, sha, mime)}
+    return out
diff --git a/frontend/src/components/settings/MaintenancePanel.vue b/frontend/src/components/settings/MaintenancePanel.vue
index 48cfcf9..68c30d1 100644
--- a/frontend/src/components/settings/MaintenancePanel.vue
+++ b/frontend/src/components/settings/MaintenancePanel.vue
@@ -28,6 +28,7 @@
         <MLThresholdSliders />
         <AllowlistTable />
         <AliasTable />
+        <TagEvalCard />
       </div>
     </section>
 
@@ -53,6 +54,7 @@ import DbMaintenanceCard from './DbMaintenanceCard.vue'
 import MLThresholdSliders from './MLThresholdSliders.vue'
 import AllowlistTable from './AllowlistTable.vue'
 import AliasTable from './AliasTable.vue'
+import TagEvalCard from './TagEvalCard.vue'
 import BackupCard from './BackupCard.vue'
 import { useSystemActivityStore } from '../../stores/systemActivity.js'
 
diff --git a/frontend/src/components/settings/TagEvalCard.vue b/frontend/src/components/settings/TagEvalCard.vue
new file mode 100644
index 0000000..70f3735
--- /dev/null
+++ b/frontend/src/components/settings/TagEvalCard.vue
@@ -0,0 +1,211 @@
+<template>
+  <MaintenanceTile
+    icon="mdi-flask-outline"
+    title="Tagging eval (heads vs centroid)"
+    blurb="Measure whether a trained head beats the old centroid on your own tags — and whether tagging more sharpens it."
+    :open="!!run"
+  >
+    <p class="fc-muted text-body-2 mb-3">
+      Reuses the SigLIP embeddings already stored on your images (no re-embed, no
+      GPU). For each concept it trains a logistic-regression <strong>head</strong>
+      on your positives + negatives and compares it to the old single
+      <strong>centroid</strong>, with cross-validated AP/F1 and a learning curve.
+      Runs as a background task; the result is saved and reloads here.
+    </p>
+
+    <v-textarea
+      v-model="conceptsText" label="Concepts (comma-separated)"
+      rows="2" auto-grow density="compact" hide-details class="mb-3"
+      :disabled="running"
+    />
+
+    <v-btn
+      v-if="!running"
+      color="accent" variant="flat" rounded="pill"
+      prepend-icon="mdi-play" :loading="busy" @click="onStart"
+    >Run eval</v-btn>
+
+    <div v-if="running" class="mt-3">
+      <v-progress-linear indeterminate color="accent" />
+      <div class="text-body-2 mt-2 fc-muted">Running… (started {{ startedAgo }})</div>
+    </div>
+
+    <v-alert
+      v-if="run && run.status === 'error'"
+      type="error" variant="tonal" density="compact" class="mt-3"
+    >Eval failed: {{ run.error }}</v-alert>
+
+    <div v-if="report" class="mt-4">
+      <div class="fc-muted text-caption mb-2">
+        Ran {{ formatTime(report.generated_at) }} ·
+        {{ report.concepts.length }} concept(s) ·
+        neg ratio {{ report.params.neg_ratio }}, {{ report.params.cv_folds }}-fold CV
+      </div>
+
+      <div v-for="c in report.concepts" :key="c.name" class="fc-cc">
+        <div class="fc-cc__head">
+          <span class="fc-cc__name">{{ c.name }}</span>
+          <span v-if="c.skipped" class="fc-muted text-caption">— skipped: {{ c.skipped }}</span>
+          <span v-else class="fc-muted text-caption">
+            {{ c.n_pos }} pos · {{ c.n_neg }} neg<span v-if="c.n_rejected"> ({{ c.n_rejected }} rejected)</span>
+          </span>
+        </div>
+
+        <template v-if="!c.skipped">
+          <table class="fc-metrics">
+            <thead>
+              <tr><th></th><th>AP</th><th>F1</th><th>Prec</th><th>Rec</th></tr>
+            </thead>
+            <tbody>
+              <tr>
+                <td class="fc-metrics__lbl">Head</td>
+                <td class="fc-num fc-win">{{ c.head.ap }}</td>
+                <td class="fc-num">{{ c.head.f1 }}</td>
+                <td class="fc-num">{{ c.head.precision }}</td>
+                <td class="fc-num">{{ c.head.recall }}</td>
+              </tr>
+              <tr>
+                <td class="fc-metrics__lbl fc-muted">Centroid</td>
+                <td class="fc-num fc-muted">{{ c.centroid.ap }}</td>
+                <td class="fc-num fc-muted">{{ c.centroid.f1 }}</td>
+                <td class="fc-num fc-muted">{{ c.centroid.precision }}</td>
+                <td class="fc-num fc-muted">{{ c.centroid.recall }}</td>
+              </tr>
+            </tbody>
+          </table>
+          <div class="text-caption mb-2" :class="apDelta(c) >= 0 ? 'fc-up' : 'fc-down'">
+            Δ AP {{ apDelta(c) >= 0 ? '+' : '' }}{{ apDelta(c).toFixed(3) }}
+            (head − centroid)
+          </div>
+
+          <div v-if="c.curve && c.curve.length" class="fc-curve">
+            <span class="fc-muted text-caption">Learning curve (AP @ N positives):</span>
+            <span v-for="p in c.curve" :key="p.n_pos" class="fc-curve__pt">
+              {{ p.n_pos }}→<strong>{{ p.ap }}</strong>
+            </span>
+          </div>
+
+          <div v-if="c.examples" class="fc-ex">
+            <div class="fc-ex__row">
+              <div class="fc-muted text-caption mb-1">Head would suggest (untagged, high score)</div>
+              <div class="fc-ex__thumbs">
+                <img
+                  v-for="it in c.examples.head_would_suggest" :key="`s${it.id}`"
+                  :src="it.thumbnail_url" class="fc-ex__thumb" loading="lazy"
+                />
+              </div>
+            </div>
+            <div class="fc-ex__row">
+              <div class="fc-muted text-caption mb-1">Head doubts these (your positives, low score)</div>
+              <div class="fc-ex__thumbs">
+                <img
+                  v-for="it in c.examples.head_doubts_positive" :key="`d${it.id}`"
+                  :src="it.thumbnail_url" class="fc-ex__thumb" loading="lazy"
+                />
+              </div>
+            </div>
+          </div>
+        </template>
+      </div>
+    </div>
+  </MaintenanceTile>
+</template>
+
+<script setup>
+import { toast } from '../../utils/toast.js'
+import { computed, onMounted, onUnmounted, ref } from 'vue'
+
+import MaintenanceTile from '../common/MaintenanceTile.vue'
+import { useTagEvalStore } from '../../stores/tagEval.js'
+
+const DEFAULT_CONCEPTS =
+  'glasses, cat, dog, horse, goblin, cum, lactation, fellatio, xray, stomach bulge'
+
+const store = useTagEvalStore()
+const run = ref(null)
+const conceptsText = ref(DEFAULT_CONCEPTS)
+const busy = ref(false)
+let pollTimer = null
+
+const running = computed(() => run.value?.status === 'running')
+const report = computed(() => (run.value?.status === 'ready' ? run.value.report : null))
+const startedAgo = computed(() =>
+  run.value?.started_at ? formatTime(run.value.started_at) : '')
+
+// Rehydrate the persisted run on mount so the report survives navigation — the
+// task runs backend-side regardless; we just reconnect to its row.
+onMounted(async () => {
+  try {
+    const latest = await store.latest()
+    if (latest) {
+      run.value = await store.getRun(latest.id)
+      if (run.value.status === 'running') startPoll(latest.id)
+    }
+  } catch { /* non-fatal — card still works for a fresh run */ }
+})
+onUnmounted(stopPoll)
+
+function startPoll(id) {
+  stopPoll()
+  pollTimer = setInterval(async () => {
+    try {
+      run.value = await store.getRun(id)
+      if (run.value.status !== 'running') stopPoll()
+    } catch (e) {
+      stopPoll()
+      toast({ text: `Eval poll failed: ${e.message}`, type: 'error' })
+    }
+  }, 5000)
+}
+function stopPoll() {
+  if (pollTimer) { clearInterval(pollTimer); pollTimer = null }
+}
+
+async function onStart() {
+  busy.value = true
+  try {
+    const concepts = conceptsText.value.split(',').map(s => s.trim()).filter(Boolean)
+    const res = await store.start({ concepts })
+    run.value = await store.getRun(res.run_id)
+    startPoll(res.run_id)
+  } catch (e) {
+    const msg = e.body?.running_id
+      ? 'An eval is already running.'
+      : e.message
+    toast({ text: `Could not start eval: ${msg}`, type: 'error' })
+  } finally {
+    busy.value = false
+  }
+}
+
+function apDelta(c) { return (c.head?.ap ?? 0) - (c.centroid?.ap ?? 0) }
+function formatTime(iso) {
+  if (!iso) return ''
+  try { return new Date(iso).toLocaleString() } catch { return iso }
+}
+</script>
+
+<style scoped>
+.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
+.fc-cc {
+  padding: 12px 0;
+  border-top: 1px solid rgb(var(--v-theme-surface-light));
+}
+.fc-cc__head { display: flex; align-items: baseline; gap: 8px; margin-bottom: 6px; }
+.fc-cc__name { font-weight: 600; }
+.fc-metrics { width: 100%; max-width: 360px; border-collapse: collapse; font-size: 13px; }
+.fc-metrics th { text-align: right; font-weight: 600; color: rgb(var(--v-theme-on-surface-variant)); padding: 0 8px; }
+.fc-metrics__lbl { text-align: left; }
+.fc-num { text-align: right; font-variant-numeric: tabular-nums; padding: 1px 8px; }
+.fc-win { color: rgb(var(--v-theme-accent)); font-weight: 600; }
+.fc-up { color: rgb(var(--v-theme-success)); }
+.fc-down { color: rgb(var(--v-theme-error)); }
+.fc-curve { margin-bottom: 8px; }
+.fc-curve__pt { margin-left: 10px; font-size: 13px; font-variant-numeric: tabular-nums; }
+.fc-ex__row { margin-top: 6px; }
+.fc-ex__thumbs { display: flex; flex-wrap: wrap; gap: 4px; }
+.fc-ex__thumb {
+  width: 56px; height: 56px; object-fit: cover; border-radius: 4px;
+  background: rgb(var(--v-theme-surface-light));
+}
+</style>
diff --git a/frontend/src/stores/tagEval.js b/frontend/src/stores/tagEval.js
new file mode 100644
index 0000000..044707b
--- /dev/null
+++ b/frontend/src/stores/tagEval.js
@@ -0,0 +1,27 @@
+import { defineStore } from 'pinia'
+
+import { useApi } from '../composables/useApi.js'
+
+// Tag-eval (#1130): trigger + revisit the head-vs-centroid learning-curve eval.
+// The run + full report live server-side (tag_eval_run), so the card rehydrates
+// from getRun() on mount — the report survives navigation.
+export const useTagEvalStore = defineStore('tagEval', () => {
+  const api = useApi()
+
+  async function start(params) {
+    return await api.post('/api/tag-eval', { body: { params } })
+  }
+
+  async function getRun(id) {
+    return await api.get(`/api/tag-eval/${id}`)   // includes the full report
+  }
+
+  // The most recent run (light row, no report) — the card calls getRun() with
+  // its id to pull the persisted report on mount.
+  async function latest() {
+    const body = await api.get('/api/tag-eval', { params: { limit: 1 } })
+    return (body.runs && body.runs[0]) || null
+  }
+
+  return { start, getRun, latest }
+})