feat: cap-aware autoscaler + token-gated whole-instance tag reset (operator feedback)

Autoscaler (agent 2026-07-02.5): the buffer-occupancy signal alone would peg downloaders at DL_MAX while the bandwidth CAP — not concurrency — is the real constraint (8 streams sharing 8 MB/s move no more data than 4). Growth is now gated on the pipe having headroom (net < 85% of cap) and a pipe pinned at the cap (>= 95%) sheds streams down to 3; dead band prevents flapping. The UI hint says 'holding at the bandwidth cap' and /status reports bw_capped, so the behavior is legible without tests that need the ML stack. Reset content tagging: stays a FULL-instance reset (operator's call), but now lives in a fenced 'Danger zone' section on Cleanup and the apply is gated by a preview-derived confirm token (mirrors the Tier-C bulk-delete pattern — stale counts are rejected server-side). Copy no longer claims suggestions repopulate: it says plainly the heads' training examples are deleted and re-tagging starts fresh. Moved out of TagMaintenanceCard into DangerZoneCard. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
2026-07-02 16:14:48 -04:00
parent eaea4308fc
commit 7c19ad91ed
9 changed files with 257 additions and 78 deletions
@@ -21,7 +21,7 @@ log = logging.getLogger("fc_agent.app")
 # Bump on every agent change. The page embeds this and /status reports it; the UI
 # warns to reload when they differ — so a stale browser-cached page can't be
 # mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.)
-VERSION = "2026-07-02.4 · bandwidth governor: aggregate download cap (MB/s dial) so the agent can't saturate the desktop's network"
+VERSION = "2026-07-02.5 · cap-aware autoscaler: downloaders stop growing (and shed) when the bandwidth cap — not concurrency — is the bottleneck"
 logbuf.install()
 cfg = Config.from_env()
@@ -344,7 +344,8 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
  // Auto on → dial reflects the auto-chosen count (read-only); off → manual.
  if(document.activeElement!==autochk) autochk.checked=!!s.auto
  conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.55:1
-  conchint.textContent=s.auto?('auto-tuning downloaders to keep the GPU fed · max '+CAP):('manual downloaders · max '+CAP)
+  conchint.textContent=(s.auto?('auto-tuning downloaders to keep the GPU fed · max '+CAP):('manual downloaders · max '+CAP))
   +(s.bw_capped?' · holding at the bandwidth cap (more downloaders would not go faster)':'')
  if(document.activeElement!==conc) conc.value=s.concurrency
  conc.max=CAP
  // Connection pill + queue come only from the /status poll (the Start/Stop POST
@@ -124,6 +124,17 @@ OCC_LOW = 0.25                # below this = buffer starving → add a downloade
 OCC_HIGH = 0.80               # above this = downloaders outpace the GPU
 UTIL_ALPHA = 0.25             # GPU-util EWMA weight
 UTIL_START = 85               # GPU has headroom below this (gate a 2nd consumer)
 # Bandwidth-cap awareness (operator 2026-07-02): with the aggregate governor in
 # place, the occupancy signal alone would peg downloaders at DL_MAX while the
 # CAP — not concurrency — is the real constraint: 8 streams sharing 8 MB/s move
 # no more data than 4, they just hold more leases + RAM and stretch every job's
 # latency. So growth is gated on the pipe having headroom, and a pipe pinned at
 # the cap sheds streams down to BW_MIN_DL (enough overlap to keep the cap
 # filled through TTFB + decode gaps). The dead band between the two thresholds
 # prevents add/trim flapping.
 BW_ADD_HEADROOM = 0.85        # add a downloader only while net < 85% of the cap
 BW_TRIM_AT = 0.95             # net ≥ 95% of the cap → shed toward BW_MIN_DL
 BW_MIN_DL = 3
 VRAM_HI = 0.90                # memory pressure → shed a consumer
 VRAM_GROW_MAX = 0.82          # don't add a consumer past this VRAM
 TPUT_ALPHA = 0.5              # throughput EWMA weight
@@ -225,6 +236,7 @@ class Worker:
        self._jpm = 0.0
        self._dpm = 0.0
        self._net_mb_s = 0.0      # smoothed aggregate download rate (UI readout)
        self._bw_capped = False   # autoscaler is holding/shedding at the cap (UI)
        self._util_smooth: float | None = None   # EWMA GPU util (set by control loop)
        # Curator queue snapshot, refreshed by a background poller so the UI
        # /status read is instant — never an inline curator HTTP call (which
@@ -584,6 +596,7 @@ class Worker:
            "transient": self.transient,
            "bandwidth_limit_mb_s": round(self.throttle.rate / 1_048_576, 1),
            "net_mb_s": round(self._net_mb_s, 1),       # observed aggregate rate
            "bw_capped": self._bw_capped,   # autoscaler holding at the cap (UI hint)
        }
    def _bump(self, *, processed=0, downloaded=0, errors=0, active=0, transient=0):
@@ -982,10 +995,19 @@ class Worker:
                tick = 0
                con_grew = False
                self._util_smooth = None
                self._bw_capped = False
                continue
            occ = self._buffer.qsize() / BUFFER_MAX
            occ_ewma = _ewma(occ_ewma, occ, OCC_ALPHA)
            # Bandwidth-cap position: compare the observed aggregate (the same
            # EWMA the UI shows) against the governor's cap. `soft` gates
            # growth; `hard` sheds streams (see BW_* rationale above).
            bw_rate = self.throttle.rate
            net_bytes = self._net_mb_s * 1_048_576
            bw_soft = bw_rate > 0 and net_bytes >= BW_ADD_HEADROOM * bw_rate
            bw_hard = bw_rate > 0 and net_bytes >= BW_TRIM_AT * bw_rate
            self._bw_capped = bw_soft
            g = gpumod.read_gpu() or {}
            mt = g.get("mem_total_mb") or 0
            vram = (g.get("mem_used_mb", 0) / mt) if mt else 0.0
@@ -1022,8 +1044,15 @@ class Worker:
                self._apply_downloaders(-1)
                con_grew = False
            elif occ_ewma < OCC_LOW:
-                # Buffer starving → GPU idle waiting on downloads → add a feeder.
+                # Buffer starving → downloads are the bottleneck. WHICH kind
-                self._apply_downloaders(+1)
+                # decides the move: a pipe pinned at the bandwidth cap gains
                # nothing from more streams (they'd split the same budget and
                # stretch per-job latency) — shed toward BW_MIN_DL; with cap
                # headroom, concurrency is genuinely short — add a feeder.
                if bw_hard and self._dl_target > BW_MIN_DL:
                    self._apply_downloaders(-1)
                elif not bw_soft:
                    self._apply_downloaders(+1)
                con_grew = False
            elif occ_ewma > OCC_HIGH:
                # Downloaders outpace the GPU. Prefer helping the GPU (add a 2nd
@@ -1049,7 +1078,9 @@ class Worker:
            if self._dl_target != d0 or self._consumer_target != c0:
                log.info(
                    "autoscale: dl %d→%d · consumers %d→%d "
-                    "(buf %d%% · util~%d%% · %.2f j/s · vram %d%%)",
+                    "(buf %d%% · util~%d%% · %.2f j/s · vram %d%% · "
                    "net %.1f MB/s%s)",
                    d0, self._dl_target, c0, self._consumer_target,
                    round(occ_ewma * 100), round(util_ewma), tput_ewma,
-                    round(vram * 100))
+                    round(vram * 100), self._net_mb_s,
                    " — at cap" if bw_soft else "")
@@ -276,18 +276,48 @@ async def posts_reconcile_duplicates():
    return await _run_dry_run_op(reconcile_duplicate_posts, source_id=source_id)
 def _reset_content_confirm_token(projection: dict) -> str:
    """Stable 8-hex token derived from the live counts (mirrors the Tier-C
    bulk-delete token): it changes whenever the data changes, so the apply can
    only ever run against numbers the operator just previewed."""
    canon = f"reset-content:{projection.get('count')}:{projection.get('applications')}"
    return hashlib.sha256(canon.encode("utf-8")).hexdigest()[:8]
@admin_bp.route("/tags/reset-content", methods=["POST"])
 async def tags_reset_content():
-    """Tier-A: delete ALL general + character tags (the Camie-suggestable
+    """Full-instance reset of the CONTENT vocabulary: deletes ALL general +
-    content vocabulary) so the operator can re-tag from scratch via
+    character tags and their image applications — INCLUDING the examples the
-    auto-suggest. fandom + series tags + series_page ordering are preserved,
+    tagging heads learned from. Suggestions do NOT repopulate on their own
-    and image_prediction rows are untouched so suggestions repopulate.
+    (the Camie predictions that once did are long retired): the operator
-    dry-run preview returns per-kind counts + applications + a sample so the
+    re-tags from scratch and the heads retrain from the new signal. fandom +
-    UI shows exactly what'll go before the operator confirms (dry_run=false).
+    series tags + series_page ordering are preserved.
-    Irreversible except via DB backup restore."""
+
    Deliberately Tier-C-gated despite the Tier-A shape (operator 2026-07-02:
    the full reset stays, but behind extra steps): dry_run returns the
    projection + a `confirm` token derived from the live counts; the apply
    must echo that token back or it is rejected."""
    from ..services.cleanup_service import reset_content_tagging
-    return await _run_dry_run_op(reset_content_tagging)
+    body = await request.get_json(silent=True) or {}
    dry_run = bool(body.get("dry_run", False))
    async with get_session() as session:
        projection = await session.run_sync(
            lambda s: reset_content_tagging(s, dry_run=True)
        )
        token = _reset_content_confirm_token(projection)
        if dry_run:
            projection["confirm"] = token
            return jsonify(projection)
        if str(body.get("confirm", "")) != token:
            return _bad(
                "confirm_mismatch",
                detail="run a fresh preview and echo its confirm token",
            )
        result = await session.run_sync(
            lambda s: reset_content_tagging(s, dry_run=False)
        )
    return jsonify(result)
@admin_bp.route("/tags/normalize", methods=["POST"])
@@ -726,7 +726,10 @@ RESETTABLE_TAG_KINDS = ("general", "character")
 def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict:
    """Count (dry_run) or DELETE every general + character tag so the operator
-    can re-tag from scratch (heads/CCIP repopulate suggestions).
+    can re-tag from scratch. NB: the deleted applications include the tagging
    heads' training positives — suggestions do NOT repopulate on their own; the
    heads retrain from whatever the operator re-tags. (The API route gates the
    live run behind a preview-derived confirm token for exactly this reason.)
    PRESERVED: fandom + series tags and their series_page ordering. CASCADE on
    image_tag / tag_alias / tag_suggestion_rejection clears each deleted tag's
@@ -0,0 +1,122 @@
 <template>
  <MaintenanceTile
    icon="mdi-nuke"
    title="Reset content tagging (whole instance)"
    blurb="Delete ALL general/character tags and their applications — a start-over. Requires a confirmation code."
    destructive
  >
    <p class="text-body-2 mb-2">
      Deletes every <code>general</code> and <code>character</code> tag and
      removes them from every image — <strong>including the examples the
      tagging heads learned from</strong>. Suggestions will <strong>not</strong>
      repopulate on their own: you re-tag from scratch, and the heads retrain
      from your new tags as they accumulate. Fandoms and series (with their
      page order) are kept.
    </p>
    <v-alert type="error" variant="tonal" density="compact" class="mb-3">
      Irreversible — no undo except restoring a DB backup
      (Settings → Maintenance → Backup). Back one up first.
    </v-alert>
    <v-btn
      color="accent" variant="flat" rounded="pill"
      prepend-icon="mdi-magnify"
      :loading="loadingPreview"
      class="mb-3"
      @click="onPreview"
    >Preview content-tag reset</v-btn>
    <div v-if="preview">
      <p class="text-body-2 mb-2">
        <strong>{{ preview.count }}</strong> content tag(s)
        <span v-for="(n, k) in preview.by_kind" :key="k" class="fc-muted">
          ({{ k }}: {{ n }})&nbsp;
        </span>
        across <strong>{{ preview.applications }}</strong> image
        application(s).
      </p>
      <SampleNameGrid
        v-if="preview.sample_names?.length"
        :names="preview.sample_names" class="mb-3"
      />
      <template v-if="preview.count">
        <p class="text-body-2 mb-2">
          To arm the reset, type the confirmation code
          <code class="fc-code">{{ preview.confirm }}</code> below.
        </p>
        <div class="d-flex align-center mb-1" style="gap: 12px">
          <v-text-field
            v-model="typed" density="compact" hide-details variant="outlined"
            label="Confirmation code" style="max-width: 200px"
            autocomplete="off" spellcheck="false"
          />
          <v-btn
            color="error" variant="flat" rounded="pill"
            prepend-icon="mdi-delete-alert"
            :disabled="typed !== preview.confirm"
            :loading="committing"
            @click="onCommit"
          >Delete {{ preview.count }} tag(s) +
            {{ preview.applications }} application(s)</v-btn>
        </div>
        <p class="fc-muted text-caption mb-0">
          The code is derived from the counts above — if tagging changes
          between preview and apply, the server rejects the stale code.
        </p>
      </template>
    </div>
  </MaintenanceTile>
 </template>
 <script setup>
 import { ref } from 'vue'
 import { toast } from '../../utils/toast.js'
 import MaintenanceTile from '../common/MaintenanceTile.vue'
 import SampleNameGrid from '../common/SampleNameGrid.vue'
 import { useAdminStore } from '../../stores/admin.js'
 const store = useAdminStore()
 const preview = ref(null)
 const loadingPreview = ref(false)
 const committing = ref(false)
 const typed = ref('')
 async function onPreview() {
  loadingPreview.value = true
  typed.value = ''
  try {
    preview.value = await store.resetContentTagging({ dryRun: true })
  } catch (e) {
    toast({ text: `Preview failed: ${e.message}`, type: 'error' })
  } finally {
    loadingPreview.value = false
  }
 }
 async function onCommit() {
  committing.value = true
  try {
    const res = await store.resetContentTagging({
      dryRun: false, confirm: typed.value,
    })
    toast({ text: `Deleted ${res.deleted} content tag(s) — re-tagging starts fresh`, type: 'success' })
    preview.value = null
    typed.value = ''
  } catch (e) {
    toast({ text: `Reset rejected: ${e.message}`, type: 'error' })
  } finally {
    committing.value = false
  }
 }
 </script>
 <style scoped>
 .fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
 .fc-code {
  background: rgb(var(--v-theme-surface-light));
  border-radius: 4px; padding: 2px 8px;
  font-family: 'JetBrains Mono', monospace; font-weight: 700;
  letter-spacing: 0.06em;
 }
 </style>
@@ -42,56 +42,6 @@
    </div>
  </MaintenanceTile>
  <MaintenanceTile
    icon="mdi-tag-multiple"
    title="Reset content tagging"
    blurb="Delete all general/character tags to re-tag from scratch."
    destructive
  >
    <p class="text-body-2 mb-2">
      Deletes every <code>general</code> and <code>character</code> tag and
      removes them from every image, so you can re-tag from scratch with the
      auto-suggest. <strong>Fandoms and series (with their page order) are
      kept</strong>, and each image's saved predictions are untouched — open
      an image and its suggestions reappear.
    </p>
    <v-alert type="warning" variant="tonal" density="compact" class="mb-3">
      Irreversible — there's no undo except restoring a DB backup.
      Back one up first (Settings → Maintenance → Backup).
    </v-alert>
    <v-btn
      color="accent" variant="flat" rounded="pill"
      prepend-icon="mdi-magnify"
      :loading="loadingResetPreview"
      class="mb-3"
      @click="onResetPreview"
    >Preview content-tag reset</v-btn>
    <div v-if="resetPreview">
      <p class="text-body-2 mb-2">
        <strong>{{ resetPreview.count }}</strong> content tag(s)
        <span v-for="(n, k) in resetPreview.by_kind" :key="k" class="fc-muted">
          ({{ k }}: {{ n }})&nbsp;
        </span>
        across <strong>{{ resetPreview.applications }}</strong> image
        application(s).
      </p>
      <SampleNameGrid
        v-if="resetPreview.sample_names?.length"
        :names="resetPreview.sample_names" class="mb-3"
      />
      <v-btn
        color="error" variant="flat" rounded="pill"
        prepend-icon="mdi-delete-alert"
        :disabled="!resetPreview.count"
        :loading="resetCommitting"
        @click="onResetCommit"
      >Delete {{ resetPreview.count }} content tag(s) +
        {{ resetPreview.applications }} application(s)</v-btn>
    </div>
  </MaintenanceTile>
  <MaintenanceTile
    icon="mdi-format-letter-case"
    title="Standardize tag casing"
@@ -169,16 +119,6 @@ const {
  emptyPreview: (r) => ({ count: 0, sample_names: r.sample_names || [] }),
 })
 // Reset content tagging (general + character).
 const {
  previewData: resetPreview, previewing: loadingResetPreview,
  committing: resetCommitting, runPreview: onResetPreview, runCommit: onResetCommit,
 } = usePreviewCommit({
  preview: () => store.resetContentTagging({ dryRun: true }),
  commit: () => store.resetContentTagging({ dryRun: false }),
  emptyPreview: { count: 0, by_kind: {}, applications: 0, sample_names: [] },
 })
 // Standardize casing. The apply DISPATCHES a self-resuming background task (no
 // poll-until-done — that would falsely report complete after the first chunk),
 // so there's no emptyPreview: leave the projection up; a truthy normResult means
@@ -101,8 +101,10 @@ export const useAdminStore = defineStore('admin', () => {
    })
  }
-  // Destructive: deletes ALL general + character tags so the operator can
+  // Destructive whole-instance reset: deletes ALL general + character tags AND
-  // re-tag from scratch via auto-suggest. fandom + series preserved.
+  // their applications (the heads' training data included) — fandom + series
  // preserved. dry-run returns a `confirm` token; the apply must pass it back
  // ({ dryRun: false, confirm }) or the server rejects it.
  function resetContentTagging(opts = {}) {
    return _dryRunPost('/api/admin/tags/reset-content', opts)
  }
@@ -39,6 +39,17 @@
        <TagMaintenanceCard />
      </div>
    </section>
    <section class="fc-section fc-danger">
      <h3 class="fc-section__title fc-danger__title">Danger zone</h3>
      <p class="fc-section__hint">
        Whole-instance resets. Each needs a fresh preview and a typed
        confirmation code.
      </p>
      <div class="fc-tile-grid">
        <DangerZoneCard />
      </div>
    </section>
  </div>
 </template>
@@ -50,6 +61,7 @@ import PostMaintenanceCard from '../components/settings/PostMaintenanceCard.vue'
 import VideoDedupCard from '../components/settings/VideoDedupCard.vue'
 import GatedPurgeCard from '../components/settings/GatedPurgeCard.vue'
 import TagMaintenanceCard from '../components/settings/TagMaintenanceCard.vue'
 import DangerZoneCard from '../components/settings/DangerZoneCard.vue'
 </script>
 <style scoped>
@@ -75,4 +87,12 @@ import TagMaintenanceCard from '../components/settings/TagMaintenanceCard.vue'
  gap: 14px;
  align-items: start;
 }
 /* Danger zone: visually fenced off from routine cleanup so the reset can't
   be mistaken for maintenance. */
 .fc-danger {
  border-top: 1px solid rgba(var(--v-theme-error), 0.45);
  padding-top: 18px;
  margin-top: 36px;
 }
 .fc-danger__title { color: rgb(var(--v-theme-error)); }
 </style>
@@ -562,7 +562,7 @@ async def test_trigger_normalize_live_queues(client, monkeypatch):
@pytest.mark.asyncio
-async def test_reset_content_tagging_dry_run_returns_counts(client, db):
+async def test_reset_content_tagging_dry_run_returns_counts_and_token(client, db):
    db.add_all([
        Tag(name="solo", kind=TagKind.general),
        Tag(name="naruto", kind=TagKind.character),
@@ -579,3 +579,33 @@ async def test_reset_content_tagging_dry_run_returns_counts(client, db):
    assert body["by_kind"] == {"general": 1, "character": 1}
    # dry-run leaves the rows in place — fandom + series untouched too.
    assert "deleted" not in body
    # The preview arms the apply: an 8-hex confirm token over the live counts.
    assert len(body["confirm"]) == 8
@pytest.mark.asyncio
 async def test_reset_content_tagging_apply_requires_confirm_token(client, db):
    from sqlalchemy import func, select
    db.add(Tag(name="solo", kind=TagKind.general))
    await db.commit()
    # No token / wrong token → rejected, nothing deleted.
    for bad in ({"dry_run": False}, {"dry_run": False, "confirm": "deadbeef"}):
        resp = await client.post("/api/admin/tags/reset-content", json=bad)
        assert resp.status_code == 400
    remaining = (await db.execute(
        select(func.count()).select_from(Tag)
    )).scalar_one()
    assert remaining == 1
    # The token from a fresh preview unlocks the apply.
    token = (await (await client.post(
        "/api/admin/tags/reset-content", json={"dry_run": True}
    )).get_json())["confirm"]
    resp = await client.post(
        "/api/admin/tags/reset-content",
        json={"dry_run": False, "confirm": token},
    )
    assert resp.status_code == 200
    assert (await resp.get_json())["deleted"] == 1