feat: cap-aware autoscaler + token-gated whole-instance tag reset (operator feedback)
Autoscaler (agent 2026-07-02.5): the buffer-occupancy signal alone would peg downloaders at DL_MAX while the bandwidth CAP — not concurrency — is the real constraint (8 streams sharing 8 MB/s move no more data than 4). Growth is now gated on the pipe having headroom (net < 85% of cap) and a pipe pinned at the cap (>= 95%) sheds streams down to 3; dead band prevents flapping. The UI hint says 'holding at the bandwidth cap' and /status reports bw_capped, so the behavior is legible without tests that need the ML stack. Reset content tagging: stays a FULL-instance reset (operator's call), but now lives in a fenced 'Danger zone' section on Cleanup and the apply is gated by a preview-derived confirm token (mirrors the Tier-C bulk-delete pattern — stale counts are rejected server-side). Copy no longer claims suggestions repopulate: it says plainly the heads' training examples are deleted and re-tagging starts fresh. Moved out of TagMaintenanceCard into DangerZoneCard. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
@@ -21,7 +21,7 @@ log = logging.getLogger("fc_agent.app")
|
||||
# Bump on every agent change. The page embeds this and /status reports it; the UI
|
||||
# warns to reload when they differ — so a stale browser-cached page can't be
|
||||
# mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.)
|
||||
VERSION = "2026-07-02.4 · bandwidth governor: aggregate download cap (MB/s dial) so the agent can't saturate the desktop's network"
|
||||
VERSION = "2026-07-02.5 · cap-aware autoscaler: downloaders stop growing (and shed) when the bandwidth cap — not concurrency — is the bottleneck"
|
||||
|
||||
logbuf.install()
|
||||
cfg = Config.from_env()
|
||||
@@ -344,7 +344,8 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
// Auto on → dial reflects the auto-chosen count (read-only); off → manual.
|
||||
if(document.activeElement!==autochk) autochk.checked=!!s.auto
|
||||
conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.55:1
|
||||
conchint.textContent=s.auto?('auto-tuning downloaders to keep the GPU fed · max '+CAP):('manual downloaders · max '+CAP)
|
||||
conchint.textContent=(s.auto?('auto-tuning downloaders to keep the GPU fed · max '+CAP):('manual downloaders · max '+CAP))
|
||||
+(s.bw_capped?' · holding at the bandwidth cap (more downloaders would not go faster)':'')
|
||||
if(document.activeElement!==conc) conc.value=s.concurrency
|
||||
conc.max=CAP
|
||||
// Connection pill + queue come only from the /status poll (the Start/Stop POST
|
||||
|
||||
@@ -124,6 +124,17 @@ OCC_LOW = 0.25 # below this = buffer starving → add a downloade
|
||||
OCC_HIGH = 0.80 # above this = downloaders outpace the GPU
|
||||
UTIL_ALPHA = 0.25 # GPU-util EWMA weight
|
||||
UTIL_START = 85 # GPU has headroom below this (gate a 2nd consumer)
|
||||
# Bandwidth-cap awareness (operator 2026-07-02): with the aggregate governor in
|
||||
# place, the occupancy signal alone would peg downloaders at DL_MAX while the
|
||||
# CAP — not concurrency — is the real constraint: 8 streams sharing 8 MB/s move
|
||||
# no more data than 4, they just hold more leases + RAM and stretch every job's
|
||||
# latency. So growth is gated on the pipe having headroom, and a pipe pinned at
|
||||
# the cap sheds streams down to BW_MIN_DL (enough overlap to keep the cap
|
||||
# filled through TTFB + decode gaps). The dead band between the two thresholds
|
||||
# prevents add/trim flapping.
|
||||
BW_ADD_HEADROOM = 0.85 # add a downloader only while net < 85% of the cap
|
||||
BW_TRIM_AT = 0.95 # net ≥ 95% of the cap → shed toward BW_MIN_DL
|
||||
BW_MIN_DL = 3
|
||||
VRAM_HI = 0.90 # memory pressure → shed a consumer
|
||||
VRAM_GROW_MAX = 0.82 # don't add a consumer past this VRAM
|
||||
TPUT_ALPHA = 0.5 # throughput EWMA weight
|
||||
@@ -225,6 +236,7 @@ class Worker:
|
||||
self._jpm = 0.0
|
||||
self._dpm = 0.0
|
||||
self._net_mb_s = 0.0 # smoothed aggregate download rate (UI readout)
|
||||
self._bw_capped = False # autoscaler is holding/shedding at the cap (UI)
|
||||
self._util_smooth: float | None = None # EWMA GPU util (set by control loop)
|
||||
# Curator queue snapshot, refreshed by a background poller so the UI
|
||||
# /status read is instant — never an inline curator HTTP call (which
|
||||
@@ -584,6 +596,7 @@ class Worker:
|
||||
"transient": self.transient,
|
||||
"bandwidth_limit_mb_s": round(self.throttle.rate / 1_048_576, 1),
|
||||
"net_mb_s": round(self._net_mb_s, 1), # observed aggregate rate
|
||||
"bw_capped": self._bw_capped, # autoscaler holding at the cap (UI hint)
|
||||
}
|
||||
|
||||
def _bump(self, *, processed=0, downloaded=0, errors=0, active=0, transient=0):
|
||||
@@ -982,10 +995,19 @@ class Worker:
|
||||
tick = 0
|
||||
con_grew = False
|
||||
self._util_smooth = None
|
||||
self._bw_capped = False
|
||||
continue
|
||||
|
||||
occ = self._buffer.qsize() / BUFFER_MAX
|
||||
occ_ewma = _ewma(occ_ewma, occ, OCC_ALPHA)
|
||||
# Bandwidth-cap position: compare the observed aggregate (the same
|
||||
# EWMA the UI shows) against the governor's cap. `soft` gates
|
||||
# growth; `hard` sheds streams (see BW_* rationale above).
|
||||
bw_rate = self.throttle.rate
|
||||
net_bytes = self._net_mb_s * 1_048_576
|
||||
bw_soft = bw_rate > 0 and net_bytes >= BW_ADD_HEADROOM * bw_rate
|
||||
bw_hard = bw_rate > 0 and net_bytes >= BW_TRIM_AT * bw_rate
|
||||
self._bw_capped = bw_soft
|
||||
g = gpumod.read_gpu() or {}
|
||||
mt = g.get("mem_total_mb") or 0
|
||||
vram = (g.get("mem_used_mb", 0) / mt) if mt else 0.0
|
||||
@@ -1022,8 +1044,15 @@ class Worker:
|
||||
self._apply_downloaders(-1)
|
||||
con_grew = False
|
||||
elif occ_ewma < OCC_LOW:
|
||||
# Buffer starving → GPU idle waiting on downloads → add a feeder.
|
||||
self._apply_downloaders(+1)
|
||||
# Buffer starving → downloads are the bottleneck. WHICH kind
|
||||
# decides the move: a pipe pinned at the bandwidth cap gains
|
||||
# nothing from more streams (they'd split the same budget and
|
||||
# stretch per-job latency) — shed toward BW_MIN_DL; with cap
|
||||
# headroom, concurrency is genuinely short — add a feeder.
|
||||
if bw_hard and self._dl_target > BW_MIN_DL:
|
||||
self._apply_downloaders(-1)
|
||||
elif not bw_soft:
|
||||
self._apply_downloaders(+1)
|
||||
con_grew = False
|
||||
elif occ_ewma > OCC_HIGH:
|
||||
# Downloaders outpace the GPU. Prefer helping the GPU (add a 2nd
|
||||
@@ -1049,7 +1078,9 @@ class Worker:
|
||||
if self._dl_target != d0 or self._consumer_target != c0:
|
||||
log.info(
|
||||
"autoscale: dl %d→%d · consumers %d→%d "
|
||||
"(buf %d%% · util~%d%% · %.2f j/s · vram %d%%)",
|
||||
"(buf %d%% · util~%d%% · %.2f j/s · vram %d%% · "
|
||||
"net %.1f MB/s%s)",
|
||||
d0, self._dl_target, c0, self._consumer_target,
|
||||
round(occ_ewma * 100), round(util_ewma), tput_ewma,
|
||||
round(vram * 100))
|
||||
round(vram * 100), self._net_mb_s,
|
||||
" — at cap" if bw_soft else "")
|
||||
|
||||
Reference in New Issue
Block a user