Compare commits
2 Commits
bfba8045e4
...
e52090a4cf
| Author | SHA1 | Date | |
|---|---|---|---|
 bvandeusen
|
e52090a4cf | ||
|
bvandeusen
|
b1cfbcc06a |
@@ -21,7 +21,7 @@ log = logging.getLogger("fc_agent.app")
|
||||
# Bump on every agent change. The page embeds this and /status reports it; the UI
|
||||
# warns to reload when they differ — so a stale browser-cached page can't be
|
||||
# mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.)
|
||||
VERSION = "2026-07-02.5 · cap-aware autoscaler: downloaders stop growing (and shed) when the bandwidth cap — not concurrency — is the bottleneck"
|
||||
VERSION = "2026-07-02.6 · sleep mode: an empty queue sheds to one downloader and backs the lease poll off to 15 min"
|
||||
|
||||
logbuf.install()
|
||||
cfg = Config.from_env()
|
||||
@@ -345,7 +345,8 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
if(document.activeElement!==autochk) autochk.checked=!!s.auto
|
||||
conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.55:1
|
||||
conchint.textContent=(s.auto?('auto-tuning downloaders to keep the GPU fed · max '+CAP):('manual downloaders · max '+CAP))
|
||||
+(s.bw_capped?' · holding at the bandwidth cap (more downloaders would not go faster)':'')
|
||||
+(s.idle?' · idle — queue empty, lease poll backed off (new work noticed within ~15 min)'
|
||||
:(s.bw_capped?' · holding at the bandwidth cap (more downloaders would not go faster)':''))
|
||||
if(document.activeElement!==conc) conc.value=s.concurrency
|
||||
conc.max=CAP
|
||||
// Connection pill + queue come only from the /status poll (the Start/Stop POST
|
||||
|
||||
@@ -50,6 +50,13 @@ from .throttle import TokenBucket
|
||||
# restart needed.
|
||||
MAX_BACKOFF_SECONDS = 60.0
|
||||
|
||||
# Sleep mode (operator 2026-07-02): an EMPTY queue must cost almost nothing —
|
||||
# the autoscaler sheds to one downloader and that one backs its lease poll off
|
||||
# exponentially to this ceiling. New work is therefore noticed within at most
|
||||
# ~15 minutes (accepted trade-off), after which polling and the pool ramp back
|
||||
# up on their own.
|
||||
IDLE_POLL_MAX_SECONDS = 900.0
|
||||
|
||||
# A job whose fetch dies transiently this many times IN ONE SESSION stops being
|
||||
# handed back and is failed instead. Transient handbacks (release) burn no
|
||||
# attempts on the server, so a poisoned transfer — an original that stalls the
|
||||
@@ -237,6 +244,7 @@ class Worker:
|
||||
self._dpm = 0.0
|
||||
self._net_mb_s = 0.0 # smoothed aggregate download rate (UI readout)
|
||||
self._bw_capped = False # autoscaler is holding/shedding at the cap (UI)
|
||||
self._idle = False # last lease came back empty → sleep mode (UI)
|
||||
self._util_smooth: float | None = None # EWMA GPU util (set by control loop)
|
||||
# Curator queue snapshot, refreshed by a background poller so the UI
|
||||
# /status read is instant — never an inline curator HTTP call (which
|
||||
@@ -597,6 +605,7 @@ class Worker:
|
||||
"bandwidth_limit_mb_s": round(self.throttle.rate / 1_048_576, 1),
|
||||
"net_mb_s": round(self._net_mb_s, 1), # observed aggregate rate
|
||||
"bw_capped": self._bw_capped, # autoscaler holding at the cap (UI hint)
|
||||
"idle": self._idle, # queue empty → poll backed off (UI hint)
|
||||
}
|
||||
|
||||
def _bump(self, *, processed=0, downloaded=0, errors=0, active=0, transient=0):
|
||||
@@ -616,6 +625,7 @@ class Worker:
|
||||
on any exit path it releases whatever it still owns so nothing is stranded
|
||||
holding a lease."""
|
||||
backoff = self.cfg.poll_idle_seconds
|
||||
idle_wait = self.cfg.poll_idle_seconds
|
||||
while not self._stopped(stop_evt):
|
||||
try:
|
||||
with self._timed("lease"):
|
||||
@@ -630,9 +640,18 @@ class Worker:
|
||||
backoff = min(backoff * 2, MAX_BACKOFF_SECONDS)
|
||||
continue
|
||||
if not jobs:
|
||||
if stop_evt.wait(self.cfg.poll_idle_seconds):
|
||||
# Empty queue → sleep mode (operator 2026-07-02): back the lease
|
||||
# poll off exponentially to IDLE_POLL_MAX_SECONDS instead of
|
||||
# chattering every 10s all night. The autoscaler reads _idle and
|
||||
# sheds to ONE polling downloader; work appearing resets both
|
||||
# within one wake-up (≤15 min latency, operator-accepted).
|
||||
self._idle = True
|
||||
if stop_evt.wait(idle_wait):
|
||||
break
|
||||
idle_wait = min(idle_wait * 2, IDLE_POLL_MAX_SECONDS)
|
||||
continue
|
||||
self._idle = False
|
||||
idle_wait = self.cfg.poll_idle_seconds
|
||||
self._hold(j["job_id"] for j in jobs)
|
||||
owned = [j["job_id"] for j in jobs] # released on any early exit
|
||||
for job in jobs:
|
||||
@@ -996,6 +1015,7 @@ class Worker:
|
||||
con_grew = False
|
||||
self._util_smooth = None
|
||||
self._bw_capped = False
|
||||
self._idle = False
|
||||
continue
|
||||
|
||||
occ = self._buffer.qsize() / BUFFER_MAX
|
||||
@@ -1044,12 +1064,18 @@ class Worker:
|
||||
self._apply_downloaders(-1)
|
||||
con_grew = False
|
||||
elif occ_ewma < OCC_LOW:
|
||||
# Buffer starving → downloads are the bottleneck. WHICH kind
|
||||
# decides the move: a pipe pinned at the bandwidth cap gains
|
||||
# nothing from more streams (they'd split the same budget and
|
||||
# stretch per-job latency) — shed toward BW_MIN_DL; with cap
|
||||
# headroom, concurrency is genuinely short — add a feeder.
|
||||
if bw_hard and self._dl_target > BW_MIN_DL:
|
||||
# Buffer starving → WHY decides the move (operator-flagged
|
||||
# 2026-07-02: the pool scaled to 8 against an EMPTY queue):
|
||||
# - queue empty (_idle): nothing to download — shed to ONE
|
||||
# polling downloader; growth would just add idle threads.
|
||||
# - pinned at the bandwidth cap: more streams split the same
|
||||
# budget and stretch per-job latency — shed toward BW_MIN_DL.
|
||||
# - cap headroom + work flowing: concurrency is genuinely
|
||||
# short — add a feeder.
|
||||
if self._idle:
|
||||
if self._dl_target > 1:
|
||||
self._apply_downloaders(-1)
|
||||
elif bw_hard and self._dl_target > BW_MIN_DL:
|
||||
self._apply_downloaders(-1)
|
||||
elif not bw_soft:
|
||||
self._apply_downloaders(+1)
|
||||
|
||||
Reference in New Issue
Block a user