From b1cfbcc06a8bfbb92722a5e9010ead96c0b0cdb9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 2 Jul 2026 20:32:11 -0400 Subject: [PATCH] =?UTF-8?q?fix(agent):=20sleep=20mode=20=E2=80=94=20an=20e?= =?UTF-8?q?mpty=20queue=20sheds=20to=20one=20downloader=20and=20backs=20th?= =?UTF-8?q?e=20lease=20poll=20off=20to=2015=20min?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator-flagged on the deployed .5 build: the autoscaler grew the pool 1→8 against an EMPTY queue (an empty buffer read as 'GPU starving' regardless of WHY), and every downloader kept polling lease every 10s all night. - New idle signal straight from the lease results: an empty lease sets _idle, any jobs clear it. The occupancy-low branch now distinguishes three cases: queue empty → shed to ONE polling downloader; pinned at the bandwidth cap → shed toward 3; cap headroom + work flowing → grow. - Idle lease polls back off exponentially per downloader to IDLE_POLL_MAX_SECONDS (15 min) and reset the moment work appears — so an idle night costs one HTTP call per 15 min, and new work is noticed within at most ~15 min (operator-accepted trade-off). - UI hint: 'idle — queue empty, lease poll backed off'; /status gains idle. Agent build 2026-07-02.6. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- agent/fc_agent/app.py | 5 +++-- agent/fc_agent/worker.py | 40 +++++++++++++++++++++++++++++++++------- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/agent/fc_agent/app.py b/agent/fc_agent/app.py index f743e84..e7c1a3f 100644 --- a/agent/fc_agent/app.py +++ b/agent/fc_agent/app.py @@ -21,7 +21,7 @@ log = logging.getLogger("fc_agent.app") # Bump on every agent change. The page embeds this and /status reports it; the UI # warns to reload when they differ — so a stale browser-cached page can't be # mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.) -VERSION = "2026-07-02.5 · cap-aware autoscaler: downloaders stop growing (and shed) when the bandwidth cap — not concurrency — is the bottleneck" +VERSION = "2026-07-02.6 · sleep mode: an empty queue sheds to one downloader and backs the lease poll off to 15 min" logbuf.install() cfg = Config.from_env() @@ -345,7 +345,8 @@ _PAGE = """ if(document.activeElement!==autochk) autochk.checked=!!s.auto conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.55:1 conchint.textContent=(s.auto?('auto-tuning downloaders to keep the GPU fed · max '+CAP):('manual downloaders · max '+CAP)) - +(s.bw_capped?' · holding at the bandwidth cap (more downloaders would not go faster)':'') + +(s.idle?' · idle — queue empty, lease poll backed off (new work noticed within ~15 min)' + :(s.bw_capped?' · holding at the bandwidth cap (more downloaders would not go faster)':'')) if(document.activeElement!==conc) conc.value=s.concurrency conc.max=CAP // Connection pill + queue come only from the /status poll (the Start/Stop POST diff --git a/agent/fc_agent/worker.py b/agent/fc_agent/worker.py index 8c65668..9c8db2d 100644 --- a/agent/fc_agent/worker.py +++ b/agent/fc_agent/worker.py @@ -50,6 +50,13 @@ from .throttle import TokenBucket # restart needed. MAX_BACKOFF_SECONDS = 60.0 +# Sleep mode (operator 2026-07-02): an EMPTY queue must cost almost nothing — +# the autoscaler sheds to one downloader and that one backs its lease poll off +# exponentially to this ceiling. New work is therefore noticed within at most +# ~15 minutes (accepted trade-off), after which polling and the pool ramp back +# up on their own. +IDLE_POLL_MAX_SECONDS = 900.0 + # A job whose fetch dies transiently this many times IN ONE SESSION stops being # handed back and is failed instead. Transient handbacks (release) burn no # attempts on the server, so a poisoned transfer — an original that stalls the @@ -237,6 +244,7 @@ class Worker: self._dpm = 0.0 self._net_mb_s = 0.0 # smoothed aggregate download rate (UI readout) self._bw_capped = False # autoscaler is holding/shedding at the cap (UI) + self._idle = False # last lease came back empty → sleep mode (UI) self._util_smooth: float | None = None # EWMA GPU util (set by control loop) # Curator queue snapshot, refreshed by a background poller so the UI # /status read is instant — never an inline curator HTTP call (which @@ -597,6 +605,7 @@ class Worker: "bandwidth_limit_mb_s": round(self.throttle.rate / 1_048_576, 1), "net_mb_s": round(self._net_mb_s, 1), # observed aggregate rate "bw_capped": self._bw_capped, # autoscaler holding at the cap (UI hint) + "idle": self._idle, # queue empty → poll backed off (UI hint) } def _bump(self, *, processed=0, downloaded=0, errors=0, active=0, transient=0): @@ -616,6 +625,7 @@ class Worker: on any exit path it releases whatever it still owns so nothing is stranded holding a lease.""" backoff = self.cfg.poll_idle_seconds + idle_wait = self.cfg.poll_idle_seconds while not self._stopped(stop_evt): try: with self._timed("lease"): @@ -630,9 +640,18 @@ class Worker: backoff = min(backoff * 2, MAX_BACKOFF_SECONDS) continue if not jobs: - if stop_evt.wait(self.cfg.poll_idle_seconds): + # Empty queue → sleep mode (operator 2026-07-02): back the lease + # poll off exponentially to IDLE_POLL_MAX_SECONDS instead of + # chattering every 10s all night. The autoscaler reads _idle and + # sheds to ONE polling downloader; work appearing resets both + # within one wake-up (≤15 min latency, operator-accepted). + self._idle = True + if stop_evt.wait(idle_wait): break + idle_wait = min(idle_wait * 2, IDLE_POLL_MAX_SECONDS) continue + self._idle = False + idle_wait = self.cfg.poll_idle_seconds self._hold(j["job_id"] for j in jobs) owned = [j["job_id"] for j in jobs] # released on any early exit for job in jobs: @@ -996,6 +1015,7 @@ class Worker: con_grew = False self._util_smooth = None self._bw_capped = False + self._idle = False continue occ = self._buffer.qsize() / BUFFER_MAX @@ -1044,12 +1064,18 @@ class Worker: self._apply_downloaders(-1) con_grew = False elif occ_ewma < OCC_LOW: - # Buffer starving → downloads are the bottleneck. WHICH kind - # decides the move: a pipe pinned at the bandwidth cap gains - # nothing from more streams (they'd split the same budget and - # stretch per-job latency) — shed toward BW_MIN_DL; with cap - # headroom, concurrency is genuinely short — add a feeder. - if bw_hard and self._dl_target > BW_MIN_DL: + # Buffer starving → WHY decides the move (operator-flagged + # 2026-07-02: the pool scaled to 8 against an EMPTY queue): + # - queue empty (_idle): nothing to download — shed to ONE + # polling downloader; growth would just add idle threads. + # - pinned at the bandwidth cap: more streams split the same + # budget and stretch per-job latency — shed toward BW_MIN_DL. + # - cap headroom + work flowing: concurrency is genuinely + # short — add a feeder. + if self._idle: + if self._dl_target > 1: + self._apply_downloaders(-1) + elif bw_hard and self._dl_target > BW_MIN_DL: self._apply_downloaders(-1) elif not bw_soft: self._apply_downloaders(+1) -- 2.52.0