bvandeusen/FabledCurator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

Agent sleep mode: empty queue sheds to one downloader, lease poll backs off to 15 min #188

Merged
bvandeusen merged 1 commits from dev into main 2026-07-02 20:37:48 -04:00
Conversation 0 Commits 1 Files Changed 2
+36 -9
2 changed files with 36 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/fc_agent/app.py
+3 -2
View File
@@ -21,7 +21,7 @@ log = logging.getLogger("fc_agent.app")
# Bump on every agent change. The page embeds this and /status reports it; the UI
# warns to reload when they differ — so a stale browser-cached page can't be
# mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.)
VERSION = "2026-07-02.5 · cap-aware autoscaler: downloaders stop growing (and shed) when the bandwidth cap — not concurrency — is the bottleneck"
VERSION = "2026-07-02.6 · sleep mode: an empty queue sheds to one downloader and backs the lease poll off to 15 min"
logbuf.install()
cfg = Config.from_env()
@@ -345,7 +345,8 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
if(document.activeElement!==autochk) autochk.checked=!!s.auto
conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.55:1
conchint.textContent=(s.auto?('auto-tuning downloaders to keep the GPU fed · max '+CAP):('manual downloaders · max '+CAP))
+(s.bw_capped?' · holding at the bandwidth cap (more downloaders would not go faster)':'')
+(s.idle?' · idle — queue empty, lease poll backed off (new work noticed within ~15 min)'
:(s.bw_capped?' · holding at the bandwidth cap (more downloaders would not go faster)':''))
if(document.activeElement!==conc) conc.value=s.concurrency
conc.max=CAP
// Connection pill + queue come only from the /status poll (the Start/Stop POST
agent/fc_agent/worker.py
+33 -7
View File
@@ -50,6 +50,13 @@ from .throttle import TokenBucket
# restart needed.
MAX_BACKOFF_SECONDS = 60.0
# Sleep mode (operator 2026-07-02): an EMPTY queue must cost almost nothing —
# the autoscaler sheds to one downloader and that one backs its lease poll off
# exponentially to this ceiling. New work is therefore noticed within at most
# ~15 minutes (accepted trade-off), after which polling and the pool ramp back
# up on their own.
IDLE_POLL_MAX_SECONDS = 900.0
# A job whose fetch dies transiently this many times IN ONE SESSION stops being
# handed back and is failed instead. Transient handbacks (release) burn no
# attempts on the server, so a poisoned transfer — an original that stalls the
@@ -237,6 +244,7 @@ class Worker:
self._dpm = 0.0
self._net_mb_s = 0.0 # smoothed aggregate download rate (UI readout)
self._bw_capped = False # autoscaler is holding/shedding at the cap (UI)
self._idle = False # last lease came back empty → sleep mode (UI)
self._util_smooth: float | None = None # EWMA GPU util (set by control loop)
# Curator queue snapshot, refreshed by a background poller so the UI
# /status read is instant — never an inline curator HTTP call (which
@@ -597,6 +605,7 @@ class Worker:
"bandwidth_limit_mb_s": round(self.throttle.rate / 1_048_576, 1),
"net_mb_s": round(self._net_mb_s, 1), # observed aggregate rate
"bw_capped": self._bw_capped, # autoscaler holding at the cap (UI hint)
"idle": self._idle, # queue empty → poll backed off (UI hint)
}
def _bump(self, *, processed=0, downloaded=0, errors=0, active=0, transient=0):
@@ -616,6 +625,7 @@ class Worker:
on any exit path it releases whatever it still owns so nothing is stranded
holding a lease."""
backoff = self.cfg.poll_idle_seconds
idle_wait = self.cfg.poll_idle_seconds
while not self._stopped(stop_evt):
try:
with self._timed("lease"):
@@ -630,9 +640,18 @@ class Worker:
backoff = min(backoff * 2, MAX_BACKOFF_SECONDS)
continue
if not jobs:
if stop_evt.wait(self.cfg.poll_idle_seconds):
# Empty queue → sleep mode (operator 2026-07-02): back the lease
# poll off exponentially to IDLE_POLL_MAX_SECONDS instead of
# chattering every 10s all night. The autoscaler reads _idle and
# sheds to ONE polling downloader; work appearing resets both
# within one wake-up (≤15 min latency, operator-accepted).
self._idle = True
if stop_evt.wait(idle_wait):
break
idle_wait = min(idle_wait * 2, IDLE_POLL_MAX_SECONDS)
continue
self._idle = False
idle_wait = self.cfg.poll_idle_seconds
self._hold(j["job_id"] for j in jobs)
owned = [j["job_id"] for j in jobs] # released on any early exit
for job in jobs:
@@ -996,6 +1015,7 @@ class Worker:
con_grew = False
self._util_smooth = None
self._bw_capped = False
self._idle = False
continue
occ = self._buffer.qsize() / BUFFER_MAX
@@ -1044,12 +1064,18 @@ class Worker:
self._apply_downloaders(-1)
con_grew = False
elif occ_ewma < OCC_LOW:
# Buffer starving → downloads are the bottleneck. WHICH kind
# decides the move: a pipe pinned at the bandwidth cap gains
# nothing from more streams (they'd split the same budget and
# stretch per-job latency) — shed toward BW_MIN_DL; with cap
# headroom, concurrency is genuinely short — add a feeder.
if bw_hard and self._dl_target > BW_MIN_DL:
# Buffer starving → WHY decides the move (operator-flagged
# 2026-07-02: the pool scaled to 8 against an EMPTY queue):
# - queue empty (_idle): nothing to download — shed to ONE
# polling downloader; growth would just add idle threads.
# - pinned at the bandwidth cap: more streams split the same
# budget and stretch per-job latency — shed toward BW_MIN_DL.
# - cap headroom + work flowing: concurrency is genuinely
# short — add a feeder.
if self._idle:
if self._dl_target > 1:
self._apply_downloaders(-1)
elif bw_hard and self._dl_target > BW_MIN_DL:
self._apply_downloaders(-1)
elif not bw_soft:
self._apply_downloaders(+1)