Merge pull request 'Agent sleep mode: empty queue sheds to one downloader, lease poll backs off to 15 min' (#188) from dev into main

2026-07-02 20:37:48 -04:00
parent bfba8045e4 b1cfbcc06a
commit e52090a4cf
2 changed files with 36 additions and 9 deletions
@@ -21,7 +21,7 @@ log = logging.getLogger("fc_agent.app")
 # Bump on every agent change. The page embeds this and /status reports it; the UI
 # warns to reload when they differ — so a stale browser-cached page can't be
 # mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.)
-VERSION = "2026-07-02.5 · cap-aware autoscaler: downloaders stop growing (and shed) when the bandwidth cap — not concurrency — is the bottleneck"
+VERSION = "2026-07-02.6 · sleep mode: an empty queue sheds to one downloader and backs the lease poll off to 15 min"
 logbuf.install()
 cfg = Config.from_env()
@@ -345,7 +345,8 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
  if(document.activeElement!==autochk) autochk.checked=!!s.auto
  conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.55:1
  conchint.textContent=(s.auto?('auto-tuning downloaders to keep the GPU fed · max '+CAP):('manual downloaders · max '+CAP))
-   +(s.bw_capped?' · holding at the bandwidth cap (more downloaders would not go faster)':'')
+   +(s.idle?' · idle — queue empty, lease poll backed off (new work noticed within ~15 min)'
     :(s.bw_capped?' · holding at the bandwidth cap (more downloaders would not go faster)':''))
  if(document.activeElement!==conc) conc.value=s.concurrency
  conc.max=CAP
  // Connection pill + queue come only from the /status poll (the Start/Stop POST
@@ -50,6 +50,13 @@ from .throttle import TokenBucket
 # restart needed.
 MAX_BACKOFF_SECONDS = 60.0
 # Sleep mode (operator 2026-07-02): an EMPTY queue must cost almost nothing —
 # the autoscaler sheds to one downloader and that one backs its lease poll off
 # exponentially to this ceiling. New work is therefore noticed within at most
 # ~15 minutes (accepted trade-off), after which polling and the pool ramp back
 # up on their own.
 IDLE_POLL_MAX_SECONDS = 900.0
 # A job whose fetch dies transiently this many times IN ONE SESSION stops being
 # handed back and is failed instead. Transient handbacks (release) burn no
 # attempts on the server, so a poisoned transfer — an original that stalls the
@@ -237,6 +244,7 @@ class Worker:
        self._dpm = 0.0
        self._net_mb_s = 0.0      # smoothed aggregate download rate (UI readout)
        self._bw_capped = False   # autoscaler is holding/shedding at the cap (UI)
        self._idle = False        # last lease came back empty → sleep mode (UI)
        self._util_smooth: float | None = None   # EWMA GPU util (set by control loop)
        # Curator queue snapshot, refreshed by a background poller so the UI
        # /status read is instant — never an inline curator HTTP call (which
@@ -597,6 +605,7 @@ class Worker:
            "bandwidth_limit_mb_s": round(self.throttle.rate / 1_048_576, 1),
            "net_mb_s": round(self._net_mb_s, 1),       # observed aggregate rate
            "bw_capped": self._bw_capped,   # autoscaler holding at the cap (UI hint)
            "idle": self._idle,             # queue empty → poll backed off (UI hint)
        }
    def _bump(self, *, processed=0, downloaded=0, errors=0, active=0, transient=0):
@@ -616,6 +625,7 @@ class Worker:
        on any exit path it releases whatever it still owns so nothing is stranded
        holding a lease."""
        backoff = self.cfg.poll_idle_seconds
        idle_wait = self.cfg.poll_idle_seconds
        while not self._stopped(stop_evt):
            try:
                with self._timed("lease"):
@@ -630,9 +640,18 @@ class Worker:
                backoff = min(backoff * 2, MAX_BACKOFF_SECONDS)
                continue
            if not jobs:
-                if stop_evt.wait(self.cfg.poll_idle_seconds):
+                # Empty queue → sleep mode (operator 2026-07-02): back the lease
                # poll off exponentially to IDLE_POLL_MAX_SECONDS instead of
                # chattering every 10s all night. The autoscaler reads _idle and
                # sheds to ONE polling downloader; work appearing resets both
                # within one wake-up (≤15 min latency, operator-accepted).
                self._idle = True
                if stop_evt.wait(idle_wait):
                    break
                idle_wait = min(idle_wait * 2, IDLE_POLL_MAX_SECONDS)
                continue
            self._idle = False
            idle_wait = self.cfg.poll_idle_seconds
            self._hold(j["job_id"] for j in jobs)
            owned = [j["job_id"] for j in jobs]   # released on any early exit
            for job in jobs:
@@ -996,6 +1015,7 @@ class Worker:
                con_grew = False
                self._util_smooth = None
                self._bw_capped = False
                self._idle = False
                continue
            occ = self._buffer.qsize() / BUFFER_MAX
@@ -1044,12 +1064,18 @@ class Worker:
                self._apply_downloaders(-1)
                con_grew = False
            elif occ_ewma < OCC_LOW:
-                # Buffer starving → downloads are the bottleneck. WHICH kind
+                # Buffer starving → WHY decides the move (operator-flagged
-                # decides the move: a pipe pinned at the bandwidth cap gains
+                # 2026-07-02: the pool scaled to 8 against an EMPTY queue):
-                # nothing from more streams (they'd split the same budget and
+                # - queue empty (_idle): nothing to download — shed to ONE
-                # stretch per-job latency) — shed toward BW_MIN_DL; with cap
+                #   polling downloader; growth would just add idle threads.
-                # headroom, concurrency is genuinely short — add a feeder.
+                # - pinned at the bandwidth cap: more streams split the same
-                if bw_hard and self._dl_target > BW_MIN_DL:
+                #   budget and stretch per-job latency — shed toward BW_MIN_DL.
                # - cap headroom + work flowing: concurrency is genuinely
                #   short — add a feeder.
                if self._idle:
                    if self._dl_target > 1:
                        self._apply_downloaders(-1)
                elif bw_hard and self._dl_target > BW_MIN_DL:
                    self._apply_downloaders(-1)
                elif not bw_soft:
                    self._apply_downloaders(+1)