From 7e74fa767c41f3f3068f12aea80ec53cb2b55fa4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 30 Jun 2026 22:32:51 -0400 Subject: [PATCH 1/2] =?UTF-8?q?fix(agent):=20load=20huge=20images=20?= =?UTF-8?q?=E2=80=94=20disable=20PIL=20decompression-bomb=20guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trusted local library, not an upload surface, so a legitimately large image (90–95M px, operator-flagged) must load. PIL only WARNS at the 89M-px default but RAISES DecompressionBombError at ~179M px, which would fail those jobs. Set Image.MAX_IMAGE_PIXELS = None. (The agent works off individual extracted files — curator's archive_extractor unpacks zip/cbz/rar/7z at import — so this is about big single images, not archives.) Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa --- agent/fc_agent/app.py | 2 +- agent/fc_agent/media.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/agent/fc_agent/app.py b/agent/fc_agent/app.py index 1f001ca..5505e56 100644 --- a/agent/fc_agent/app.py +++ b/agent/fc_agent/app.py @@ -15,7 +15,7 @@ from .worker import Worker # Bump on every agent change. The page embeds this and /status reports it; the UI # warns to reload when they differ — so a stale browser-cached page can't be # mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.) -VERSION = "2026-06-30.7 · ubuntu24.04-py312-cuda12.9" +VERSION = "2026-06-30.8 · ubuntu24.04-py312 · big-images" logbuf.install() cfg = Config.from_env() diff --git a/agent/fc_agent/media.py b/agent/fc_agent/media.py index 24a9f6e..29cbc24 100644 --- a/agent/fc_agent/media.py +++ b/agent/fc_agent/media.py @@ -13,6 +13,13 @@ from PIL import Image, ImageFile # and would otherwise fail the job 3× then error (operator-flagged 2026-06-30). ImageFile.LOAD_TRUNCATED_IMAGES = True +# Disable PIL's decompression-bomb guard: this is a TRUSTED local library, not an +# untrusted upload surface, so a legitimately huge image (high-res scans/prints, +# 90M+ pixels) must load. The default 89M-pixel limit only WARNS, but PIL raises +# DecompressionBombError at 2× (~179M px) — which would fail those jobs outright +# (operator-flagged 2026-06-30, images of 90–95M px). +Image.MAX_IMAGE_PIXELS = None + def is_video(mime: str) -> bool: return bool(mime) and (mime.startswith("video/") or mime in {"image/gif"}) -- 2.52.0 From c587ac667c20aa271990db48cef8c42c41ffcd41 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 30 Jun 2026 22:42:50 -0400 Subject: [PATCH 2/2] fix(agent): cap figures + global region cap + reset active on stop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three safety/robustness fixes from the operator's run logs: - Cap figures per frame (MAX_FIGURES, default 8) like components/panels already are. Uncapped, a huge/busy image yielded hundreds of figure boxes → hundreds of per-figure CCIP calls + crops → a 38s job AND a submit too big to accept (image 81602 looped on 413). This is the acute fix. - Global per-JOB backstop (MAX_REGIONS, default 128): if total regions still exceed the cap (long video), keep the highest-scoring and log the drop, so a submit body can never blow past curator's limit. - Stale "active" meter: stop() now resets _active to 0 (no slots remain, so the meter must read 0 at once), and _bump clamps at 0 so a slot finishing after the reset can't drive it negative. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa --- agent/fc_agent/app.py | 2 +- agent/fc_agent/config.py | 4 ++++ agent/fc_agent/detectors.py | 7 +++++-- agent/fc_agent/worker.py | 17 ++++++++++++++++- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/agent/fc_agent/app.py b/agent/fc_agent/app.py index 5505e56..dda9591 100644 --- a/agent/fc_agent/app.py +++ b/agent/fc_agent/app.py @@ -15,7 +15,7 @@ from .worker import Worker # Bump on every agent change. The page embeds this and /status reports it; the UI # warns to reload when they differ — so a stale browser-cached page can't be # mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.) -VERSION = "2026-06-30.8 · ubuntu24.04-py312 · big-images" +VERSION = "2026-06-30.9 · region-caps + active-reset" logbuf.install() cfg = Config.from_env() diff --git a/agent/fc_agent/config.py b/agent/fc_agent/config.py index 42a7c75..101522e 100644 --- a/agent/fc_agent/config.py +++ b/agent/fc_agent/config.py @@ -34,6 +34,8 @@ class Config: panel_conf: float max_components: int # cap anatomy component crops per frame max_panels: int # cap panel crops per frame + max_figures: int # cap figure boxes per frame (each = a CCIP call + crop) + max_regions: int # hard cap on total regions per JOB (submit-size backstop) @classmethod def from_env(cls) -> Config: @@ -58,4 +60,6 @@ class Config: panel_conf=float(os.environ.get("PANEL_CONF", "0.30")), max_components=int(os.environ.get("MAX_COMPONENTS", "8")), max_panels=int(os.environ.get("MAX_PANELS", "8")), + max_figures=int(os.environ.get("MAX_FIGURES", "8")), + max_regions=int(os.environ.get("MAX_REGIONS", "128")), ) diff --git a/agent/fc_agent/detectors.py b/agent/fc_agent/detectors.py index f91957f..3338850 100644 --- a/agent/fc_agent/detectors.py +++ b/agent/fc_agent/detectors.py @@ -144,11 +144,14 @@ class Proposers: def figures(self, image, base_boxes): """Merge imgutils person boxes (base_boxes: [(bbox, score)]) with the - general COCO person detector → NMS'd figure boxes [(bbox, score, label)].""" + general COCO person detector → NMS'd figure boxes [(bbox, score, label)], + capped to the highest-scoring max_figures. Uncapped, a busy/huge image + (many characters) yields hundreds of boxes → hundreds of per-figure CCIP + calls + crops → a 30s+ job and an oversized submit (operator-flagged).""" boxes = [(bb, sc if sc is not None else 1.0, "person") for bb, sc in base_boxes] if self._person is not None: boxes += self._person.detect(image) - return nms_merge(boxes) + return nms_merge(boxes)[: self.cfg.max_figures] # nms_merge is score-desc def components(self, image): if self._anatomy is None: diff --git a/agent/fc_agent/worker.py b/agent/fc_agent/worker.py index f09b254..2016ac7 100644 --- a/agent/fc_agent/worker.py +++ b/agent/fc_agent/worker.py @@ -210,6 +210,8 @@ class Worker: self._ctrl_stop.set() with self._lock: slots, self._slots = self._slots, [] + self._active = 0 # no slots left → the meter reads 0 at once; any + # lagging decrement is clamped (see _bump) for s in slots: s.stop.set() # each slot releases its inflight on exit @@ -265,7 +267,9 @@ class Worker: self.processed += processed self.errors += errors self.transient += transient - self._active += active + # Clamp at 0: a Stop resets _active to 0, so a slot that was mid-image + # decrements afterwards — that must not drive the meter negative. + self._active = max(0, self._active + active) # --- per-slot loop ----------------------------------------------------- def _loop(self, slot: _Slot): @@ -556,6 +560,17 @@ class Worker: tmpl["embedding_version"] = embed_version regions.append(tmpl) self._record("gpu", time.monotonic() - _t_gpu) + # Backstop: never submit an unbounded pile of regions (a pathological + # image / long video). Keep the highest-scoring max_regions so the + # POST body stays sane — curator rejects an oversized one with 413 + # (operator-flagged: image 81602 looped on 413). + if len(regions) > self.cfg.max_regions: + regions.sort(key=lambda r: r.get("score", 0.0) or 0.0, reverse=True) + dropped = len(regions) - self.cfg.max_regions + regions = regions[: self.cfg.max_regions] + log.info("job %s: capped regions %d→%d (dropped %d)", + job.get("job_id"), len(regions) + dropped, + len(regions), dropped) _t = time.monotonic() self.client.submit(job["job_id"], regions, replace_kinds) self._record("submit", time.monotonic() - _t) -- 2.52.0