feat(agent): crop proposers — booru_yolo anatomy + COCO person + comic panels (#1202)
CI / lint (push) Successful in 3s
CI / frontend-build (push) Successful in 17s
CI / backend-lint-and-test (push) Successful in 26s
CI / integration (push) Successful in 3m25s

Better region PROPOSERS feeding the existing crop→SigLIP→max-over-bag heads (no
change to the learned-tagging approach; no per-tag cost — propose once, embed
each region, all heads in one matmul).

- detectors.py: lazy ultralytics YOLO wrapper, each proposer independently
  optional + guarded (a bad weight spec / inference error self-disables that one,
  logged, never breaks the worker). Weights resolve from an ultralytics name |
  http(s) URL | "hf_repo::file", cached under HF_HOME. NMS merge so a figure two
  detectors both find collapses to one crop.
- worker: figure boxes = imgutils detect_person ∪ general COCO person (merged)
  → CCIP + concept (anime + Western/realistic coverage); booru_yolo anatomy
  components (head/cat-head/anatomy/…) → concept crops; comic panels → kind=
  'panel' concept crops. Capped per frame (MAX_COMPONENTS/MAX_PANELS).
- config + compose: PERSON_WEIGHTS (default yolo11n.pt, works OOB),
  ANATOMY_WEIGHTS + PANEL_WEIGHTS (operator sets booru_yolo URL + mosesb panel
  hf::file; empty = off). ultralytics added to requirements.
- backend: image_region 'kind' doc notes 'panel'; no migration (free String,
  and the bag scorer keys on a non-null siglip_embedding, not the kind, so any
  SigLIP region joins the bag automatically).

Agent is outside CI — py-compiled here; operator tests on the GPU and checks
Western-vs-anime crop quality via /api/ccip observability.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
This commit is contained in:
2026-06-30 15:27:26 -04:00
parent 3d7f60a6e3
commit d5f29f7056
6 changed files with 250 additions and 15 deletions
+51 -14
View File
@@ -76,6 +76,9 @@ class Worker:
# needs it, from the model the server announces — one shared instance.
self._embedder = None
self._embedder_lock = threading.Lock()
# Region proposers (extra YOLO detectors) — lazily built once, shared.
self._proposers = None
self._proposers_lock = threading.Lock()
# --- control -----------------------------------------------------------
def start(self):
@@ -177,6 +180,15 @@ class Worker:
self._embedder = CropEmbedder(model_name, self.cfg.embed_dtype)
return self._embedder
def _ensure_proposers(self):
if self._proposers is not None:
return self._proposers
with self._proposers_lock:
if self._proposers is None:
from .detectors import Proposers
self._proposers = Proposers(self.cfg)
return self._proposers
def _process(self, job: dict) -> bool:
"""Process one job. Returns True when handled (completed, or hard-failed
because the job itself is bad) and False on a TRANSPORT error (curator
@@ -227,43 +239,68 @@ class Worker:
want_ccip = task in ("ccip", "both")
want_siglip = task in ("ccip", "siglip", "both")
replace_kinds = (
["concept"] if task == "siglip" else ["figure", "face", "concept"]
["concept", "panel"] if task == "siglip"
else ["figure", "face", "concept", "panel"]
)
embedder = self._ensure_embedder(model_name) if want_siglip else None
proposers = self._ensure_proposers()
regions = []
ccip_ev = self.cfg.ccip_model or "ccip-default"
dv = f"person-{self.cfg.detector_level}"
def _concept(frame, bbox, t, score, detver, kind="concept"):
"""A SigLIP region for one crop (None if below the size floor)."""
crop = crop_region(frame, bbox)
if crop is None:
return None
return {
"kind": kind, "bbox": list(bbox), "frame_time": t,
"score": score, "siglip_embedding": embedder.embed(crop),
"embedding_version": embed_version, "detector_version": detver,
}
for t, frame in frames:
figs = models.detect_figures(frame, self.cfg.detector_level)
# FIGURE boxes: imgutils detect_person general COCO person,
# NMS-merged → CCIP identity (+ a concept crop). Covers anime +
# Western/realistic figures.
base = models.detect_figures(frame, self.cfg.detector_level)
figs = proposers.figures(frame, base)
if not figs:
figs = [((0.0, 0.0, 1.0, 1.0), None)] # whole-frame fallback
for bbox, score in figs:
figs = [((0.0, 0.0, 1.0, 1.0), 1.0, "whole")] # whole-frame fallback
for bbox, score, _label in figs:
crop = crop_region(frame, bbox)
if crop is None:
continue
if want_ccip:
regions.append({
"kind": "figure",
"bbox": list(bbox),
"frame_time": t,
"kind": "figure", "bbox": list(bbox), "frame_time": t,
"score": score,
"ccip_embedding": models.ccip_vector(
crop, self.cfg.ccip_model or None
),
"embedding_version": ccip_ev,
"detector_version": dv,
"embedding_version": ccip_ev, "detector_version": dv,
})
if want_siglip:
regions.append({
"kind": "concept",
"bbox": list(bbox),
"frame_time": t,
"kind": "concept", "bbox": list(bbox), "frame_time": t,
"score": score,
"siglip_embedding": embedder.embed(crop),
"embedding_version": embed_version,
"detector_version": dv,
"embedding_version": embed_version, "detector_version": dv,
})
if not want_siglip:
continue
# ANATOMY components (booru_yolo: head/cat-head/anatomy/…) →
# concept crops only (not full characters, so no CCIP).
for bbox, score, label in proposers.components(frame):
r = _concept(frame, bbox, t, score, f"booru:{label}")
if r is not None:
regions.append(r)
# PANEL crops (comic page → panels) → kind='panel' (still SigLIP).
for bbox, score, _label in proposers.panels(frame):
r = _concept(frame, bbox, t, score, "panel", kind="panel")
if r is not None:
regions.append(r)
self.client.submit(job["job_id"], regions, replace_kinds)
self._bump(processed=1)
return True