feat(agent): crop proposers — booru_yolo anatomy + COCO person + comic panels (#1202)
Better region PROPOSERS feeding the existing crop→SigLIP→max-over-bag heads (no change to the learned-tagging approach; no per-tag cost — propose once, embed each region, all heads in one matmul). - detectors.py: lazy ultralytics YOLO wrapper, each proposer independently optional + guarded (a bad weight spec / inference error self-disables that one, logged, never breaks the worker). Weights resolve from an ultralytics name | http(s) URL | "hf_repo::file", cached under HF_HOME. NMS merge so a figure two detectors both find collapses to one crop. - worker: figure boxes = imgutils detect_person ∪ general COCO person (merged) → CCIP + concept (anime + Western/realistic coverage); booru_yolo anatomy components (head/cat-head/anatomy/…) → concept crops; comic panels → kind= 'panel' concept crops. Capped per frame (MAX_COMPONENTS/MAX_PANELS). - config + compose: PERSON_WEIGHTS (default yolo11n.pt, works OOB), ANATOMY_WEIGHTS + PANEL_WEIGHTS (operator sets booru_yolo URL + mosesb panel hf::file; empty = off). ultralytics added to requirements. - backend: image_region 'kind' doc notes 'panel'; no migration (free String, and the bag scorer keys on a non-null siglip_embedding, not the kind, so any SigLIP region joins the bag automatically). Agent is outside CI — py-compiled here; operator tests on the GPU and checks Western-vs-anime crop quality via /api/ccip observability. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
This commit is contained in:
+51
-14
@@ -76,6 +76,9 @@ class Worker:
|
||||
# needs it, from the model the server announces — one shared instance.
|
||||
self._embedder = None
|
||||
self._embedder_lock = threading.Lock()
|
||||
# Region proposers (extra YOLO detectors) — lazily built once, shared.
|
||||
self._proposers = None
|
||||
self._proposers_lock = threading.Lock()
|
||||
|
||||
# --- control -----------------------------------------------------------
|
||||
def start(self):
|
||||
@@ -177,6 +180,15 @@ class Worker:
|
||||
self._embedder = CropEmbedder(model_name, self.cfg.embed_dtype)
|
||||
return self._embedder
|
||||
|
||||
def _ensure_proposers(self):
|
||||
if self._proposers is not None:
|
||||
return self._proposers
|
||||
with self._proposers_lock:
|
||||
if self._proposers is None:
|
||||
from .detectors import Proposers
|
||||
self._proposers = Proposers(self.cfg)
|
||||
return self._proposers
|
||||
|
||||
def _process(self, job: dict) -> bool:
|
||||
"""Process one job. Returns True when handled (completed, or hard-failed
|
||||
because the job itself is bad) and False on a TRANSPORT error (curator
|
||||
@@ -227,43 +239,68 @@ class Worker:
|
||||
want_ccip = task in ("ccip", "both")
|
||||
want_siglip = task in ("ccip", "siglip", "both")
|
||||
replace_kinds = (
|
||||
["concept"] if task == "siglip" else ["figure", "face", "concept"]
|
||||
["concept", "panel"] if task == "siglip"
|
||||
else ["figure", "face", "concept", "panel"]
|
||||
)
|
||||
embedder = self._ensure_embedder(model_name) if want_siglip else None
|
||||
proposers = self._ensure_proposers()
|
||||
|
||||
regions = []
|
||||
ccip_ev = self.cfg.ccip_model or "ccip-default"
|
||||
dv = f"person-{self.cfg.detector_level}"
|
||||
|
||||
def _concept(frame, bbox, t, score, detver, kind="concept"):
|
||||
"""A SigLIP region for one crop (None if below the size floor)."""
|
||||
crop = crop_region(frame, bbox)
|
||||
if crop is None:
|
||||
return None
|
||||
return {
|
||||
"kind": kind, "bbox": list(bbox), "frame_time": t,
|
||||
"score": score, "siglip_embedding": embedder.embed(crop),
|
||||
"embedding_version": embed_version, "detector_version": detver,
|
||||
}
|
||||
|
||||
for t, frame in frames:
|
||||
figs = models.detect_figures(frame, self.cfg.detector_level)
|
||||
# FIGURE boxes: imgutils detect_person ∪ general COCO person,
|
||||
# NMS-merged → CCIP identity (+ a concept crop). Covers anime +
|
||||
# Western/realistic figures.
|
||||
base = models.detect_figures(frame, self.cfg.detector_level)
|
||||
figs = proposers.figures(frame, base)
|
||||
if not figs:
|
||||
figs = [((0.0, 0.0, 1.0, 1.0), None)] # whole-frame fallback
|
||||
for bbox, score in figs:
|
||||
figs = [((0.0, 0.0, 1.0, 1.0), 1.0, "whole")] # whole-frame fallback
|
||||
for bbox, score, _label in figs:
|
||||
crop = crop_region(frame, bbox)
|
||||
if crop is None:
|
||||
continue
|
||||
if want_ccip:
|
||||
regions.append({
|
||||
"kind": "figure",
|
||||
"bbox": list(bbox),
|
||||
"frame_time": t,
|
||||
"kind": "figure", "bbox": list(bbox), "frame_time": t,
|
||||
"score": score,
|
||||
"ccip_embedding": models.ccip_vector(
|
||||
crop, self.cfg.ccip_model or None
|
||||
),
|
||||
"embedding_version": ccip_ev,
|
||||
"detector_version": dv,
|
||||
"embedding_version": ccip_ev, "detector_version": dv,
|
||||
})
|
||||
if want_siglip:
|
||||
regions.append({
|
||||
"kind": "concept",
|
||||
"bbox": list(bbox),
|
||||
"frame_time": t,
|
||||
"kind": "concept", "bbox": list(bbox), "frame_time": t,
|
||||
"score": score,
|
||||
"siglip_embedding": embedder.embed(crop),
|
||||
"embedding_version": embed_version,
|
||||
"detector_version": dv,
|
||||
"embedding_version": embed_version, "detector_version": dv,
|
||||
})
|
||||
if not want_siglip:
|
||||
continue
|
||||
# ANATOMY components (booru_yolo: head/cat-head/anatomy/…) →
|
||||
# concept crops only (not full characters, so no CCIP).
|
||||
for bbox, score, label in proposers.components(frame):
|
||||
r = _concept(frame, bbox, t, score, f"booru:{label}")
|
||||
if r is not None:
|
||||
regions.append(r)
|
||||
# PANEL crops (comic page → panels) → kind='panel' (still SigLIP).
|
||||
for bbox, score, _label in proposers.panels(frame):
|
||||
r = _concept(frame, bbox, t, score, "panel", kind="panel")
|
||||
if r is not None:
|
||||
regions.append(r)
|
||||
self.client.submit(job["job_id"], regions, replace_kinds)
|
||||
self._bump(processed=1)
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user