From 62ec70b9e448ea4e09a5d0d20ae0557681fee9c9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 5 Jul 2026 19:35:59 -0400 Subject: [PATCH 1/3] feat(ml): detector config in MLSettings with working defaults (#134 step 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the crop-proposer config (per-proposer enable + weights + conf, caps, dedupe IoU) into the DB so it's UI-tunable and can be announced to the GPU agent in the lease (like the embedder model) — no restart, agent env becomes bootstrap-only. Migration 0078 adds the columns with working server_defaults so existing rows + fresh installs crop out-of-the-box with all three proposers ON (operator: default-on): person=yolo11n.pt, anatomy=booru_yolo yolov11m_aa22 (URL, license unstated/private-homelab-OK), panel=mosesb best.pt. Plain columns, no CHECK enum. Steps 2 (lease announce + agent apply) and 3 (Settings UI) follow. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- .../versions/0078_ml_settings_detectors.py | 83 +++++++++++++++++++ backend/app/models/ml_settings.py | 64 ++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 alembic/versions/0078_ml_settings_detectors.py diff --git a/alembic/versions/0078_ml_settings_detectors.py b/alembic/versions/0078_ml_settings_detectors.py new file mode 100644 index 0000000..6d04601 --- /dev/null +++ b/alembic/versions/0078_ml_settings_detectors.py @@ -0,0 +1,83 @@ +"""ml_settings crop-proposer / detector config (#134) + +Move the WHERE-to-crop detector config (per-proposer enable + weights + conf, +plus caps + dedupe IoU) into the DB so it's UI-tunable and announced to the GPU +agent in the lease (like the embedder model) — no restart, agent env is now +bootstrap-only. All server_defaults are the working values so existing rows + +fresh installs crop out-of-the-box with all three proposers ON. + +Revision ID: 0078 +Revises: 0077 +Create Date: 2026-07-05 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0078" +down_revision: Union[str, None] = "0077" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +_ANATOMY_DEFAULT = ( + "https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt" +) +_PANEL_DEFAULT = "mosesb/best-comic-panel-detection::best.pt" + + +def upgrade() -> None: + op.add_column("ml_settings", sa.Column( + "detector_person_enabled", sa.Boolean(), nullable=False, + server_default=sa.true())) + op.add_column("ml_settings", sa.Column( + "detector_person_weights", sa.String(512), nullable=False, + server_default="yolo11n.pt")) + op.add_column("ml_settings", sa.Column( + "detector_person_conf", sa.Float(), nullable=False, + server_default=sa.text("0.35"))) + op.add_column("ml_settings", sa.Column( + "detector_anatomy_enabled", sa.Boolean(), nullable=False, + server_default=sa.true())) + op.add_column("ml_settings", sa.Column( + "detector_anatomy_weights", sa.String(512), nullable=False, + server_default=_ANATOMY_DEFAULT)) + op.add_column("ml_settings", sa.Column( + "detector_anatomy_conf", sa.Float(), nullable=False, + server_default=sa.text("0.30"))) + op.add_column("ml_settings", sa.Column( + "detector_panel_enabled", sa.Boolean(), nullable=False, + server_default=sa.true())) + op.add_column("ml_settings", sa.Column( + "detector_panel_weights", sa.String(512), nullable=False, + server_default=_PANEL_DEFAULT)) + op.add_column("ml_settings", sa.Column( + "detector_panel_conf", sa.Float(), nullable=False, + server_default=sa.text("0.30"))) + op.add_column("ml_settings", sa.Column( + "detector_max_figures", sa.Integer(), nullable=False, + server_default=sa.text("8"))) + op.add_column("ml_settings", sa.Column( + "detector_max_components", sa.Integer(), nullable=False, + server_default=sa.text("8"))) + op.add_column("ml_settings", sa.Column( + "detector_max_panels", sa.Integer(), nullable=False, + server_default=sa.text("8"))) + op.add_column("ml_settings", sa.Column( + "detector_max_regions", sa.Integer(), nullable=False, + server_default=sa.text("128"))) + op.add_column("ml_settings", sa.Column( + "detector_dedupe_iou", sa.Float(), nullable=False, + server_default=sa.text("0.85"))) + + +def downgrade() -> None: + for col in ( + "detector_person_enabled", "detector_person_weights", "detector_person_conf", + "detector_anatomy_enabled", "detector_anatomy_weights", "detector_anatomy_conf", + "detector_panel_enabled", "detector_panel_weights", "detector_panel_conf", + "detector_max_figures", "detector_max_components", "detector_max_panels", + "detector_max_regions", "detector_dedupe_iou", + ): + op.drop_column("ml_settings", col) diff --git a/backend/app/models/ml_settings.py b/backend/app/models/ml_settings.py index a45cc3d..dfb97d5 100644 --- a/backend/app/models/ml_settings.py +++ b/backend/app/models/ml_settings.py @@ -91,6 +91,70 @@ class MLSettings(Base): embedder_model_name: Mapped[str] = mapped_column( String(128), nullable=False, default="google/siglip2-so400m-patch16-512" ) + # -- Crop proposers / detectors (#1202, #134) -------------------------- + # WHERE-to-crop YOLO detectors feeding the crop→SigLIP bag + CCIP. Config + # lives HERE (DB) and is announced to the GPU agent in the lease — same as + # the embedder model — so it is UI-tunable with NO restart, and the agent's + # env is bootstrap-only. Each weights spec is an ultralytics builtin name, + # an http(s) URL, or "hf_repo::file" (agent's _resolve). enabled off (or an + # empty weights) skips that proposer. All ON by default (operator 2026-07-05) + # so a fresh install crops out-of-the-box. + # person: general COCO figure detector for Western/realistic art the anime + # person-detector misses → NMS-merged with imgutils → CCIP + concept. + detector_person_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True + ) + detector_person_weights: Mapped[str] = mapped_column( + String(512), nullable=False, default="yolo11n.pt" + ) + detector_person_conf: Mapped[float] = mapped_column( + Float, nullable=False, default=0.35 + ) + # anatomy: booru_yolo anime/furry/NSFW torso components → concept crops. + # Default = yolov11m_aa22 (26 classes, best mAP50-95 0.96), committed in the + # upstream repo so the URL resolves. License UNSTATED — fine for a private + # homelab (operator accepted #1202). + detector_anatomy_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True + ) + detector_anatomy_weights: Mapped[str] = mapped_column( + String(512), nullable=False, + default=( + "https://github.com/aperveyev/booru_yolo/raw/main/models/" + "yolov11m_aa22.pt" + ), + ) + detector_anatomy_conf: Mapped[float] = mapped_column( + Float, nullable=False, default=0.30 + ) + # panel: comic page → panel regions → concept crops (Apache-2.0, YOLOv12x). + detector_panel_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True + ) + detector_panel_weights: Mapped[str] = mapped_column( + String(512), nullable=False, + default="mosesb/best-comic-panel-detection::best.pt", + ) + detector_panel_conf: Mapped[float] = mapped_column( + Float, nullable=False, default=0.30 + ) + # Per-frame caps bound the crop→embed explosion; max_regions is the hard + # per-job backstop; dedupe_iou drops near-duplicate crops before the embed. + detector_max_figures: Mapped[int] = mapped_column( + Integer, nullable=False, default=8 + ) + detector_max_components: Mapped[int] = mapped_column( + Integer, nullable=False, default=8 + ) + detector_max_panels: Mapped[int] = mapped_column( + Integer, nullable=False, default=8 + ) + detector_max_regions: Mapped[int] = mapped_column( + Integer, nullable=False, default=128 + ) + detector_dedupe_iou: Mapped[float] = mapped_column( + Float, nullable=False, default=0.85 + ) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) -- 2.52.0 From a4df2793430b7b75c6a0bddb4a1826ca39533642 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 5 Jul 2026 19:42:59 -0400 Subject: [PATCH 2/3] feat(ml): lease announces detector config; agent builds proposers from it live (#134 step 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GPU lease now carries the crop-proposer config from MLSettings in a per-job 'detectors' block (same pattern as embed_model_name). The agent's worker builds its Proposers from the announced config via _effective_cfg (lease block overlaid on env) + _proposers_for (rebuilds only when a config signature changes) — so an operator's UI edit takes effect on the next lease with NO restart, and env is now just the bootstrap fallback until the server announces. enabled-off maps to empty weights (proposer skipped); dedupe_iou + max_regions also come from the effective cfg. Test: lease announces the detectors block with the seeded default weights. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- agent/fc_agent/worker.py | 59 ++++++++++++++++++++++++++++++++++------ backend/app/api/gpu.py | 28 +++++++++++++++++++ tests/test_api_gpu.py | 24 ++++++++++++++++ 3 files changed, 102 insertions(+), 9 deletions(-) diff --git a/agent/fc_agent/worker.py b/agent/fc_agent/worker.py index 9c8db2d..2601d7e 100644 --- a/agent/fc_agent/worker.py +++ b/agent/fc_agent/worker.py @@ -265,6 +265,8 @@ class Worker: self._embedder = None self._embedder_lock = threading.Lock() self._proposers = None + self._proposers_sig = None # detector-config signature the current + # proposers were built for (#134) self._proposers_lock = threading.Lock() # --- held-lease bookkeeping -------------------------------------------- @@ -796,14 +798,52 @@ class Worker: self._embedder = CropEmbedder(model_name, self.cfg.embed_dtype) return self._embedder - def _ensure_proposers(self): - if self._proposers is not None: - return self._proposers + def _effective_cfg(self, det: dict | None): + """The detector config for a job: the LEASE-ANNOUNCED block (the DB/UI + source of truth, #134) overlaid on the env cfg, or the env cfg unchanged + when the server announced nothing (bootstrap / older backend). An enabled + toggle off maps to empty weights — the 'weights == "" → proposer skipped' + contract in detectors.py. detector_level + ccip_model stay env (separate + settings, not part of the crop-proposer config).""" + if not det: + return self.cfg + + def _w(p: str) -> str: + slot = det.get(p) or {} + return (slot.get("weights") or "") if slot.get("enabled", True) else "" + + from dataclasses import replace + return replace( + self.cfg, + person_weights=_w("person"), + person_conf=float(det["person"]["conf"]), + anatomy_weights=_w("anatomy"), + anatomy_conf=float(det["anatomy"]["conf"]), + panel_weights=_w("panel"), + panel_conf=float(det["panel"]["conf"]), + max_figures=int(det["max_figures"]), + max_components=int(det["max_components"]), + max_panels=int(det["max_panels"]), + max_regions=int(det["max_regions"]), + dedupe_iou=float(det["dedupe_iou"]), + ) + + def _proposers_for(self, eff): + """Proposers built from the effective detector config, rebuilt ONLY when + that config changes — so an operator's UI edit takes effect on the next + lease with no restart, and steady-state reuses the loaded YOLO models.""" + sig = ( + eff.person_weights, eff.person_conf, + eff.anatomy_weights, eff.anatomy_conf, + eff.panel_weights, eff.panel_conf, + eff.max_figures, eff.max_components, eff.max_panels, + ) with self._proposers_lock: - if self._proposers is None: + if self._proposers is None or self._proposers_sig != sig: from .detectors import Proposers - self._proposers = Proposers(self.cfg) - return self._proposers + self._proposers = Proposers(eff) + self._proposers_sig = sig + return self._proposers def _consume(self, job: dict, frames: list, stop_evt: threading.Event) -> bool: """Detect + embed the decoded frames and submit the result. Returns True @@ -856,7 +896,8 @@ class Worker: else ["figure", "face", "concept", "panel"] ) embedder = self._ensure_embedder(model_name) if want_siglip else None - proposers = self._ensure_proposers() + eff = self._effective_cfg(job.get("detectors")) + proposers = self._proposers_for(eff) regions = [] ccip_ev = self.cfg.ccip_model or "ccip-default" @@ -922,7 +963,7 @@ class Worker: # we never SigLIP the same region twice — saves GPU and a # slot against max_regions. High-IoU + kind-aware, so # intentional nested crops (figure ⊃ head) survive. - pending = dedupe_crops(pending, self.cfg.dedupe_iou) + pending = dedupe_crops(pending, eff.dedupe_iou) vecs = embedder.embed_batch([c for c, _ in pending]) for (_c, tmpl), vec in zip(pending, vecs, strict=True): tmpl["siglip_embedding"] = vec @@ -932,7 +973,7 @@ class Worker: # mp4, 64 sampled frames × ~32 regions) would otherwise burn # ~38s of GPU across every frame before the submit is even # truncated. Bounds the WORK, not just the POST body. - if len(regions) >= self.cfg.max_regions: + if len(regions) >= eff.max_regions: break # A Stop mid-frame-loop leaves partial regions — don't submit those; diff --git a/backend/app/api/gpu.py b/backend/app/api/gpu.py index c6e0cd8..44a7aac 100644 --- a/backend/app/api/gpu.py +++ b/backend/app/api/gpu.py @@ -269,6 +269,33 @@ async def lease(): ).scalars() } if ids else {} await session.commit() + # Crop-proposer config, announced FROM THE SETTING like embed_model_name + # (#134): the agent builds its detectors from this, rebuilding live when + # it changes — so tuning is a DB/UI edit, never an agent restart. Same + # block for every job in the batch (it's global), built once. An enabled + # toggle off is carried through so the agent skips that proposer. + detectors = { + "person": { + "enabled": ml.detector_person_enabled, + "weights": ml.detector_person_weights, + "conf": ml.detector_person_conf, + }, + "anatomy": { + "enabled": ml.detector_anatomy_enabled, + "weights": ml.detector_anatomy_weights, + "conf": ml.detector_anatomy_conf, + }, + "panel": { + "enabled": ml.detector_panel_enabled, + "weights": ml.detector_panel_weights, + "conf": ml.detector_panel_conf, + }, + "max_figures": ml.detector_max_figures, + "max_components": ml.detector_max_components, + "max_panels": ml.detector_max_panels, + "max_regions": ml.detector_max_regions, + "dedupe_iou": ml.detector_dedupe_iou, + } out = [] for j in jobs: img = imgs.get(j.image_record_id) @@ -290,6 +317,7 @@ async def lease(): # re-embed, never an agent change. "embed_model_name": ml.embedder_model_name, "embed_version": ml.embedder_model_version, + "detectors": detectors, }) return jsonify({"jobs": out}) diff --git a/tests/test_api_gpu.py b/tests/test_api_gpu.py index e476c6b..25f0a7b 100644 --- a/tests/test_api_gpu.py +++ b/tests/test_api_gpu.py @@ -105,6 +105,30 @@ async def test_lease_announces_embed_model_then_submit_embedding(client, db): assert img.siglip_embedding is not None and len(list(img.siglip_embedding)) == 1152 +@pytest.mark.asyncio +async def test_lease_announces_detector_config(client, db): + # #134: the lease carries the crop-proposer config from MLSettings, so the + # agent builds its detectors from the DB/UI (no restart). Defaults are all-on + # with the pinned working weights. + img = await _img(db, "c" * 64) + await GpuJobService(db).enqueue(img.id, "embed") + await db.commit() + token = (await (await client.post("/api/gpu/token/rotate")).get_json())["token"] + hdr = {"Authorization": f"Bearer {token}"} + + leased = await client.post( + "/api/gpu/jobs/lease", json={"agent_id": "a1", "batch_size": 5}, headers=hdr, + ) + det = (await leased.get_json())["jobs"][0]["detectors"] + assert det["person"]["enabled"] + assert det["anatomy"]["enabled"] + assert det["panel"]["enabled"] + assert "yolov11m_aa22" in det["anatomy"]["weights"] # booru_yolo default + assert det["panel"]["weights"].endswith("::best.pt") # mosesb default + assert det["max_regions"] == 128 + assert det["dedupe_iou"] == 0.85 + + @pytest.mark.asyncio async def test_submit_with_stale_lease_is_409(client, db): img = await _img(db, "b" * 64) -- 2.52.0 From ab362bc79cd31095048e928544fd0e14402cb15a Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 5 Jul 2026 19:51:55 -0400 Subject: [PATCH 3/3] =?UTF-8?q?feat(ml):=20Settings=20=E2=86=92=20Tagging?= =?UTF-8?q?=20'Crop=20proposers'=20card=20(#134=20step=203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exposes the detector config (per-proposer enable + weights + confidence, caps, dedupe IoU) in Settings → Tagging, backed by MLSettings via /api/ml/settings. ml_admin adds the detector fields to _EDITABLE + GET payload + validation (conf 0..1, caps >=1, IoU 0..1). New CropProposersCard.vue (mirrors HeadsCard) with working defaults pre-filled, per-field live-save (no restart — the agent picks changes up on its next lease), weights-format help, switch-revert on error. Closes milestone #134: all three proposers are on out-of-the-box and tunable in the UI. Test: detector defaults GET + patch round-trip + range validation. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- backend/app/api/ml_admin.py | 35 ++++ .../components/settings/CropProposersCard.vue | 151 ++++++++++++++++++ .../components/settings/MaintenancePanel.vue | 2 + tests/test_api_ml_admin.py | 29 ++++ 4 files changed, 217 insertions(+) create mode 100644 frontend/src/components/settings/CropProposersCard.vue diff --git a/backend/app/api/ml_admin.py b/backend/app/api/ml_admin.py index 62fbc26..c146b70 100644 --- a/backend/app/api/ml_admin.py +++ b/backend/app/api/ml_admin.py @@ -8,6 +8,26 @@ from ..models import MLSettings ml_admin_bp = Blueprint("ml_admin", __name__, url_prefix="/api/ml") +# Crop-proposer / detector config (#134). Announced to the GPU agent in the lease +# → tunable here with no restart. weights = ultralytics name | URL | hf_repo::file +# (empty, or enabled off, skips that proposer). +_DETECTOR_FIELDS = ( + "detector_person_enabled", + "detector_person_weights", + "detector_person_conf", + "detector_anatomy_enabled", + "detector_anatomy_weights", + "detector_anatomy_conf", + "detector_panel_enabled", + "detector_panel_weights", + "detector_panel_conf", + "detector_max_figures", + "detector_max_components", + "detector_max_panels", + "detector_max_regions", + "detector_dedupe_iou", +) + _EDITABLE = ( "cpu_embed_enabled", "video_frame_interval_seconds", @@ -21,6 +41,7 @@ _EDITABLE = ( "ccip_auto_apply_threshold", "embedder_model_name", "embedder_model_version", + *_DETECTOR_FIELDS, ) @@ -76,6 +97,7 @@ async def get_settings(): "ccip_auto_apply_enabled": s.ccip_auto_apply_enabled, "ccip_auto_apply_threshold": s.ccip_auto_apply_threshold, "embedder_model_name": s.embedder_model_name, + **{f: getattr(s, f) for f in _DETECTOR_FIELDS}, } ) @@ -133,6 +155,19 @@ def _validate(p: dict) -> str | None: for key in ("embedder_model_name", "embedder_model_version"): if not str(p[key]).strip(): return f"{key} must not be empty" + # Crop proposers (#134). Weights may be empty (that proposer is just off); + # confidences are probabilities; caps are positive counts; IoU is [0,1]. + for key in ("detector_person_conf", "detector_anatomy_conf", "detector_panel_conf"): + if not (0.0 <= float(p[key]) <= 1.0): + return f"{key} must be between 0 and 1" + for key in ( + "detector_max_figures", "detector_max_components", + "detector_max_panels", "detector_max_regions", + ): + if int(p[key]) < 1: + return f"{key} must be >= 1" + if not (0.0 <= float(p["detector_dedupe_iou"]) <= 1.0): + return "detector_dedupe_iou must be between 0 and 1" return None diff --git a/frontend/src/components/settings/CropProposersCard.vue b/frontend/src/components/settings/CropProposersCard.vue new file mode 100644 index 0000000..46edd54 --- /dev/null +++ b/frontend/src/components/settings/CropProposersCard.vue @@ -0,0 +1,151 @@ + + + + + diff --git a/frontend/src/components/settings/MaintenancePanel.vue b/frontend/src/components/settings/MaintenancePanel.vue index 26cae1f..dfc5693 100644 --- a/frontend/src/components/settings/MaintenancePanel.vue +++ b/frontend/src/components/settings/MaintenancePanel.vue @@ -36,6 +36,7 @@

+
@@ -75,6 +76,7 @@ import MissingFileRepairCard from './MissingFileRepairCard.vue' import GpuTriageCard from './GpuTriageCard.vue' import DbMaintenanceCard from './DbMaintenanceCard.vue' import MLThresholdSliders from './MLThresholdSliders.vue' +import CropProposersCard from './CropProposersCard.vue' import HeadsCard from './HeadsCard.vue' import GpuAgentCard from './GpuAgentCard.vue' import AliasTable from './AliasTable.vue' diff --git a/tests/test_api_ml_admin.py b/tests/test_api_ml_admin.py index 5602f8c..33faf09 100644 --- a/tests/test_api_ml_admin.py +++ b/tests/test_api_ml_admin.py @@ -52,6 +52,35 @@ async def test_embedder_model_default_settable_and_empty_rejected(client): assert bad.status_code == 400 +@pytest.mark.asyncio +async def test_detector_settings_defaults_patch_and_validation(client): + # #134: crop-proposer config is exposed + editable here (announced to the + # agent via the lease). Defaults are all-on with the pinned weights. + body = await (await client.get("/api/ml/settings")).get_json() + assert body["detector_person_enabled"] is True + assert body["detector_anatomy_enabled"] is True + assert "yolov11m_aa22" in body["detector_anatomy_weights"] + assert body["detector_panel_weights"].endswith("::best.pt") + assert body["detector_max_regions"] == 128 + + ok = await client.patch("/api/ml/settings", json={ + "detector_anatomy_enabled": False, + "detector_person_conf": 0.5, + "detector_max_panels": 4, + }) + assert ok.status_code == 200 + out = await ok.get_json() + assert out["detector_anatomy_enabled"] is False + assert out["detector_person_conf"] == 0.5 + assert out["detector_max_panels"] == 4 + + # Out-of-range confidence + non-positive cap are rejected. + assert (await client.patch( + "/api/ml/settings", json={"detector_panel_conf": 1.5})).status_code == 400 + assert (await client.patch( + "/api/ml/settings", json={"detector_max_regions": 0})).status_code == 400 + + @pytest.mark.asyncio async def test_embedder_models_list(client): # #1203: the dropdown reads the supported-model list from the server. -- 2.52.0