Files
FabledCurator/backend/app/api/ml_admin.py
T
bvandeusen 19b962f1a7
CI / lint (push) Successful in 3s
CI / frontend-build (push) Successful in 20s
CI / backend-lint-and-test (push) Successful in 30s
CI / integration (push) Successful in 3m31s
feat(b3): ml-worker becomes optional — embed-only role, decoupled GPU coordination, cpu-embed switch
The ml-worker's ONLY processing role is now the CPU whole-image embed fallback
(tag_and_embed renamed embed_image — Camie tagging was retired #1189 and the
name kept implying otherwise; videos were already handled agent-style: frame
sampling + mean-pool). Detection/cropping/CCIP stay GPU-agent-only, and their
completion is judged per-pipeline: ccip by gpu_job rows, siglip by concept
regions at the current model version — never by image_record.siglip_embedding.
A CPU embed therefore can NEVER close crop work for the agent (regression test
pins this; only the whole-image 'embed' job, the same artifact, is satisfied).

Making removal actually safe (operator will drop the container):
- GPU-queue coordination (enqueue_gpu_backfill, recover_orphaned_gpu_jobs,
  reprocess_gpu_jobs) moved verbatim to tasks/gpu_queue.py on the maintenance
  quick lane — it lived on the 'ml' queue only by module colocation, which made
  the ml-worker a hard dependency of the whole agent pipeline.
- New ml_settings.cpu_embed_enabled (migration 0074, default ON so agent-less
  installs keep working): OFF stops the four import hooks queueing embed work
  nothing will consume and no-ops the manual backfill; switch lives on the
  renamed 'CPU embedding backfill' card.
- NB heads training / auto-apply still run on the ml image (sklearn) — a stack
  that removes the container gives those up too.

Deploy note: in-flight messages under the old task names are dropped by the
new workers; the 60s orphan sweep + hourly backfill re-fire under the new
names immediately.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
2026-07-02 16:53:08 -04:00

145 lines
5.1 KiB
Python

"""ML admin API: settings + backfill trigger."""
from quart import Blueprint, jsonify, request
from ..extensions import get_session
from ..models import MLSettings
ml_admin_bp = Blueprint("ml_admin", __name__, url_prefix="/api/ml")
_EDITABLE = (
"cpu_embed_enabled",
"video_frame_interval_seconds",
"video_max_frames",
"head_min_positives",
"head_auto_apply_precision",
"head_auto_apply_enabled",
"head_auto_apply_min_positives",
"ccip_match_threshold",
"ccip_auto_apply_enabled",
"ccip_auto_apply_threshold",
"embedder_model_name",
"embedder_model_version",
)
# Supported embedders for the Settings dropdown — all 1152-d so a swap is a
# drop-in (re-embed + retrain, no schema change). Server-authoritative so the UI
# never free-types a model name.
SUPPORTED_EMBEDDERS = (
{
"name": "google/siglip2-so400m-patch16-512",
"version": "siglip2-so400m-patch16-512",
"label": "SigLIP 2 · so400m · 512px (recommended)",
"dim": 1152,
},
{
"name": "google/siglip2-so400m-patch16-384",
"version": "siglip2-so400m-patch16-384",
"label": "SigLIP 2 · so400m · 384px (faster)",
"dim": 1152,
},
{
"name": "google/siglip-so400m-patch14-384",
"version": "siglip-so400m-patch14-384",
"label": "SigLIP 1 · so400m · 384px (original)",
"dim": 1152,
},
)
@ml_admin_bp.route("/embedder-models", methods=["GET"])
async def embedder_models():
return jsonify({"models": list(SUPPORTED_EMBEDDERS)})
@ml_admin_bp.route("/settings", methods=["GET"])
async def get_settings():
from sqlalchemy import select
async with get_session() as session:
s = (
await session.execute(select(MLSettings).where(MLSettings.id == 1))
).scalar_one()
return jsonify(
{
"cpu_embed_enabled": s.cpu_embed_enabled,
"video_frame_interval_seconds": s.video_frame_interval_seconds,
"video_max_frames": s.video_max_frames,
"embedder_model_version": s.embedder_model_version,
"head_min_positives": s.head_min_positives,
"head_auto_apply_precision": s.head_auto_apply_precision,
"head_auto_apply_enabled": s.head_auto_apply_enabled,
"head_auto_apply_min_positives": s.head_auto_apply_min_positives,
"ccip_match_threshold": s.ccip_match_threshold,
"ccip_auto_apply_enabled": s.ccip_auto_apply_enabled,
"ccip_auto_apply_threshold": s.ccip_auto_apply_threshold,
"embedder_model_name": s.embedder_model_name,
}
)
@ml_admin_bp.route("/settings", methods=["PATCH"])
async def patch_settings():
from sqlalchemy import select
body = await request.get_json()
if not isinstance(body, dict):
return jsonify({"error": "body must be an object"}), 400
async with get_session() as session:
s = (
await session.execute(select(MLSettings).where(MLSettings.id == 1))
).scalar_one()
# Merge the patch over current values, then validate the result as a
# whole — the store-floor invariant couples three fields, so they
# can't be checked one at a time.
proposed = {f: getattr(s, f) for f in _EDITABLE}
for field in _EDITABLE:
if field in body:
proposed[field] = body[field]
err = _validate(proposed)
if err is not None:
return jsonify({"error": err}), 400
for field in _EDITABLE:
setattr(s, field, proposed[field])
await session.commit()
return await get_settings()
def _validate(p: dict) -> str | None:
"""Returns an error string if the proposed settings are invalid, else None."""
# Video embedding (#747).
if p["video_frame_interval_seconds"] <= 0:
return "video_frame_interval_seconds must be > 0"
if p["video_max_frames"] < 1:
return "video_max_frames must be >= 1"
# Head training (#114).
if int(p["head_min_positives"]) < 1:
return "head_min_positives must be >= 1"
if not (0.5 <= float(p["head_auto_apply_precision"]) <= 0.999):
return "head_auto_apply_precision must be between 0.5 and 0.999"
if int(p["head_auto_apply_min_positives"]) < 1:
return "head_auto_apply_min_positives must be >= 1"
if not (0.5 <= float(p["ccip_match_threshold"]) <= 0.999):
return "ccip_match_threshold must be between 0.5 and 0.999"
if not (0.5 <= float(p["ccip_auto_apply_threshold"]) <= 0.999):
return "ccip_auto_apply_threshold must be between 0.5 and 0.999"
# Embedder model swap (#1190): both must be non-empty. Changing them means a
# different embedding space — the operator must re-embed + retrain after.
for key in ("embedder_model_name", "embedder_model_version"):
if not str(p[key]).strip():
return f"{key} must not be empty"
return None
@ml_admin_bp.route("/backfill", methods=["POST"])
async def trigger_backfill():
from ..tasks.ml import backfill
r = backfill.delay()
return jsonify({"celery_task_id": r.id}), 202