feat(ml): tag-eval backend — head-vs-centroid learning-curve eval (persisted)
Slice 1 of milestone #114 (tagging v2). Proves the frozen-embedding + trained- head spine on the operator's own data, reusing the SigLIP embeddings already stored on image_record — no re-embedding, no GPU. Per concept: train a logistic-regression HEAD (positives + negatives = explicit rejections + sampled unlabeled) vs the old single-CENTROID baseline; report cross-validated precision/recall/AP for both, a LEARNING CURVE (AP/F1 as tagged positives grow 10→30→100→300), and example image ids (head-would-suggest / head-doubts-positive) to eyeball. Persisted so the report SURVIVES navigation (operator-flagged): the run + full report live in a new tag_eval_run row (mirrors library_audit_run); the admin card will rehydrate from GET on mount, not transient state. - models.TagEvalRun + migration 0056; runs on the ml queue (only worker with numpy/sklearn) — numpy/sklearn lazy-imported so the API can still enqueue. - services/ml/tag_eval (compute + start helper, one-running guard), tasks.ml .tag_eval_run, api/tag-eval (POST create, GET history light / detail w/ report). - recover_stalled_tag_eval_runs sweep + retention (keep last 20) + 5-min beat (rule 89). scikit-learn added to requirements-ml. - tests: param normalization + the rehydrate read-path + create/conflict. Frontend admin card (trigger + render persisted report) follows next. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -36,6 +36,7 @@ def all_blueprints() -> list[Blueprint]:
|
||||
from .suggestions import suggestions_bp
|
||||
from .system_activity import system_activity_bp
|
||||
from .system_backup import system_backup_bp
|
||||
from .tag_eval import tag_eval_bp
|
||||
from .tags import tags_bp
|
||||
from .thumbnails import thumbnails_bp
|
||||
return [
|
||||
@@ -56,6 +57,7 @@ def all_blueprints() -> list[Blueprint]:
|
||||
suggestions_bp,
|
||||
allowlist_bp,
|
||||
aliases_bp,
|
||||
tag_eval_bp,
|
||||
ml_admin_bp,
|
||||
thumbnails_bp,
|
||||
sources_bp,
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Tag-eval API (#1130): trigger + revisit the head-vs-centroid eval.
|
||||
|
||||
The run + full report live in the tag_eval_run row, so the admin card rehydrates
|
||||
from GET (history / detail) on mount — the report survives navigation rather than
|
||||
living in transient frontend state.
|
||||
"""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import TagEvalRun
|
||||
from ..services.ml.tag_eval import EvalAlreadyRunning, start_tag_eval_run
|
||||
|
||||
tag_eval_bp = Blueprint("tag_eval", __name__, url_prefix="/api/tag-eval")
|
||||
|
||||
|
||||
def _serialize(run: TagEvalRun, *, include_report: bool) -> dict:
|
||||
out = {
|
||||
"id": run.id,
|
||||
"params": run.params,
|
||||
"status": run.status,
|
||||
"started_at": run.started_at.isoformat() if run.started_at else None,
|
||||
"finished_at": run.finished_at.isoformat() if run.finished_at else None,
|
||||
"error": run.error,
|
||||
}
|
||||
if include_report:
|
||||
out["report"] = run.report
|
||||
return out
|
||||
|
||||
|
||||
@tag_eval_bp.route("", methods=["POST"])
|
||||
async def create():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
params = body.get("params") or body or {}
|
||||
async with get_session() as session:
|
||||
try:
|
||||
run_id = await session.run_sync(
|
||||
lambda s: start_tag_eval_run(s, params)
|
||||
)
|
||||
except EvalAlreadyRunning as running:
|
||||
return jsonify({
|
||||
"error": "eval_already_running",
|
||||
"running_id": int(running.args[0]),
|
||||
}), 409
|
||||
await session.commit()
|
||||
return jsonify({"run_id": run_id, "status": "running"}), 202
|
||||
|
||||
|
||||
@tag_eval_bp.route("", methods=["GET"])
|
||||
async def history():
|
||||
try:
|
||||
limit = min(int(request.args.get("limit", "20")), 100)
|
||||
except ValueError:
|
||||
return jsonify({"error": "invalid_limit"}), 400
|
||||
async with get_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(TagEvalRun).order_by(TagEvalRun.id.desc()).limit(limit)
|
||||
)).scalars().all()
|
||||
# List is light — no full report (the detail endpoint carries it).
|
||||
return jsonify({"runs": [_serialize(r, include_report=False) for r in rows]})
|
||||
|
||||
|
||||
@tag_eval_bp.route("/<int:run_id>", methods=["GET"])
|
||||
async def detail(run_id: int):
|
||||
async with get_session() as session:
|
||||
run = await session.get(TagEvalRun, run_id)
|
||||
if run is None:
|
||||
return jsonify({"error": "not_found"}), 404
|
||||
return jsonify(_serialize(run, include_report=True))
|
||||
Reference in New Issue
Block a user