feat(ml): tag-eval backend — head-vs-centroid learning-curve eval (persisted)
Slice 1 of milestone #114 (tagging v2). Proves the frozen-embedding + trained- head spine on the operator's own data, reusing the SigLIP embeddings already stored on image_record — no re-embedding, no GPU. Per concept: train a logistic-regression HEAD (positives + negatives = explicit rejections + sampled unlabeled) vs the old single-CENTROID baseline; report cross-validated precision/recall/AP for both, a LEARNING CURVE (AP/F1 as tagged positives grow 10→30→100→300), and example image ids (head-would-suggest / head-doubts-positive) to eyeball. Persisted so the report SURVIVES navigation (operator-flagged): the run + full report live in a new tag_eval_run row (mirrors library_audit_run); the admin card will rehydrate from GET on mount, not transient state. - models.TagEvalRun + migration 0056; runs on the ml queue (only worker with numpy/sklearn) — numpy/sklearn lazy-imported so the API can still enqueue. - services/ml/tag_eval (compute + start helper, one-running guard), tasks.ml .tag_eval_run, api/tag-eval (POST create, GET history light / detail w/ report). - recover_stalled_tag_eval_runs sweep + retention (keep last 20) + 5-min beat (rule 89). scikit-learn added to requirements-ml. - tests: param normalization + the rehydrate read-path + create/conflict. Frontend admin card (trigger + render persisted report) follows next. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,77 @@
|
||||
import pytest
|
||||
|
||||
from backend.app.models import TagEvalRun
|
||||
from backend.app.services.ml.tag_eval import (
|
||||
DEFAULT_CONCEPTS,
|
||||
_normalize_params,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def test_normalize_params_defaults_and_overrides():
|
||||
d = _normalize_params(None)
|
||||
assert d["concepts"] == DEFAULT_CONCEPTS
|
||||
assert d["neg_ratio"] >= 1 and d["cv_folds"] >= 2
|
||||
over = _normalize_params(
|
||||
{"concepts": ["glasses", " ", "cat"], "neg_ratio": "4",
|
||||
"cv_folds": "1", "curve_points": [30, 10, 10]}
|
||||
)
|
||||
assert over["concepts"] == ["glasses", "cat"] # blanks dropped
|
||||
assert over["neg_ratio"] == 4
|
||||
assert over["cv_folds"] == 2 # clamped to >=2
|
||||
assert over["curve_points"] == [10, 30] # deduped + sorted
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_and_detail_rehydrate(client, db):
|
||||
# A finished run with a report — the persisted row IS the survives-navigation
|
||||
# source: history is light (no report), detail carries it.
|
||||
run = TagEvalRun(
|
||||
params={"concepts": ["glasses"]},
|
||||
status="ready",
|
||||
report={"concepts": [{"name": "glasses", "head": {"ap": 0.9}}]},
|
||||
)
|
||||
db.add(run)
|
||||
await db.flush()
|
||||
await db.commit()
|
||||
rid = run.id
|
||||
|
||||
h = await client.get("/api/tag-eval?limit=10")
|
||||
assert h.status_code == 200
|
||||
hbody = await h.get_json()
|
||||
row = next(r for r in hbody["runs"] if r["id"] == rid)
|
||||
assert row["status"] == "ready"
|
||||
assert "report" not in row # list stays light
|
||||
|
||||
d = await client.get(f"/api/tag-eval/{rid}")
|
||||
assert d.status_code == 200
|
||||
dbody = await d.get_json()
|
||||
assert dbody["report"]["concepts"][0]["name"] == "glasses"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_enqueues_running(client, db, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"backend.app.tasks.ml.tag_eval_run.delay", lambda *a, **k: None
|
||||
)
|
||||
resp = await client.post("/api/tag-eval", json={"params": {"concepts": ["cat"]}})
|
||||
assert resp.status_code == 202
|
||||
body = await resp.get_json()
|
||||
assert body["status"] == "running"
|
||||
got = await db.get(TagEvalRun, body["run_id"])
|
||||
assert got is not None and got.status == "running"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_conflicts_when_one_running(client, db, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"backend.app.tasks.ml.tag_eval_run.delay", lambda *a, **k: None
|
||||
)
|
||||
db.add(TagEvalRun(params={}, status="running"))
|
||||
await db.flush()
|
||||
await db.commit()
|
||||
resp = await client.post("/api/tag-eval", json={"params": {}})
|
||||
assert resp.status_code == 409
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "eval_already_running"
|
||||
Reference in New Issue
Block a user