feat(ml): tag-eval backend — head-vs-centroid learning-curve eval (persisted)
Slice 1 of milestone #114 (tagging v2). Proves the frozen-embedding + trained- head spine on the operator's own data, reusing the SigLIP embeddings already stored on image_record — no re-embedding, no GPU. Per concept: train a logistic-regression HEAD (positives + negatives = explicit rejections + sampled unlabeled) vs the old single-CENTROID baseline; report cross-validated precision/recall/AP for both, a LEARNING CURVE (AP/F1 as tagged positives grow 10→30→100→300), and example image ids (head-would-suggest / head-doubts-positive) to eyeball. Persisted so the report SURVIVES navigation (operator-flagged): the run + full report live in a new tag_eval_run row (mirrors library_audit_run); the admin card will rehydrate from GET on mount, not transient state. - models.TagEvalRun + migration 0056; runs on the ml queue (only worker with numpy/sklearn) — numpy/sklearn lazy-imported so the API can still enqueue. - services/ml/tag_eval (compute + start helper, one-running guard), tasks.ml .tag_eval_run, api/tag-eval (POST create, GET history light / detail w/ report). - recover_stalled_tag_eval_runs sweep + retention (keep last 20) + 5-min beat (rule 89). scikit-learn added to requirements-ml. - tests: param normalization + the rehydrate read-path + create/conflict. Frontend admin card (trigger + render persisted report) follows next. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -29,6 +29,7 @@ from .subscribestar_seen_media import SubscribeStarSeenMedia
|
||||
from .tag import Tag, TagKind, image_tag
|
||||
from .tag_alias import TagAlias
|
||||
from .tag_allowlist import TagAllowlist
|
||||
from .tag_eval_run import TagEvalRun
|
||||
from .tag_reference_embedding import TagReferenceEmbedding
|
||||
from .tag_suggestion_rejection import TagSuggestionRejection
|
||||
from .task_run import TaskRun
|
||||
@@ -65,6 +66,7 @@ __all__ = [
|
||||
"MLSettings",
|
||||
"TagAlias",
|
||||
"TagAllowlist",
|
||||
"TagEvalRun",
|
||||
"TagReferenceEmbedding",
|
||||
"TagSuggestionRejection",
|
||||
"TaskRun",
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
"""TagEvalRun — persisted lifecycle of a head-vs-centroid tagging eval (#1130).
|
||||
|
||||
Mirrors LibraryAuditRun so the result SURVIVES navigation: the run + its full
|
||||
report live in this row, and the admin card rehydrates from it on mount instead
|
||||
of holding the report in transient frontend state. State machine:
|
||||
running → ready / error. The async ml-queue task writes `report` (JSONB) when
|
||||
done; a maintenance recovery sweep flips a stalled `running` row to `error`.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import DateTime, Integer, String, Text, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class TagEvalRun(Base):
|
||||
__tablename__ = "tag_eval_run"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
# The eval parameters: {concepts: [...], curve_points: [...], neg_ratio,
|
||||
# cv_folds, ...} — echoed back so the report is self-describing.
|
||||
params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="running", index=True,
|
||||
)
|
||||
# running | ready | error
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now(),
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
)
|
||||
# The full result: per-concept metrics (head vs centroid), learning-curve
|
||||
# points, and example image ids. Null until the task finishes.
|
||||
report: Mapped[dict[str, Any] | None] = mapped_column(JSONB, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
# Last time the task made progress — the recovery sweep tells a live run
|
||||
# from a SIGKILL'd one by this (mirrors LibraryAuditRun).
|
||||
last_progress_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
)
|
||||
Reference in New Issue
Block a user