chore: retire the tag-eval harness — it proved the heads system, job done (operator-approved)
CI / lint (push) Successful in 4s
CI / frontend-build (push) Successful in 19s
CI / backend-lint-and-test (push) Successful in 33s
CI / integration (push) Successful in 3m24s

The head-vs-centroid eval (#1130) existed to prove the 'frozen embedding +
trained head' spine; the operator accepted the tagging system and dropped the
harness. Removed per rule 22: TagEvalCard + store, /api/tag_eval blueprint,
tag_eval_run ml task, recover-stalled-tag-eval-runs sweep + beat entry,
TagEvalRun model + table (migration 0073), and its tests.

The eval's data loaders + metric helpers were NOT eval-specific — the nightly
heads trainer runs on them — so they moved verbatim to
services/ml/training_data.py (heads.py import updated; behavior unchanged).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
2026-07-02 12:41:24 -04:00
parent a7abcc41ca
commit eaea4308fc
17 changed files with 178 additions and 1091 deletions
-44
View File
@@ -21,7 +21,6 @@ from ..models import (
ImportTask,
LibraryAuditRun,
Source,
TagEvalRun,
TaskRun,
)
from ..utils.phash import compute_phash
@@ -96,9 +95,6 @@ BACKUP_DB_STALL_THRESHOLD_MINUTES = 40
# Library audit: scan_library_for_rule has time_limit=7500s (2h5m).
# 2h15m gives a 10-min buffer.
LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES = 135
# tag-eval (#1130) has a 30-min soft limit; flag a run with no progress past 40.
TAG_EVAL_STALL_THRESHOLD_MINUTES = 40
TAG_EVAL_KEEP_RUNS = 20
# head training (#114) has a 60-min soft limit; flag no-progress past 75.
HEAD_TRAINING_STALL_THRESHOLD_MINUTES = 75
HEAD_TRAINING_KEEP_RUNS = 20
@@ -743,46 +739,6 @@ def recover_stalled_library_audit_runs() -> int:
return recovered
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_tag_eval_runs")
def recover_stalled_tag_eval_runs() -> int:
"""Flip TagEvalRun rows stuck in 'running' past the stall threshold to
'error', and prune old runs to the last TAG_EVAL_KEEP_RUNS (retention,
rule 89). Runs every 5 min on the maintenance lane; no-op when idle."""
SessionLocal = _sync_session_factory()
now = datetime.now(UTC)
cutoff = now - timedelta(minutes=TAG_EVAL_STALL_THRESHOLD_MINUTES)
with SessionLocal() as session:
result = session.execute(
update(TagEvalRun)
.where(TagEvalRun.status == "running")
.where(
func.coalesce(TagEvalRun.last_progress_at, TagEvalRun.started_at)
< cutoff
)
.values(
status="error", finished_at=now,
error=(
f"stranded by recovery sweep (no progress for "
f"{TAG_EVAL_STALL_THRESHOLD_MINUTES} min)"
),
)
)
# Retention: keep only the most recent N runs.
keep = session.execute(
select(TagEvalRun.id).order_by(TagEvalRun.id.desc())
.limit(TAG_EVAL_KEEP_RUNS)
).scalars().all()
if keep:
session.execute(
delete(TagEvalRun).where(TagEvalRun.id.not_in(keep))
)
session.commit()
recovered = result.rowcount or 0
if recovered:
log.info("recover_stalled_tag_eval_runs: recovered %d rows", recovered)
return recovered
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_head_training_runs")
def recover_stalled_head_training_runs() -> int:
"""Flip HeadTrainingRun rows stuck in 'running' past the stall threshold to
-45
View File
@@ -250,51 +250,6 @@ def backfill(self) -> int:
return enqueued
@celery.task(
name="backend.app.tasks.ml.tag_eval_run",
bind=True,
# The head-vs-centroid eval (#1130) loads embeddings + fits sklearn heads
# for several concepts — minutes, not seconds. Runs on the ml queue because
# only that worker has numpy/scikit-learn.
soft_time_limit=1800, time_limit=2100,
)
def tag_eval_run(self, run_id: int) -> str:
"""Compute the eval report into the persisted TagEvalRun row so it survives
navigation (the admin card rehydrates from the row, not transient state)."""
from datetime import UTC, datetime
from ..models import TagEvalRun
from ..services.ml.tag_eval import run_eval
SessionLocal = _sync_session_factory()
with SessionLocal() as session:
run = session.get(TagEvalRun, run_id)
if run is None:
return "missing"
run.last_progress_at = datetime.now(UTC)
session.commit()
try:
report = run_eval(session, run.params)
except SoftTimeLimitExceeded:
run.status = "error"
run.error = "timed out"
run.finished_at = datetime.now(UTC)
session.commit()
raise
except Exception as exc:
log.exception("tag_eval_run %d failed", run_id)
run.status = "error"
run.error = str(exc)
run.finished_at = datetime.now(UTC)
session.commit()
return "error"
run.report = report
run.status = "ready"
run.finished_at = datetime.now(UTC)
session.commit()
return "ready"
@celery.task(
name="backend.app.tasks.ml.train_heads",
bind=True,