chore: retire the tag-eval harness — it proved the heads system, job done (operator-approved)
The head-vs-centroid eval (#1130) existed to prove the 'frozen embedding + trained head' spine; the operator accepted the tagging system and dropped the harness. Removed per rule 22: TagEvalCard + store, /api/tag_eval blueprint, tag_eval_run ml task, recover-stalled-tag-eval-runs sweep + beat entry, TagEvalRun model + table (migration 0073), and its tests. The eval's data loaders + metric helpers were NOT eval-specific — the nightly heads trainer runs on them — so they moved verbatim to services/ml/training_data.py (heads.py import updated; behavior unchanged). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
@@ -21,7 +21,6 @@ from ..models import (
|
||||
ImportTask,
|
||||
LibraryAuditRun,
|
||||
Source,
|
||||
TagEvalRun,
|
||||
TaskRun,
|
||||
)
|
||||
from ..utils.phash import compute_phash
|
||||
@@ -96,9 +95,6 @@ BACKUP_DB_STALL_THRESHOLD_MINUTES = 40
|
||||
# Library audit: scan_library_for_rule has time_limit=7500s (2h5m).
|
||||
# 2h15m gives a 10-min buffer.
|
||||
LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES = 135
|
||||
# tag-eval (#1130) has a 30-min soft limit; flag a run with no progress past 40.
|
||||
TAG_EVAL_STALL_THRESHOLD_MINUTES = 40
|
||||
TAG_EVAL_KEEP_RUNS = 20
|
||||
# head training (#114) has a 60-min soft limit; flag no-progress past 75.
|
||||
HEAD_TRAINING_STALL_THRESHOLD_MINUTES = 75
|
||||
HEAD_TRAINING_KEEP_RUNS = 20
|
||||
@@ -743,46 +739,6 @@ def recover_stalled_library_audit_runs() -> int:
|
||||
return recovered
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_tag_eval_runs")
|
||||
def recover_stalled_tag_eval_runs() -> int:
|
||||
"""Flip TagEvalRun rows stuck in 'running' past the stall threshold to
|
||||
'error', and prune old runs to the last TAG_EVAL_KEEP_RUNS (retention,
|
||||
rule 89). Runs every 5 min on the maintenance lane; no-op when idle."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
now = datetime.now(UTC)
|
||||
cutoff = now - timedelta(minutes=TAG_EVAL_STALL_THRESHOLD_MINUTES)
|
||||
with SessionLocal() as session:
|
||||
result = session.execute(
|
||||
update(TagEvalRun)
|
||||
.where(TagEvalRun.status == "running")
|
||||
.where(
|
||||
func.coalesce(TagEvalRun.last_progress_at, TagEvalRun.started_at)
|
||||
< cutoff
|
||||
)
|
||||
.values(
|
||||
status="error", finished_at=now,
|
||||
error=(
|
||||
f"stranded by recovery sweep (no progress for "
|
||||
f"{TAG_EVAL_STALL_THRESHOLD_MINUTES} min)"
|
||||
),
|
||||
)
|
||||
)
|
||||
# Retention: keep only the most recent N runs.
|
||||
keep = session.execute(
|
||||
select(TagEvalRun.id).order_by(TagEvalRun.id.desc())
|
||||
.limit(TAG_EVAL_KEEP_RUNS)
|
||||
).scalars().all()
|
||||
if keep:
|
||||
session.execute(
|
||||
delete(TagEvalRun).where(TagEvalRun.id.not_in(keep))
|
||||
)
|
||||
session.commit()
|
||||
recovered = result.rowcount or 0
|
||||
if recovered:
|
||||
log.info("recover_stalled_tag_eval_runs: recovered %d rows", recovered)
|
||||
return recovered
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_head_training_runs")
|
||||
def recover_stalled_head_training_runs() -> int:
|
||||
"""Flip HeadTrainingRun rows stuck in 'running' past the stall threshold to
|
||||
|
||||
@@ -250,51 +250,6 @@ def backfill(self) -> int:
|
||||
return enqueued
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.tag_eval_run",
|
||||
bind=True,
|
||||
# The head-vs-centroid eval (#1130) loads embeddings + fits sklearn heads
|
||||
# for several concepts — minutes, not seconds. Runs on the ml queue because
|
||||
# only that worker has numpy/scikit-learn.
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def tag_eval_run(self, run_id: int) -> str:
|
||||
"""Compute the eval report into the persisted TagEvalRun row so it survives
|
||||
navigation (the admin card rehydrates from the row, not transient state)."""
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from ..models import TagEvalRun
|
||||
from ..services.ml.tag_eval import run_eval
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
run = session.get(TagEvalRun, run_id)
|
||||
if run is None:
|
||||
return "missing"
|
||||
run.last_progress_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
try:
|
||||
report = run_eval(session, run.params)
|
||||
except SoftTimeLimitExceeded:
|
||||
run.status = "error"
|
||||
run.error = "timed out"
|
||||
run.finished_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
raise
|
||||
except Exception as exc:
|
||||
log.exception("tag_eval_run %d failed", run_id)
|
||||
run.status = "error"
|
||||
run.error = str(exc)
|
||||
run.finished_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
return "error"
|
||||
run.report = report
|
||||
run.status = "ready"
|
||||
run.finished_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
return "ready"
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.train_heads",
|
||||
bind=True,
|
||||
|
||||
Reference in New Issue
Block a user